Shopee Ratings Crawler

The Shopee Review Scraper is a web scraping tool designed to extract ratings and reviews from Shopee product pages. It utilizes asynchronous requests to fetch reviews efficiently and can be run as an Apify Actor for automated data extraction.

Features

• Extracts item_id and shop_id from Shopee product URLs.
• Fetches ratings for a given Shopee item and shop using the Shopee API.
• Utilizes multiprocessing to fetch data in parallel.
• Integrates with Apify for automation and data storage.

How to use

Example JSON output from a scraping session:

https://api.apify.com/v2/datasets/qdassZVbgrxkIeOzq/items?clean=true&format=html&limit=1000

[
    {
        "orderid": 0,
        "itemid": 26552068706,
        "cmtid": 16370083915,
        "ctime": 1725119465,
        "rating": 1,
        "userid": 1231005243,
        "shopid": 882565700,
        "comment": "Đúng với mô tả:giống hình\nChất lượng sản phẩm:ok\n\nSản phẩm đẹp nha mọi người,đeo vào nhìn rất thời trang và lạ mắt nhé kkkkk",
        "rating_star": 5,
        "status": 2,
        "mtime": 1732333505,
        "editable": 1,
        "opt": 2,
        "filter": 7,
        "is_hidden": false,
        "submit_time": 1725119465,
        "origin_itemid": 0,
        "origin_shopid": 0,
        "author_username": "gwgfdsqoa2",
        "author_portrait": "https://down-vn.img.susercontent.com/file/",
        "author_shopid": 1230597245,
        "anonymous": false,
        "images": [
            "https://down-vn.img.susercontent.com/file/vn-11134103-7r98o-lzm4zlanjrkd6d"
        ],
        "videos": [
            {
                "id": "vn-11110103-6khw9-lzm4zqnhqz4dab",
                "cover": "https://down-bs-sg.img.susercontent.com/vn-11110103-7r98o-lzm4zqnhsdot5f",
                "url": "https://down-tx-sg.vod.susercontent.com/api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16000051725119411.mp4",
                "duration": 10008,
                "mms_ext": "{\"vid\":\"vn-11110103-6khw9-lzm4zqnhqz4dab\",\"biz\":103,\"duration\":10008,\"create_time\":1725119410,\"serviceID\":6,\"cover\":\"https://down-bs-sg.img.susercontent.com/vn-11110103-7r98o-lzm4zqnhsdot5f\",\"cover_list\":null,\"formats\":[{\"format\":1600005,\"defn\":\"V640P\",\"profile\":\"MP4\",\"path\":\"api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16000051725119411.mp4\",\"url\":\"https://down-tx-sg.vod.susercontent.com/api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16000051725119411.mp4\",\"width\":360,\"height\":640,\"bitrate\":382748,\"size\":605570,\"fps\":25.10081,\"duration\":10008,\"update_time\":1725119413}],\"default_format\":{\"format\":1600323,\"defn\":\"V540P\",\"profile\":\"MP4\",\"path\":\"\",\"url\":\"https://mms.vod.susercontent.com/api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.default.mp4\",\"width\":360,\"height\":640,\"bitrate\":381872,\"size\":564426,\"fps\":25.10081,\"duration\":10008,\"update_time\":1725119426},\"extend_formats\":{\"H265\":[{\"format\":1600308,\"defn\":\"V640P\",\"profile\":\"MP4\",\"path\":\"api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16003081726058115.mp4\",\"url\":\"https://down-tx-sg.vod.susercontent.com/api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16003081726058115.mp4\",\"width\":360,\"height\":640,\"bitrate\":150052,\"size\":359344,\"fps\":25.10081,\"duration\":10008,\"update_time\":1726058120}]},\"ab_test\":\"\"}",
                "cover_image_id": "sg-11134206-7rdx9-lzox5k7n3l964c",
                "highlight": {
                    "id": "vn-11110103-6khw9-lzm4zqnhqz4dab",
                    "cover": "https://down-bs-sg.img.susercontent.com/vn-11110103-7r98o-lzm4zqnhsdot5f",
                    "url": "https://down-tx-sg.vod.susercontent.com/api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16000051725119411.mp4",
                    "duration": 10008,
                    "mms_ext": "{\"vid\":\"vn-11110103-6khw9-lzm4zqnhqz4dab\",\"biz\":103,\"duration\":10008,\"create_time\":1725119410,\"serviceID\":6,\"cover\":\"https://down-bs-sg.img.susercontent.com/vn-11110103-7r98o-lzm4zqnhsdot5f\",\"cover_list\":null,\"formats\":[{\"format\":1600005,\"defn\":\"V640P\",\"profile\":\"MP4\",\"path\":\"api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16000051725119411.mp4\",\"url\":\"https://down-tx-sg.vod.susercontent.com/api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16000051725119411.mp4\",\"width\":360,\"height\":640,\"bitrate\":382748,\"size\":605570,\"fps\":25.10081,\"duration\":10008,\"update_time\":1725119413}],\"default_format\":{\"format\":1600323,\"defn\":\"V540P\",\"profile\":\"MP4\",\"path\":\"\",\"url\":\"https://mms.vod.susercontent.com/api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.default.mp4\",\"width\":360,\"height\":640,\"bitrate\":381872,\"size\":564426,\"fps\":25.10081,\"duration\":10008,\"update_time\":1725119426},\"extend_formats\":{\"H265\":[{\"format\":1600308,\"defn\":\"V640P\",\"profile\":\"MP4\",\"path\":\"api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16003081726058115.mp4\",\"url\":\"https://down-tx-sg.vod.susercontent.com/api/v4/11110103/mms/vn-11110103-6khw9-lzm4zqnhqz4dab.16003081726058115.mp4\",\"width\":360,\"height\":640,\"bitrate\":150052,\"size\":359344,\"fps\":25.10081,\"duration\":10008,\"update_time\":1726058120}]},\"ab_test\":\"\"}",
                    "cover_image_id": "sg-11134206-7rdx9-lzox5k7n3l964c"
                }
            }
        ],
        "product_items": [
            {
                "itemid": 26552068706,
                "shopid": 882565700,
                "name": "Ốp điện thoại silicon mờ lỏng chống trượt dày màu đỏ cho IP IPhone 7 8 14 15 Plus X XR XS Max 11 12 13 14 15 Pro Max 11prm 12prm 13prm 14prm 15prm Logo Apple cho bé gái",
                "image": "sg-11134201-7rd4w-lw35effggpyb1a",
                "is_snapshot": 1,
                "snapshotid": 37127943020,
                "modelid": 89580585035,
                "model_name": "Big Red & LOGO,Iphone 12Pro",
                "options": [
                    "Big Red & LOGO",
                    "Iphone 12Pro"
                ]
            }
        ],
        "ItemRatingReply": {
            "orderid": 0,
            "cmtid": 16370083915,
            "ctime": 1725119598,
            "userid": 882603510,
            "comment": "Thank you for your support. Look forward to your next visit.",
            "mtime": 1725119598,
            "is_hidden": false
        },
        "editable_date": 1727711465,
        "like_count": 5,
        "sync_to_social": false,
        "detailed_rating": {
            "product_quality": 5,
            "seller_service": 5,
            "delivery_service": 5
        },
        "exclude_scoring_due_low_logistic": false,
        "template_tags": [
            "Đúng với mô tả",
            "Chất lượng sản phẩm"
        ],
        "has_template_tag": true,
        "sip_info": {
            "is_oversea": false,
            "origin_region": "vn"
        },
        "is_repeated_purchase": false,
        "display_variation_filter": true,
        "overall_fit": 0,
        "is_normal_item": true,
        "size_info_abt": "",
        "image_data": [
            {
                "image_id": "vn-11134103-7r98o-lzm4zlanjrkd6d",
                "cover_image_id": "vn-11134103-7r98o-lzm4zlanjrkd6d"
            }
        ],
        "key_media": {
            "key_media_code": 0,
            "key_media_id": ""
        },
        "is_super_review": false,
        "region": "VN",
        "original_item_info": {
            "itemid": 26552068706,
            "shopid": 882565700,
            "name": "Ốp điện thoại silicon mờ lỏng chống trượt dày màu đỏ cho IP IPhone 7 8 14 15 Plus X XR XS Max 11 12 13 14 15 Pro Max 11prm 12prm 13prm 14prm 15prm Logo Apple cho bé gái",
            "image": "sg-11134201-7rd4w-lw35effggpyb1a"
        },
        "show_super_review_tag": false,
        "super_review_status": 0,
        "review_type": 0
    }
]