GameSpot Reviews Search Scraper

Try for free

Scrape game reviews from GameSpot.com by providing a search URL, with automatic pagination and comprehensive review extraction.

Pricing

Pay per usage

Rating

5.0

(1)

Developer

PowerAI

Actor stats

Bookmarked

Total users

Monthly active users

24 days ago

Last modified

.actor/Dockerfile

# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome

# Check preinstalled packages
RUN npm ls apify

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent

.actor/actor.json

{
    "actorSpecification": 1,
    "name": "gamespot-reviews-search-scraper",
    "title": "Empty JavaScript project",
    "description": "Empty project in JavaScript.",
    "version": "0.0",
    "buildTag": "latest",
    "meta": {
        "templateId": "js-empty"
    },
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile"
}

.actor/input_schema.json

{
  "type": "object",
  "schemaVersion": 1,
  "title": "GameSpot Reviews Search Input",
  "properties": {
    "searchUrl": {
      "title": "Search URL",
      "type": "string",
      "description": "The complete GameSpot search URL to scrape",
      "minLength": 1,
      "editor": "textfield",
      "default": "https://www.gamespot.com/search/?i=reviews&q=war&page=1",
      "prefill": "https://www.gamespot.com/search/?i=reviews&q=war&page=1"
    },
    "maxItems": {
      "title": "Max Items",
      "type": "integer",
      "description": "Maximum number of reviews to fetch",
      "default": 50,
      "minimum": 1,
      "maximum": 10000,
      "editor": "number",
      "prefill": 50
    },
    "proxyConfiguration": {
      "title": "Proxy Configuration",
      "type": "object",
      "description": "Proxy settings for the actor",
      "default": {
        "useApifyProxy": false,
        "apifyProxyGroups": [
          "RESIDENTIAL"
        ]
      },
      "editor": "proxy"
    }
  },
  "required": [
    "searchUrl"
  ]
}

src/main.js

1import { Actor } from 'apify';
2import { connect as connectRealBrowser } from 'puppeteer-real-browser';
3import * as cheerio from 'cheerio';
4
5// ============================================================================
6// 配置区域 - 可根据不同网站修改
7// ============================================================================
8const SCRAPER_CONFIG = {
9    // 无新数据时的最大等待次数
10    maxNoNewDataCount: 3,
11    // 无头浏览器
12    headless: false,
13    // 
14    turnstile: true,
15    // 过滤资源
16    filterResources: false,
17    // 最大数量
18    maxItems: 50,
19    // 请求url
20    requestUrl: 'https://www.gamespot.com/search/?i=reviews&q=war',
21    // TODO4: 用于去重的唯一标识符字段名
22    uniqueIdField: 'reviewUrl',
23    // TODO3: 等待的选择器（列表元素选择器）
24    waitSelector: 'li.media.clearfix',
25    // TODO2: 自动翻页函数
26    autoScrollPage: async (page) => {
27        /**
28         * 翻页的 DOM 结构：
29         * 见 翻页DOM.html 文件
30         */
31        const nextButton = await page.$('a[title="Next page"]');
32        if (nextButton) {
33            await nextButton.click();
34        }
35    }
36};
37
38// ============================================================================
39// 核心 Scraper 类 - 模板部分（一般不需要修改）
40// ============================================================================
41class BaseScraper {
42    /**
43     * 解析页面数据 - 需要自定义
44     * @param {CheerioAPI} $ - Cheerio 实例
45     * @returns {Array} - 解析后的数据数组
46     * 
47     * ⚠️ 自动生成的解析代码应该替换此方法的实现
48     */
49    parsePageItems($) {
50        // ==================== 开始：页面解析代码（可替换区域） ====================
51
52        const items = $(SCRAPER_CONFIG.waitSelector).map((i, el) => {
53            /**
54             * 数据列表的 DOM 结构：
55             * 见 数据列表DOM.html 文件
56             */
57            const $item = $(el);
58            const titleLink = $item.find('h4.media-title a');
59            const imageLink = $item.find('.media-figure a');
60            const image = $item.find('.media-img img');
61            const dateSpan = $item.find('time.media-date span[itemprop="datePublished"]');
62            const deck = $item.find('p.media-deck');
63
64            return {
65                // TODO1: 按页面 DOM 结构，提取所有有价值数据
66                reviewTitle: titleLink.text().trim() || '',
67                reviewUrl: imageLink.attr('href') ? 'https://www.gamespot.com' + imageLink.attr('href') : '',
68                imageUrl: image.attr('src') || '',
69                publishedDate: dateSpan.text().trim() || '',
70                deck: deck.text().trim() || ''
71            };
72        }).get();
73
74        // ==================== 结束：页面解析代码（可替换区域） ====================
75
76        return items;
77    }
78
79    async run(input) {
80        const { searchUrl, maxItems = Infinity, proxyConfiguration } = input;
81        this.maxItems = maxItems;
82        this.searchUrl = searchUrl;
83        const proxyConfig = await Actor.createProxyConfiguration(proxyConfiguration);
84        const proxyUrl = proxyConfig ? await proxyConfig.newUrl() : undefined;
85        const proxyOptions = this.buildProxyOptions(proxyUrl);
86
87        const realBrowserOption = {
88            args: ["--start-maximized"],
89            turnstile: SCRAPER_CONFIG.turnstile,
90            headless: SCRAPER_CONFIG.headless,
91            // disableXvfb: true,
92            // ignoreAllFlags:true,
93            customConfig: {},
94            connectOption: {
95                defaultViewport: null
96            },
97            ...(proxyOptions ? { proxy: proxyOptions } : {}),
98            plugins: []
99        }
100
101        const { page, browser } = await connectRealBrowser(realBrowserOption);
102
103        try {
104            page.setDefaultNavigationTimeout(300 * 1000);
105            page.setDefaultTimeout(3600 * 1000);
106
107            // if (userAgent) {
108            //     await page.setUserAgent(userAgent);
109            // }
110
111            await this.handlePage(page, this.searchUrl);
112        } finally {
113            await browser?.close();
114        }
115    }
116
117    async handlePage(page, url) {
118        console.log(`Processing ${url}...`);
119        let count = 0;
120        let endOfResults = false;
121        let interceptedData = [];
122        let lastDataLength = 0;
123        let noNewDataCount = 0;
124
125        // await this.setupInterceptors(page, interceptedData);
126        await page.goto(url);
127
128        // 等待新内容加载
129        if (SCRAPER_CONFIG.waitSelector) {
130            await page.waitForSelector(SCRAPER_CONFIG.waitSelector, { timeout: 60000 }).catch(() => {
131                console.log(`data not found...`);
132            });
133        }
134
135        // 处理首页数据
136        const firstPageHtml = await page.content();
137        await this.processResponseData(firstPageHtml, interceptedData);
138
139        // 循环翻页获取数据
140        while (!endOfResults && interceptedData.length < this.maxItems) {
141            count++;
142            console.log(`data fetching...${count}`);
143            await this.randomDelay();
144
145            // 执行翻页操作
146            await SCRAPER_CONFIG.autoScrollPage(page);
147            await this.randomDelay(3000, 5000);
148
149            // 等待新内容加载
150            if (SCRAPER_CONFIG.waitSelector) {
151                await page.waitForSelector(SCRAPER_CONFIG.waitSelector, { timeout: 10000 }).catch(() => {
152                    console.log(`data not found...`);
153                });
154            }
155
156            // 处理新页面数据
157            const afterScrollHtml = await page.content();
158            await this.processResponseData(afterScrollHtml, interceptedData);
159
160            if (interceptedData.length === lastDataLength) {
161                noNewDataCount++;
162                if (noNewDataCount >= SCRAPER_CONFIG.maxNoNewDataCount) {
163                    endOfResults = true;
164                }
165            } else {
166                noNewDataCount = 0;
167                lastDataLength = interceptedData.length;
168            }
169        }
170
171        console.log(`Scraping completed. Total items: ${interceptedData.length}`);
172    }
173
174    // 设置请求拦截器（资源过滤）
175    async setupInterceptors(page, interceptedData) {
176        await page.setRequestInterception(true);
177        page.on('request', (request) => {
178            const resourceType = request.resourceType();
179            const blockedResourceTypes = [
180                'image',
181                'media',
182                'font',
183                'texttrack',
184                'stylesheet',
185                'ping'
186            ];
187
188            if (SCRAPER_CONFIG.filterResources && blockedResourceTypes.includes(resourceType)) {
189                request.abort();
190            } else {
191                request.continue();
192            }
193        });
194        page.on('response', async (response) => {
195            // 可选：拦截特定的 API 响应
196        });
197    }
198
199    // 处理响应数据（模板方法）
200    async processResponseData(htmlContent, interceptedData) {
201        let currentData = [];
202
203        if (!htmlContent) {
204            return;
205        }
206
207        const $ = cheerio.load(htmlContent);
208
209        // 调用自定义的解析方法
210        const items = this.parsePageItems($);
211
212        if (!items || items.length === 0) {
213            return;
214        }
215
216        // 去重处理
217        const uniqueField = SCRAPER_CONFIG.uniqueIdField;
218        const existingIds = new Set(interceptedData.map(item => item[uniqueField]));
219
220        let skippedCount = 0;
221        for (const listing of items) {
222            // 跳过已存在的项目
223            if (existingIds.has(listing[uniqueField])) {
224                skippedCount++;
225                continue;
226            }
227
228            const data = {
229                searchUrl: this.searchUrl,
230                ...listing,
231                scrapedAt: new Date().toISOString()
232            };
233
234            interceptedData.push(data);
235            currentData.push(data);
236            existingIds.add(listing[uniqueField]);
237
238            // 如果达到最大数量，停止处理
239            if (interceptedData.length >= this.maxItems) {
240                break;
241            }
242        }
243
244
245        // 保存数据
246        if (currentData.length > 0) {
247            console.log(`Saved ${currentData.length} items, total: ${interceptedData.length}`);
248            await Actor.pushData(currentData);
249        }
250    }
251
252    // 随机延迟
253    async randomDelay(min = 1000, max = 3000) {
254        const delay = Math.floor(Math.random() * (max - min + 1) + min);
255        await new Promise(resolve => setTimeout(resolve, delay));
256    }
257
258    // 可选：设置 cookies
259    async setCookies(page) {
260        // const cookies = [
261        //     {
262        //         name: 'cookie_name',
263        //         value: 'cookie_value',
264        //         domain: '.example.com',
265        //         path: '/',
266        //         expires: new Date('2025-12-31').getTime() / 1000,
267        //         secure: false
268        //     }
269        // ];
270        // await Promise.all(cookies.map(cookie => page.setCookie(cookie)));
271    }
272
273
274    buildProxyOptions(proxyUrl) {
275        if (!proxyUrl) {
276            return undefined;
277        }
278
279        try {
280            const parsed = new URL(proxyUrl);
281            if (!parsed.hostname || !parsed.port) {
282                return undefined;
283            }
284
285            return {
286                host: parsed.hostname,
287                port: Number(parsed.port),
288                username: parsed.username || undefined,
289                password: parsed.password || undefined,
290            };
291        } catch (error) {
292            console.warn(`Invalid proxy URL detected: ${proxyUrl}`, error);
293            return undefined;
294        }
295    }
296
297}
298
299
300// ============================================================================
301// Actor 入口
302// ============================================================================
303await Actor.init();
304
305Actor.main(async () => {
306    let input = await Actor.getInput();
307    if (!input) {
308        input = {
309            "searchUrl": SCRAPER_CONFIG.requestUrl,
310            "maxItems": SCRAPER_CONFIG.maxItems
311        };
312    }
313    const crawler = new BaseScraper();
314    await crawler.run(input);
315});

.dockerignore

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git

.editorconfig

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf

.eslintrc

{
    "extends": "@apify",
    "root": true
}

.gitignore

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
.zed
node_modules
storage

package.json

{
    "name": "js-scrape-single-page",
    "version": "0.0.1",
    "type": "module",
    "description": "This is an example of an Apify actor.",
    "engines": {
        "node": ">=18.0.0"
    },
    "dependencies": {
		"apify": "^3.4.4",
		"crawlee": "^3.11.5",
		"puppeteer": "^23.5.0",
		"puppeteer-core": "^23.5.0",
		"puppeteer-extra": "^3.3.6",
		"puppeteer-extra-plugin-stealth": "^2.11.2",
		"random-useragent": "^0.5.0",
        "rebrowser-puppeteer": "^23.10.3",
		"rebrowser-puppeteer-core": "^23.10.3",
        "puppeteer-real-browser": "^1.4.4",
        "ioredis": "^5.4.1",
        "marked": "^15.0.7",
		"node-fetch": "3",
        "uuid": "^11.1.0"
    },
    "scripts": {
        "start": "node ./src/main.js",
        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
    },
    "author": "It's not you it's me",
    "license": "ISC"
}

GameSpot Video Game Reviews Scraper

fortuitous_pirate/gamespot-scraper

Scrape video game reviews from GameSpot. Get ratings, reviews, platforms, and release dates. Perfect for gaming apps and price trackers.

Fortuitous Pirate

Shopify App Reviews Scraper

powerai/shopify-app-reviews-scraper

Scrape customer reviews from Shopify App Store by providing a review page URL, with automatic pagination and comprehensive review information extraction.