Pricing

Pay per usage

Try for free

Go to Apify Store

NovelUpdates Series Search Scraper

Try for free

Search and discover web novels on NovelUpdates.com with automatic pagination and comprehensive data extraction.

Pricing

Pay per usage

Rating

5.0

(1)

Developer

PowerAI

Actor stats

Bookmarked

Total users

Monthly active users

6 days ago

Last modified

.actor/Dockerfile

# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome

# Check preinstalled packages
RUN npm ls apify

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent

.actor/actor.json

{
    "actorSpecification": 1,
    "name": "novelupdates-series-search-scraper",
    "title": "Empty JavaScript project",
    "description": "Empty project in JavaScript.",
    "version": "0.0",
    "buildTag": "latest",
    "meta": {
        "templateId": "js-empty"
    },
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile"
}

.actor/input_schema.json

{
  "type": "object",
  "schemaVersion": 1,
  "title": "NovelUpdates Series Search Input",
  "properties": {
    "searchUrl": {
      "title": "Search URL",
      "type": "string",
      "description": "The complete NovelUpdates series finder URL to scrape",
      "minLength": 1,
      "editor": "textfield",
      "default": "https://www.novelupdates.com/series-finder/?sf=1&sh=live&sort=sdate&order=desc",
      "prefill": "https://www.novelupdates.com/series-finder/?sf=1&sh=live&sort=sdate&order=desc"
    },
    "maxItems": {
      "title": "Max Items",
      "type": "integer",
      "description": "Maximum number of novels to fetch",
      "default": 50,
      "prefill": 50,
      "minimum": 1,
      "maximum": 10000,
      "editor": "number"
    },
    "proxyConfiguration": {
      "title": "Proxy Configuration",
      "type": "object",
      "description": "Proxy settings for the actor",
      "default": {
        "useApifyProxy": false,
        "apifyProxyGroups": [
          "RESIDENTIAL"
        ]
      },
      "editor": "proxy"
    }
  },
  "required": [
    "searchUrl"
  ]
}

src/main.js

1import { Actor } from 'apify';
2import { connect as connectRealBrowser } from 'puppeteer-real-browser';
3import * as cheerio from 'cheerio';
4
5// ============================================================================
6// 配置区域 - 可根据不同网站修改
7// ============================================================================
8const SCRAPER_CONFIG = {
9    // 无新数据时的最大等待次数
10    maxNoNewDataCount: 3,
11    // 无头浏览器
12    headless: false,
13    // 
14    turnstile: true,
15    // 过滤资源
16    filterResources: false,
17    // 最大数量
18    maxItems: 50,
19    // 请求url
20    requestUrl: 'https://www.novelupdates.com/series-finder/?sf=1&sh=live&sort=sdate&order=desc',
21    // TODO4: 用于去重的唯一标识符字段名
22    uniqueIdField: 'seriesId',
23    // TODO3: 等待的选择器（列表元素选择器）
24    waitSelector: '.search_main_box_nu',
25    // TODO2: 自动翻页函数
26    autoScrollPage: async (page) => {
27        /**
28         * 翻页的 DOM 结构：
29         * 见 翻页DOM.html 文件
30         */
31        const nextButton = await page.$('a.next_page');
32        if (nextButton) {
33            await nextButton.click();
34            await page.waitForSelector('.search_main_box_nu', { timeout: 10000 }).catch(() => {});
35        }
36    }
37};
38
39// ============================================================================
40// 核心 Scraper 类 - 模板部分（一般不需要修改）
41// ============================================================================
42class BaseScraper {
43    /**
44     * 解析页面数据 - 需要自定义
45     * @param {CheerioAPI} $ - Cheerio 实例
46     * @returns {Array} - 解析后的数据数组
47     * 
48     * ⚠️ 自动生成的解析代码应该替换此方法的实现
49     */
50    parsePageItems($) {
51        // ==================== 开始：页面解析代码（可替换区域） ====================
52
53        const items = $(SCRAPER_CONFIG.waitSelector).map((i, el) => {
54            /**
55             * 数据列表的 DOM 结构：
56             * 见 数据列表DOM.html 文件
57             */
58            const $item = $(el);
59
60            // 提取图片URL中的series ID
61            const imageSrc = $item.find('.search_img_nu img').attr('src') || '';
62            const seriesIdMatch = imageSrc.match(/series_(\d+)\.jpg/);
63            const seriesId = seriesIdMatch ? seriesIdMatch[1] : '';
64
65            // 提取标题和链接
66            const titleLink = $item.find('.search_title a');
67            const title = titleLink.text().trim();
68            const seriesUrl = titleLink.attr('href') || '';
69
70            // 提取图片URL
71            const imageUrl = imageSrc;
72
73            // 提取评分
74            const ratingText = $item.find('.search_ratings').text().trim();
75            const ratingMatch = ratingText.match(/[\d.]+/);
76            const rating = ratingMatch ? ratingMatch[0] : '';
77
78            // 提取语言
79            const langSpan = $item.find('.orgjp, .orgcn, .orgkr');
80            const language = langSpan.text().trim();
81
82            // 提取统计信息
83            const stats = {};
84            $item.find('.search_stats .ss_desk').each((idx, statEl) => {
85                const $stat = $(statEl);
86                const title = $stat.find('i').attr('title') || '';
87                const text = $stat.clone().children().remove().end().text().trim();
88
89                if (title.includes('Chapter')) {
90                    stats.chapterCount = text.replace(/[^0-9]/g, '');
91                } else if (title.includes('Frequency')) {
92                    stats.updateFrequency = text;
93                } else if (title.includes('Readers')) {
94                    stats.readerCount = text.replace(/[^0-9]/g, '');
95                } else if (title.includes('Reviews')) {
96                    stats.reviewCount = text.replace(/[^0-9]/g, '');
97                } else if (title.includes('Last Updated')) {
98                    stats.lastUpdated = text;
99                }
100            });
101
102            // 提取类型
103            const genres = [];
104            $item.find('.search_genre a').each((idx, genreEl) => {
105                genres.push({
106                    name: $(genreEl).text().trim(),
107                    url: $(genreEl).attr('href') || '',
108                    gid: $(genreEl).attr('gid') || ''
109                });
110            });
111
112            // 提取描述
113            let description = $item.find('.search_body_nu').contents().filter(function() {
114                return this.nodeType === 3 && this.textContent?.trim()?.length > 0;
115            }).text().trim();
116
117            if (!description) {
118                description = $item.find('.search_body_nu').text().split('\n').map(line => line.trim()).filter(line => line.length > 20 && !line.includes(' Chapters') && !line.includes(' Day') && !line.includes(' Readers') && !line.includes(' Reviews') && !line.includes(' Updated')).join(' ').trim();
119            }
120
121            return {
122                seriesId: seriesId,
123                title: title,
124                seriesUrl: seriesUrl,
125                imageUrl: imageUrl,
126                rating: rating,
127                language: language,
128                chapterCount: parseInt(stats.chapterCount) || 0,
129                updateFrequency: stats.updateFrequency || '',
130                readerCount: parseInt(stats.readerCount) || 0,
131                reviewCount: parseInt(stats.reviewCount) || 0,
132                lastUpdated: stats.lastUpdated || '',
133                genres: genres,
134                description: description
135            };
136        }).get();
137
138        // ==================== 结束：页面解析代码（可替换区域） ====================
139
140        return items;
141    }
142
143    async run(input) {
144        const { searchUrl, maxItems = Infinity, proxyConfiguration } = input;
145        this.maxItems = maxItems;
146        this.searchUrl = searchUrl;
147        const proxyConfig = await Actor.createProxyConfiguration(proxyConfiguration);
148        const proxyUrl = proxyConfig ? await proxyConfig.newUrl() : undefined;
149        const proxyOptions = this.buildProxyOptions(proxyUrl);
150
151        const realBrowserOption = {
152            args: ["--start-maximized"],
153            turnstile: SCRAPER_CONFIG.turnstile,
154            headless: SCRAPER_CONFIG.headless,
155            // disableXvfb: true,
156            // ignoreAllFlags:true,
157            customConfig: {},
158            connectOption: {
159                defaultViewport: null
160            },
161            ...(proxyOptions ? { proxy: proxyOptions } : {}),
162            plugins: []
163        }
164
165        const { page, browser } = await connectRealBrowser(realBrowserOption);
166
167        try {
168            page.setDefaultNavigationTimeout(300 * 1000);
169            page.setDefaultTimeout(3600 * 1000);
170
171            // if (userAgent) {
172            //     await page.setUserAgent(userAgent);
173            // }
174
175            await this.handlePage(page, this.searchUrl);
176        } finally {
177            await browser?.close();
178        }
179    }
180
181    async handlePage(page, url) {
182        console.log(`Processing ${url}...`);
183        let count = 0;
184        let endOfResults = false;
185        let interceptedData = [];
186        let lastDataLength = 0;
187        let noNewDataCount = 0;
188
189        // await this.setupInterceptors(page, interceptedData);
190        await page.goto(url);
191
192        // 等待新内容加载
193        if (SCRAPER_CONFIG.waitSelector) {
194            await page.waitForSelector(SCRAPER_CONFIG.waitSelector, { timeout: 60000 }).catch(() => {
195                console.log(`data not found...`);
196            });
197        }
198
199        // 处理首页数据
200        const firstPageHtml = await page.content();
201        await this.processResponseData(firstPageHtml, interceptedData);
202
203        // 循环翻页获取数据
204        while (!endOfResults && interceptedData.length < this.maxItems) {
205            count++;
206            console.log(`data fetching...${count}`);
207            await this.randomDelay();
208
209            // 执行翻页操作
210            await SCRAPER_CONFIG.autoScrollPage(page);
211            await this.randomDelay(3000, 5000);
212
213            // 等待新内容加载
214            if (SCRAPER_CONFIG.waitSelector) {
215                await page.waitForSelector(SCRAPER_CONFIG.waitSelector, { timeout: 10000 }).catch(() => {
216                    console.log(`data not found...`);
217                });
218            }
219
220            // 处理新页面数据
221            const afterScrollHtml = await page.content();
222            await this.processResponseData(afterScrollHtml, interceptedData);
223
224            if (interceptedData.length === lastDataLength) {
225                noNewDataCount++;
226                if (noNewDataCount >= SCRAPER_CONFIG.maxNoNewDataCount) {
227                    endOfResults = true;
228                }
229            } else {
230                noNewDataCount = 0;
231                lastDataLength = interceptedData.length;
232            }
233        }
234
235        console.log(`Scraping completed. Total items: ${interceptedData.length}`);
236    }
237
238    // 设置请求拦截器（资源过滤）
239    async setupInterceptors(page, interceptedData) {
240        await page.setRequestInterception(true);
241        page.on('request', (request) => {
242            const resourceType = request.resourceType();
243            const blockedResourceTypes = [
244                'image',
245                'media',
246                'font',
247                'texttrack',
248                'stylesheet',
249                'ping'
250            ];
251
252            if (SCRAPER_CONFIG.filterResources && blockedResourceTypes.includes(resourceType)) {
253                request.abort();
254            } else {
255                request.continue();
256            }
257        });
258        page.on('response', async (response) => {
259            // 可选：拦截特定的 API 响应
260        });
261    }
262
263    // 处理响应数据（模板方法）
264    async processResponseData(htmlContent, interceptedData) {
265        let currentData = [];
266
267        if (!htmlContent) {
268            return;
269        }
270
271        const $ = cheerio.load(htmlContent);
272
273        // 调用自定义的解析方法
274        const items = this.parsePageItems($);
275
276        if (!items || items.length === 0) {
277            return;
278        }
279
280        // 去重处理
281        const uniqueField = SCRAPER_CONFIG.uniqueIdField;
282        const existingIds = new Set(interceptedData.map(item => item[uniqueField]));
283
284        let skippedCount = 0;
285        for (const listing of items) {
286            // 跳过已存在的项目
287            if (existingIds.has(listing[uniqueField])) {
288                skippedCount++;
289                continue;
290            }
291
292            const data = {
293                searchUrl: this.searchUrl,
294                ...listing,
295                scrapedAt: new Date().toISOString()
296            };
297
298            interceptedData.push(data);
299            currentData.push(data);
300            existingIds.add(listing[uniqueField]);
301
302            // 如果达到最大数量，停止处理
303            if (interceptedData.length >= this.maxItems) {
304                break;
305            }
306        }
307
308
309        // 保存数据
310        if (currentData.length > 0) {
311            console.log(`Saved ${currentData.length} items, total: ${interceptedData.length}`);
312            await Actor.pushData(currentData);
313        }
314    }
315
316    // 随机延迟
317    async randomDelay(min = 1000, max = 3000) {
318        const delay = Math.floor(Math.random() * (max - min + 1) + min);
319        await new Promise(resolve => setTimeout(resolve, delay));
320    }
321
322    // 可选：设置 cookies
323    async setCookies(page) {
324        // const cookies = [
325        //     {
326        //         name: 'cookie_name',
327        //         value: 'cookie_value',
328        //         domain: '.example.com',
329        //         path: '/',
330        //         expires: new Date('2025-12-31').getTime() / 1000,
331        //         secure: false
332        //     }
333        // ];
334        // await Promise.all(cookies.map(cookie => page.setCookie(cookie)));
335    }
336
337
338    buildProxyOptions(proxyUrl) {
339        if (!proxyUrl) {
340            return undefined;
341        }
342
343        try {
344            const parsed = new URL(proxyUrl);
345            if (!parsed.hostname || !parsed.port) {
346                return undefined;
347            }
348
349            return {
350                host: parsed.hostname,
351                port: Number(parsed.port),
352                username: parsed.username || undefined,
353                password: parsed.password || undefined,
354            };
355        } catch (error) {
356            console.warn(`Invalid proxy URL detected: ${proxyUrl}`, error);
357            return undefined;
358        }
359    }
360
361}
362
363
364// ============================================================================
365// Actor 入口
366// ============================================================================
367await Actor.init();
368
369Actor.main(async () => {
370    let input = await Actor.getInput();
371    if (!input) {
372        input = {
373            "searchUrl": SCRAPER_CONFIG.requestUrl,
374            "maxItems": SCRAPER_CONFIG.maxItems
375        };
376    }
377    const crawler = new BaseScraper();
378    await crawler.run(input);
379});

.dockerignore

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git

.editorconfig

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf

.eslintrc

{
    "extends": "@apify",
    "root": true
}

.gitignore

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
.zed
node_modules
storage

package.json

{
    "name": "js-scrape-single-page",
    "version": "0.0.1",
    "type": "module",
    "description": "This is an example of an Apify actor.",
    "engines": {
        "node": ">=18.0.0"
    },
    "dependencies": {
		"apify": "^3.4.4",
		"crawlee": "^3.11.5",
		"puppeteer": "^23.5.0",
		"puppeteer-core": "^23.5.0",
		"puppeteer-extra": "^3.3.6",
		"puppeteer-extra-plugin-stealth": "^2.11.2",
		"random-useragent": "^0.5.0",
        "rebrowser-puppeteer": "^23.10.3",
		"rebrowser-puppeteer-core": "^23.10.3",
        "puppeteer-real-browser": "^1.4.4",
        "ioredis": "^5.4.1",
        "marked": "^15.0.7",
		"node-fetch": "3",
        "uuid": "^11.1.0"
    },
    "scripts": {
        "start": "node ./src/main.js",
        "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
    },
    "author": "It's not you it's me",
    "license": "ISC"
}

Wegmans Products Search Scraper

powerai/wegmans-products-search-scraper

Search and discover products on Wegmans.com with automatic pagination and comprehensive product information extraction.

PowerAI

5.0

Softpedia Apps Search Scraper

powerai/softpedia-apps-search-scraper

Search and discover applications on Softpedia.com with comprehensive metadata extraction and automatic pagination.

PowerAI

5.0

Artsy Search Scraper

powerai/artsy-search-scraper

Search and discover artworks on Artsy.net with automatic pagination and comprehensive data extraction.

PowerAI

5.0

Action.com Search Scraper

powerai/action-com-search-scraper

Scrape products from Action.com by providing a search URL, with automatic pagination and comprehensive product information extraction.

PowerAI

5.0

Indiegogo Projects Search Scraper

powerai/indiegogo-projects-search-scraper

Search and discover crowdfunding projects on Indiegogo by providing a search URL, with automatic pagination and comprehensive project information extraction.

PowerAI

5.0

Apartments Search Scraper

powerai/apartments-search-scraper

Scrape apartment listings from Apartments.com with automatic pagination and comprehensive property data extraction.

PowerAI

5.0

Google Maps Search Scraper

powerai/google-map-search-scraper

Scrape Google Maps search results with automatic pagination and location-based search support

PowerAI

4.5

Web Search Scraper

yesintelligent/web-search-scraper

Advanced web search scraper and data extraction API that delivers real-time search results with comprehensive content snippets. Perfect for research, competitive analysis, content discovery, and automated information gathering. Extract structured data from web searches with high accuracy and speed.