Crawlee + Cheerio

1// Apify SDK - toolkit for building Apify Actors (Read more at https://docs.apify.com/sdk/js/)
2import { Actor } from 'apify';
3// Crawlee - web scraping and browser automation library (Read more at https://crawlee.dev)
4import { CheerioCrawler, Dataset } from 'crawlee';
5// this is ESM project, and as such, it requires you to specify extensions in your relative imports
6// read more about this here: https://nodejs.org/docs/latest-v18.x/api/esm.html#mandatory-file-extensions
7// import { router } from './routes.js';
8
9// The init() call configures the Actor for its environment. It's recommended to start every Actor with an init()
10await Actor.init();
11
12// Structure of input is defined in input_schema.json
13const {
14    startUrls = ['https://crawlee.dev'],
15    maxRequestsPerCrawl = 100,
16} = await Actor.getInput() ?? {};
17
18const proxyConfiguration = await Actor.createProxyConfiguration();
19
20const crawler = new CheerioCrawler({
21    proxyConfiguration,
22    maxRequestsPerCrawl,
23    async requestHandler({ enqueueLinks, request, $, log }) {
24        log.info('enqueueing new URLs');
25        await enqueueLinks();
26
27        // Extract title from the page.
28        const title = $('title').text();
29        log.info(`${title}`, { url: request.loadedUrl });
30
31        // Save url and title to Dataset - a table-like storage.
32        await Dataset.pushData({ url: request.loadedUrl, title });
33    },
34});
35
36await crawler.run(startUrls);
37
38// Gracefully exit the Actor process. It's recommended to quit all Actors with an exit()
39await Actor.exit();