TypeScript Playwright scraper

1/**
2 * This template is a production ready boilerplate for developing with `PlaywrightCrawler`.
3 * Use this to bootstrap your projects using the most up-to-date code.
4 * If you're looking for examples or want to learn more, see README.
5 */
6
7// For more information, see https://crawlee.dev
8import { PlaywrightCrawler } from '@crawlee/playwright';
9// For more information, see https://docs.apify.com/sdk/js
10import { Actor } from 'apify';
11
12// this is ESM project, and as such, it requires you to specify extensions in your relative imports
13// read more about this here: https://nodejs.org/docs/latest-v18.x/api/esm.html#mandatory-file-extensions
14// note that we need to use `.js` even when inside TS files
15import { router } from './routes.js';
16
17interface Input {
18    startUrls: {
19        url: string;
20        method?: 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH';
21        headers?: Record<string, string>;
22        userData: Record<string, unknown>;
23    }[];
24    maxRequestsPerCrawl: number;
25}
26
27// Initialize the Apify SDK
28await Actor.init();
29
30// Structure of input is defined in input_schema.json
31const { startUrls = ['https://apify.com'], maxRequestsPerCrawl = 100 } =
32    (await Actor.getInput<Input>()) ?? ({} as Input);
33
34// `checkAccess` flag ensures the proxy credentials are valid, but the check can take a few hundred milliseconds.
35// Disable it for short runs if you are sure your proxy configuration is correct
36const proxyConfiguration = await Actor.createProxyConfiguration({ checkAccess: true });
37
38const crawler = new PlaywrightCrawler({
39    proxyConfiguration,
40    maxRequestsPerCrawl,
41    requestHandler: router,
42    launchContext: {
43        launchOptions: {
44            args: [
45                '--disable-gpu', // Mitigates the "crashing GPU process" issue in Docker containers
46            ],
47        },
48    },
49});
50
51await crawler.run(startUrls);
52
53// Exit successfully
54await Actor.exit();