TypeScript Camoufox scraper

1/**
2 * This template is a production ready boilerplate for developing with `PlaywrightCrawler`.
3 * Use this to bootstrap your projects using the most up-to-date code.
4 * If you're looking for examples or want to learn more, see README.
5 */
6
7// For more information, see https://crawlee.dev
8import { PlaywrightCrawler } from '@crawlee/playwright';
9// For more information, see https://docs.apify.com/sdk/js
10import { Actor } from 'apify';
11import { launchOptions as camoufoxLaunchOptions } from 'camoufox-js';
12import { firefox } from 'playwright';
13
14// this is ESM project, and as such, it requires you to specify extensions in your relative imports
15// read more about this here: https://nodejs.org/docs/latest-v18.x/api/esm.html#mandatory-file-extensions
16// note that we need to use `.js` even when inside TS files
17import { router } from './routes.js';
18
19interface Input {
20    startUrls: {
21        url: string;
22        method?: 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH';
23        headers?: Record<string, string>;
24        userData: Record<string, unknown>;
25    }[];
26    maxRequestsPerCrawl: number;
27}
28
29// Initialize the Apify SDK
30await Actor.init();
31
32// Structure of input is defined in input_schema.json
33const { startUrls = ['https://apify.com'], maxRequestsPerCrawl = 100 } =
34    (await Actor.getInput<Input>()) ?? ({} as Input);
35
36// `checkAccess` flag ensures the proxy credentials are valid, but the check can take a few hundred milliseconds.
37// Disable it for short runs if you are sure your proxy configuration is correct
38const proxyConfiguration = await Actor.createProxyConfiguration({ checkAccess: true });
39
40const crawler = new PlaywrightCrawler({
41    proxyConfiguration,
42    maxRequestsPerCrawl,
43    requestHandler: router,
44    launchContext: {
45        launcher: firefox,
46        launchOptions: await camoufoxLaunchOptions({
47            headless: true,
48            proxy: await proxyConfiguration?.newUrl(),
49            geoip: true,
50            // fonts: ['Times New Roman'] // <- custom Camoufox options
51        }),
52    },
53});
54
55await crawler.run(startUrls);
56
57// Exit successfully
58await Actor.exit();