Pricing
Pay per usage
Crawls websites with stealthy Camoufox browser and Playwright library using a provided server-side Node.js code. Supports both recursive crawling and a list of URLs. Supports login to a website.
0.0 (0)
Pricing
Pay per usage
1
39
12
Issues response
76 days
Last modified
2 months ago
# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestoragedist!./storage/key_value_stores/default/INPUT.json
# installed filesnode_modulesFROM apify/actor-node-playwright-firefox:22 AS builder
COPY package*.json ./
RUN npm install --include=dev --audit=false
COPY . ./
RUN npm run build
FROM apify/actor-node-playwright-firefox:22
COPY /home/myuser/dist ./dist
COPY package*.json ./
RUN npm --quiet set progress=false \ && npm install --omit=dev \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version \ && rm -r ~/.npm
RUN npm run get-binaries
COPY . ./
RUN rm -rf ./pw-browsers/
ENV APIFY_DISABLE_OUTDATED_WARNING=1
CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent{ "$schema": "https://raw.githubusercontent.com/apify/apify-shared-js/refs/heads/master/packages/input_schema/src/schema.json", "title": "Camoufox Scraper Input", "description": "Use the following form to configure Camoufox Scraper. The Start URLs and Page function are required and all other fields are optional. To learn more about the different options, click on their title or on the nearby question marks. For details about the Page function visit the Actor's README in the ACTOR INFO tab.", "type": "object", "schemaVersion": 1, "properties": { "startUrls": { "sectionCaption": "Basic configuration", "title": "Start URLs", "type": "array", "description": "URLs to start with", "prefill": [{ "url": "https://crawlee.dev" }], "editor": "requestListSources" }, "globs": { "title": "Glob Patterns", "type": "array", "description": "Glob patterns to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Glob patterns will cause the scraper to enqueue all links matched by the Link selector.", "editor": "globs", "default": [], "prefill": [ { "glob": "https://crawlee.dev/*/*" } ] }, "pseudoUrls": { "title": "Pseudo-URLs", "type": "array", "description": "Pseudo-URLs to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Pseudo-URLs will cause the scraper to enqueue all links matched by the Link selector.", "editor": "pseudoUrls", "default": [], "prefill": [] }, "excludes": { "title": "Exclude Glob Patterns", "type": "array", "description": "Glob patterns to match links in the page that you want to exclude from being enqueued.", "editor": "globs", "default": [], "prefill": [ { "glob": "/**/*.{png,jpg,jpeg,pdf}" } ] }, "linkSelector": { "title": "Link selector", "type": "string", "description": "CSS selector matching elements with 'href' attributes that should be enqueued. To enqueue urls from <code><div class=\"my-class\" href=...></code> tags, you would enter <strong>div.my-class</strong>. Leave empty to ignore all links.", "editor": "textfield", "prefill": "a" }, "keepUrlFragments": { "title": "Keep URL fragments", "type": "boolean", "description": "URL fragments (the parts of URL after a <code>#</code>) are not considered when the scraper determines whether a URL has already been visited. This means that when adding URLs such as <code>https://example.com/#foo</code> and <code>https://example.com/#bar</code>, only the first will be visited. Turn this option on to tell the scraper to visit both.", "default": false }, "respectRobotsTxtFile": { "title": "Respect the robots.txt file", "type": "boolean", "description": "If enabled, the crawler will consult the robots.txt file for the target website before crawling each page. At the moment, the crawler does not use any specific user agent identifier. The crawl-delay directive is also not supported yet.", "default": false, "prefill": true }, "pageFunction": { "title": "Page function", "type": "string", "description": "Function executed for each request", "prefill": "async function pageFunction(context) {\n const { page, request, log } = context;\n const title = await page.title();\n log.info(`URL: ${request.url} TITLE: ${title}`);\n return {\n url: request.url,\n title\n };\n}", "editor": "javascript" }, "proxyConfiguration": { "sectionCaption": "Proxy configuration", "title": "Proxy configuration", "type": "object", "description": "Specifies proxy servers that will be used by the scraper in order to hide its origin.<br><br>For details, see <a href='https://apify.com/apify/camoufox-scraper#proxy-configuration' target='_blank' rel='noopener'>Proxy configuration</a> in README.", "prefill": { "useApifyProxy": true }, "default": { "useApifyProxy": true }, "editor": "proxy" }, "proxyRotation": { "title": "Proxy rotation", "type": "string", "description": "This property indicates the strategy of proxy rotation and can only be used in conjunction with Apify Proxy. The recommended setting automatically picks the best proxies from your available pool and rotates them evenly, discarding proxies that become blocked or unresponsive. If this strategy does not work for you for any reason, you may configure the scraper to either use a new proxy for each request, or to use one proxy as long as possible, until the proxy fails. IMPORTANT: This setting will only use your available Apify Proxy pool, so if you don't have enough proxies for a given task, no rotation setting will produce satisfactory results.", "default": "RECOMMENDED", "editor": "select", "enum": ["RECOMMENDED", "PER_REQUEST", "UNTIL_FAILURE"], "enumTitles": [ "Use recommended settings", "Rotate proxy after each request", "Use one proxy until failure" ] }, "sessionPoolName": { "title": "Session pool name", "type": "string", "description": "<b>Use only english alphanumeric characters dashes and underscores.</b> A session is a representation of a user. It has it's own IP and cookies which are then used together to emulate a real user. Usage of the sessions is controlled by the Proxy rotation option. By providing a session pool name, you enable sharing of those sessions across multiple Actor runs. This is very useful when you need specific cookies for accessing the websites or when a lot of your proxies are already blocked. Instead of trying randomly, a list of working sessions will be saved and a new Actor run can reuse those sessions. Note that the IP lock on sessions expires after 24 hours, unless the session is used again in that window.", "editor": "textfield", "minLength": 3, "maxLength": 200, "pattern": "[0-9A-z-]" }, "initialCookies": { "title": "Initial cookies", "type": "array", "description": "The provided cookies will be pre-set to all pages the scraper opens.", "default": [], "prefill": [], "editor": "json" }, "headless": { "title": "Run browsers in headless mode", "type": "boolean", "description": "By default, browsers run in headless mode. You can toggle this off to run them in headful mode, which can help with certain rare anti-scraping protections but is slower and more costly.", "default": true }, "ignoreSslErrors": { "title": "Ignore SSL errors", "type": "boolean", "description": "Scraper will ignore SSL certificate errors.", "default": false, "groupCaption": "Security", "groupDescription": "Various options that help you disable browser security features such as HTTPS certificates and CORS." }, "downloadMedia": { "sectionCaption": "Performance and limits", "title": "Download media", "type": "boolean", "description": "Scraper will download media such as images, fonts, videos and sounds. Disabling this may speed up the scrape, but certain websites could stop working correctly.", "default": true, "groupCaption": "Resources", "groupDescription": "Settings that help to disable downloading web page resources." }, "maxRequestRetries": { "title": "Max request retries", "type": "integer", "description": "Maximum number of times the request for the page will be retried in case of an error. Setting it to 0 means that the request will be attempted once and will not be retried if it fails.", "minimum": 0, "default": 3, "unit": "retries" }, "maxPagesPerCrawl": { "title": "Max pages per run", "type": "integer", "description": "Maximum number of pages that the scraper will open. 0 means unlimited.", "minimum": 0, "default": 0, "unit": "pages" }, "maxResultsPerCrawl": { "title": "Max result records", "type": "integer", "description": "Maximum number of results that will be saved to dataset. The scraper will terminate afterwards. 0 means unlimited.", "minimum": 0, "default": 0, "unit": "results" }, "maxCrawlingDepth": { "title": "Max crawling depth", "type": "integer", "description": "Defines how many links away from the StartURLs will the scraper descend. 0 means unlimited.", "minimum": 0, "default": 0 }, "maxConcurrency": { "title": "Max concurrency", "type": "integer", "description": "Defines how many pages can be processed by the scraper in parallel. The scraper automatically increases and decreases concurrency based on available system resources. Use this option to set a hard limit.", "minimum": 1, "default": 50 }, "pageLoadTimeoutSecs": { "title": "Page load timeout", "type": "integer", "description": "Maximum time the scraper will allow a web page to load in seconds.", "minimum": 1, "default": 60, "unit": "secs" }, "pageFunctionTimeoutSecs": { "title": "Page function timeout", "type": "integer", "description": "Maximum time the scraper will wait for the page function to execute in seconds.", "minimum": 1, "default": 60, "unit": "secs" }, "waitUntil": { "title": "Navigation wait until", "description": "The scraper will wait until the selected events are triggered in the page before executing the page function. Available events are <code>domcontentloaded</code>, <code>load</code> and <code>networkidle</code> <a href=\"https://playwright.dev/docs/api/class-page#page-goto-option-wait-until\" target=\"_blank\">See Playwright docs</a>.", "type": "string", "editor": "select", "enum": ["networkidle", "load", "domcontentloaded"], "prefill": "networkidle", "default": "networkidle" }, "preNavigationHooks": { "sectionCaption": "Advanced configuration", "title": "Pre-navigation hooks", "type": "string", "description": "Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`, which are passed to the `page.goto()` function the crawler calls to navigate.", "prefill": "// We need to return array of (possibly async) functions here.\n// The functions accept two arguments: the \"crawlingContext\" object\n// and \"gotoOptions\".\n[\n async (crawlingContext, gotoOptions) => {\n const { page } = crawlingContext;\n // ...\n },\n]", "editor": "javascript" }, "postNavigationHooks": { "title": "Post-navigation hooks", "type": "string", "description": "Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts `crawlingContext` as the only parameter.", "prefill": "// We need to return array of (possibly async) functions here.\n// The functions accept a single argument: the \"crawlingContext\" object.\n[\n async (crawlingContext) => {\n const { page } = crawlingContext;\n // ...\n },\n]", "editor": "javascript" }, "closeCookieModals": { "title": "Dismiss cookie modals", "type": "boolean", "description": "Using the [I don't care about cookies](https://addons.mozilla.org/en-US/firefox/addon/i-dont-care-about-cookies/) browser extension. When on, the crawler will automatically try to dismiss cookie consent modals. This can be useful when crawling European websites that show cookie consent modals.", "default": false }, "maxScrollHeightPixels": { "title": "Maximum scrolling distance in pixels", "type": "integer", "description": "The crawler will scroll down the page until all content is loaded or the maximum scrolling distance is reached. Setting this to `0` disables scrolling altogether.", "default": 5000 }, "debugLog": { "title": "Debug log", "type": "boolean", "description": "Debug messages will be included in the log. Use <code>context.log.debug('message')</code> to log your own debug messages.", "default": false, "groupCaption": "Enable logs", "groupDescription": "Logs settings" }, "browserLog": { "title": "Browser log", "type": "boolean", "description": "Console messages from the Browser will be included in the log. This may result in the log being flooded by error messages, warnings and other messages of little value, especially with high concurrency.", "default": false }, "customData": { "title": "Custom data", "type": "object", "description": "This object will be available on pageFunction's context as customData.", "default": {}, "prefill": {}, "editor": "json" }, "datasetName": { "title": "Dataset name", "type": "string", "description": "Name or ID of the dataset that will be used for storing results. If left empty, the default dataset of the run will be used.", "editor": "textfield" }, "keyValueStoreName": { "title": "Key-value store name", "type": "string", "description": "Name or ID of the key-value store that will be used for storing records. If left empty, the default key-value store of the run will be used.", "editor": "textfield" }, "requestQueueName": { "title": "Request queue name", "type": "string", "description": "Name of the request queue that will be used for storing requests. If left empty, the default request queue of the run will be used.", "editor": "textfield" }, "os": { "sectionCaption": "Camoufox options", "title": "Emulated operating system", "type": "array", "description": "Operating system to use for the fingerprint generation. Can be 'windows', 'macos', 'linux', or a list to randomly choose from.", "editor": "select", "items": { "type": "string", "enum": ["linux", "windows", "macos"], "enumTitles": ["Linux", "Windows", "MacOS"] } }, "block_images": { "title": "Block image loading", "type": "boolean", "description": "Blocks the image loading on the page. Saves bandwidth and speeds up the scraping, but might cause detection of the scraper.", "default": false }, "block_webrtc": { "title": "Block WebRTC", "type": "boolean", "description": "Blocks WebRTC capabilities in Camoufox. Note that this might break some web applications.", "default": false }, "block_webgl": { "title": "Block WebGL", "type": "boolean", "description": "Blocks WebGL capabilities in Camoufox. Note that this might break some websites." }, "disable_coop": { "title": "Disable Cross-Origin-Opener-Policy", "type": "boolean", "description": "Disables the Cross-Origin-Opener-Policy." }, "geoip": { "title": "GeoIP (⚠️ might not fully work with Apify Proxy ⚠️).", "description": "Calculate geo-data based on IP address. Might not fully work with Apify Proxy.", "type": "boolean", "default": false }, "humanize": { "title": "Humanize cursor movement (speed in seconds)", "type": "string", "editor": "textfield", "description": "Humanize cursor movement." }, "locale": { "title": "Locales", "type": "array", "editor": "stringList", "description": "Locales to use in the browser / OS." }, "fonts": { "title": "Fonts", "type": "array", "editor": "stringList", "description": "Fonts to load into the browser / OS." }, "custom_fonts_only": { "title": "Custom fonts only", "type": "boolean", "description": "If enabled, only custom fonts (from the option `fonts`) are used. This will disable the default OS fonts." }, "enable_cache": { "title": "Enable cache", "type": "boolean", "description": "Cache previous pages and requests in Camoufox." }, "debug": { "title": "Camoufox debug", "type": "boolean", "description": "Prints the config being sent to Camoufox." } }, "required": ["startUrls", "pageFunction", "proxyConfiguration"]}{ "name": "actor-camoufox-scraper", "version": "3.1.0", "private": true, "description": "Crawl web pages using Apify, headless browser and Playwright with Camoufox", "type": "module", "dependencies": { "@apify/scraper-tools": "^1.1.4", "@crawlee/core": "^3.13.2", "@crawlee/playwright": "^3.13.2", "@crawlee/utils": "^3.13.2", "apify": "^3.2.6", "camoufox-js": "^0.3.0", "idcac-playwright": "^0.1.3", "playwright": "*" }, "devDependencies": { "@apify/tsconfig": "^0.1.0", "@types/node": "^22.7.4", "tsx": "^4.19.1", "typescript": "~5.8.0" }, "scripts": { "start": "npm run start:dev", "start:prod": "node dist/main.js", "start:dev": "tsx src/main.ts", "build": "tsc", "get-binaries": "camoufox-js fetch" }, "repository": { "type": "git", "url": "https://github.com/apify/apify-sdk-js" }, "author": { "name": "Apify Technologies", "email": "support@apify.com", "url": "https://apify.com" }, "license": "Apache-2.0", "homepage": "https://github.com/apify/apify-sdk-js"}{ "extends": "@apify/tsconfig", "compilerOptions": { "outDir": "dist", "module": "ESNext", "allowJs": true, "skipLibCheck": true }, "include": ["src"]}{ "actorSpecification": 1, "name": "camoufox-scraper", "version": "0.1", "buildTag": "latest"}1import { runActor } from '@apify/scraper-tools';2
3import { CrawlerSetup } from './internals/crawler_setup.js';4
5runActor(CrawlerSetup);1import type {2 GlobInput,3 ProxyConfigurationOptions,4 PseudoUrlInput,5 RegExpInput,6 RequestOptions,7 Session,8} from '@crawlee/core';9import type { Dictionary } from '@crawlee/utils';10
11export const enum ProxyRotation {12 Recommended = 'RECOMMENDED',13 PerRequest = 'PER_REQUEST',14 UntilFailure = 'UNTIL_FAILURE',15}16
17/**18 * Replicates the INPUT_SCHEMA with TypeScript types for quick reference19 * and IDE type check integration.20 */21export interface Input {22 startUrls: RequestOptions[];23 globs: GlobInput[];24 regexps: RegExpInput[];25 pseudoUrls: PseudoUrlInput[];26 excludes: GlobInput[];27 linkSelector?: string;28 keepUrlFragments: boolean;29 respectRobotsTxtFile: boolean;30 pageFunction: string;31 preNavigationHooks?: string;32 postNavigationHooks?: string;33 proxyConfiguration: ProxyConfigurationOptions;34 proxyRotation: ProxyRotation;35 sessionPoolName?: string;36 initialCookies: Parameters<Session['setCookies']>[0];37 maxScrollHeightPixels: number;38 ignoreSslErrors: boolean;39 ignoreCorsAndCsp: boolean;40 closeCookieModals: boolean;41 downloadMedia: boolean;42 maxRequestRetries: number;43 maxPagesPerCrawl: number;44 maxResultsPerCrawl: number;45 maxCrawlingDepth: number;46 maxConcurrency: number;47 pageLoadTimeoutSecs: number;48 pageFunctionTimeoutSecs: number;49 waitUntil: 'networkidle' | 'load' | 'domcontentloaded';50 debugLog: boolean;51 browserLog: boolean;52 customData: Dictionary;53 datasetName?: string;54 keyValueStoreName?: string;55 requestQueueName?: string;56 headless: boolean;57 os: ('linux' | 'macos' | 'windows')[];58 blockImages: boolean;59 blockWebrtc: boolean;60 blockWebgl: boolean;61 disableCoop: boolean;62 geoip: boolean;63 humanize: string;64 locale: string[];65 fonts: string[];66 customFontsOnly: boolean;67 enableCache: boolean;68 debug: boolean;69}1import { readFile } from 'node:fs/promises';2import { dirname } from 'node:path';3import { fileURLToPath, URL } from 'node:url';4
5import type {6 AutoscaledPool,7 EnqueueLinksOptions,8 PlaywrightCrawlerOptions,9 PlaywrightCrawlingContext,10 PlaywrightLaunchContext,11 ProxyConfiguration,12 Request,13} from '@crawlee/playwright';14import {15 Dataset,16 KeyValueStore,17 log,18 PlaywrightCrawler,19 RequestList,20 RequestQueueV2,21} from '@crawlee/playwright';22import type { Awaitable, Dictionary } from '@crawlee/utils';23import { sleep } from '@crawlee/utils';24import type { ApifyEnv } from 'apify';25import { Actor } from 'apify';26import { launchOptions } from 'camoufox-js';27import { getInjectableScript } from 'idcac-playwright';28import type { Response } from 'playwright';29import { firefox } from 'playwright';30
31import type {32 CrawlerSetupOptions,33 RequestMetadata,34} from '@apify/scraper-tools';35import {36 browserTools,37 constants as scraperToolsConstants,38 createContext,39 tools,40} from '@apify/scraper-tools';41
42import type { Input } from './consts.js';43import { ProxyRotation } from './consts.js';44
45const SESSION_STORE_NAME = 'APIFY-PLAYWRIGHT-SCRAPER-SESSION-STORE';46
47const { META_KEY, DEVTOOLS_TIMEOUT_SECS, SESSION_MAX_USAGE_COUNTS } =48 scraperToolsConstants;49const SCHEMA = JSON.parse(50 await readFile(new URL('../../INPUT_SCHEMA.json', import.meta.url), 'utf8'),51);52
53/**54 * Holds all the information necessary for constructing a crawler55 * instance and creating a context for a pageFunction invocation.56 */57export class CrawlerSetup implements CrawlerSetupOptions {58 name = 'Playwright Scraper';59 rawInput: string;60 env: ApifyEnv;61 /**62 * Used to store data that persist navigations63 */64 globalStore = new Map();65 requestQueue: RequestQueueV2;66 keyValueStore: KeyValueStore;67 customData: unknown;68 input: Input;69 maxSessionUsageCount: number;70 evaledPageFunction: (...args: unknown[]) => unknown;71 evaledPreNavigationHooks: ((...args: unknown[]) => Awaitable<void>)[];72 evaledPostNavigationHooks: ((...args: unknown[]) => Awaitable<void>)[];73 devtools: boolean;74 datasetName?: string;75 keyValueStoreName?: string;76 requestQueueName?: string;77
78 crawler!: PlaywrightCrawler;79 requestList!: RequestList;80 dataset!: Dataset;81 pagesOutputted!: number;82 private initPromise: Promise<void>;83
84 constructor(input: Input) {85 // Set log level early to prevent missed messages.86 if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);87
88 // Keep this as string to be immutable.89 this.rawInput = JSON.stringify(input);90
91 // Attempt to load page function from disk if not present on input.92 tools.maybeLoadPageFunctionFromDisk(93 input,94 dirname(fileURLToPath(import.meta.url)),95 );96
97 // Validate INPUT if not running on Apify Cloud Platform.98 if (!Actor.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);99
100 this.input = input;101 this.env = Actor.getEnv();102
103 // Validations104 this.input.pseudoUrls.forEach((purl) => {105 if (!tools.isPlainObject(purl)) {106 throw new Error(107 'The pseudoUrls Array must only contain Objects.',108 );109 }110 if (purl.userData && !tools.isPlainObject(purl.userData)) {111 throw new Error(112 'The userData property of a pseudoUrl must be an Object.',113 );114 }115 });116
117 this.input.initialCookies.forEach((cookie) => {118 if (!tools.isPlainObject(cookie)) {119 throw new Error(120 'The initialCookies Array must only contain Objects.',121 );122 }123 });124
125 if (126 !/^(domcontentloaded|load|networkidle)$/.test(this.input.waitUntil)127 ) {128 throw new Error(129 'Navigation wait until event must be valid. See tooltip.',130 );131 }132
133 // solving proxy rotation settings134 this.maxSessionUsageCount =135 SESSION_MAX_USAGE_COUNTS[this.input.proxyRotation];136
137 // Functions need to be evaluated.138 this.evaledPageFunction = tools.evalFunctionOrThrow(139 this.input.pageFunction,140 );141
142 if (this.input.preNavigationHooks) {143 this.evaledPreNavigationHooks = tools.evalFunctionArrayOrThrow(144 this.input.preNavigationHooks,145 'preNavigationHooks',146 );147 } else {148 this.evaledPreNavigationHooks = [];149 }150
151 if (this.input.postNavigationHooks) {152 this.evaledPostNavigationHooks = tools.evalFunctionArrayOrThrow(153 this.input.postNavigationHooks,154 'postNavigationHooks',155 );156 } else {157 this.evaledPostNavigationHooks = [];158 }159
160 // Start Chromium with Debugger any time the page function includes the keyword.161 this.devtools = this.input.pageFunction.includes('debugger;');162
163 // Named storages164 this.datasetName = this.input.datasetName;165 this.keyValueStoreName = this.input.keyValueStoreName;166 this.requestQueueName = this.input.requestQueueName;167
168 // Initialize async operations.169 this.crawler = null!;170 this.requestList = null!;171 this.requestQueue = null!;172 this.dataset = null!;173 this.keyValueStore = null!;174 this.initPromise = this._initializeAsync();175 }176
177 private async _initializeAsync() {178 // RequestList179 const startUrls = this.input.startUrls.map((req) => {180 req.useExtendedUniqueKey = true;181 req.keepUrlFragment = this.input.keepUrlFragments;182 return req;183 });184
185 this.requestList = await RequestList.open(186 'PLAYWRIGHT_SCRAPER',187 startUrls,188 );189
190 // RequestQueue191 this.requestQueue = await RequestQueueV2.open(this.requestQueueName);192
193 // Dataset194 this.dataset = await Dataset.open(this.datasetName);195 const info = await this.dataset.getInfo();196 this.pagesOutputted = info?.itemCount ?? 0;197
198 // KeyValueStore199 this.keyValueStore = await KeyValueStore.open(this.keyValueStoreName);200 }201
202 /**203 * Resolves to a `PlaywrightCrawler` instance.204 */205 async createCrawler() {206 await this.initPromise;207
208 const options: PlaywrightCrawlerOptions = {209 requestHandler: this._requestHandler.bind(this),210 requestList: this.requestList,211 requestQueue: this.requestQueue,212 requestHandlerTimeoutSecs: this.devtools213 ? DEVTOOLS_TIMEOUT_SECS214 : this.input.pageFunctionTimeoutSecs,215 preNavigationHooks: [],216 postNavigationHooks: [],217 failedRequestHandler: this._failedRequestHandler.bind(this),218 respectRobotsTxtFile: this.input.respectRobotsTxtFile,219 maxConcurrency: this.input.maxConcurrency,220 maxRequestRetries: this.input.maxRequestRetries,221 maxRequestsPerCrawl: this.input.maxPagesPerCrawl,222 proxyConfiguration: (await Actor.createProxyConfiguration(223 this.input.proxyConfiguration,224 )) as any as ProxyConfiguration,225 launchContext: {226 launcher: firefox,227 launchOptions: await launchOptions({228 ...this.input,229 humanize: this.input.humanize230 ? Number(this.input.humanize)231 : 0,232 }),233 } as PlaywrightLaunchContext,234 useSessionPool: true,235 persistCookiesPerSession: true,236 sessionPoolOptions: {237 persistStateKeyValueStoreId: this.input.sessionPoolName238 ? SESSION_STORE_NAME239 : undefined,240 persistStateKey: this.input.sessionPoolName,241 sessionOptions: {242 maxUsageCount: this.maxSessionUsageCount,243 },244 },245 experiments: {246 requestLocking: true,247 },248 };249
250 this._createNavigationHooks(options);251
252 if (this.input.proxyRotation === ProxyRotation.UntilFailure) {253 options.sessionPoolOptions!.maxPoolSize = 1;254 }255
256 this.crawler = new PlaywrightCrawler(options);257
258 return this.crawler;259 }260
261 private _createNavigationHooks(options: PlaywrightCrawlerOptions) {262 options.preNavigationHooks!.push(263 async ({ request, page, session }, gotoOptions) => {264 // Attach a console listener to get all logs from Browser context.265 if (this.input.browserLog) browserTools.dumpConsole(page);266
267 // Add initial cookies, if any.268 if (269 this.input.initialCookies &&270 this.input.initialCookies.length271 ) {272 const cookiesToSet = session273 ? tools.getMissingCookiesFromSession(274 session,275 this.input.initialCookies,276 request.url,277 )278 : this.input.initialCookies;279
280 if (cookiesToSet?.length) {281 // setting initial cookies that are not already in the session and page282 session?.setCookies(cookiesToSet, request.url);283 await page.context().addCookies(cookiesToSet);284 }285 }286
287 if (gotoOptions) {288 gotoOptions.timeout =289 (this.devtools290 ? DEVTOOLS_TIMEOUT_SECS291 : this.input.pageLoadTimeoutSecs) * 1000;292 gotoOptions.waitUntil = this.input.waitUntil;293 }294 },295 );296
297 options.preNavigationHooks!.push(298 ...this._runHookWithEnhancedContext(this.evaledPreNavigationHooks),299 );300 options.postNavigationHooks!.push(301 ...this._runHookWithEnhancedContext(this.evaledPostNavigationHooks),302 );303 }304
305 private _runHookWithEnhancedContext(306 hooks: ((...args: unknown[]) => Awaitable<void>)[],307 ) {308 return hooks.map((hook) => (ctx: Dictionary, ...args: unknown[]) => {309 const { customData } = this.input;310 return hook({ ...ctx, Apify: Actor, Actor, customData }, ...args);311 });312 }313
314 private async _failedRequestHandler({315 request,316 }: PlaywrightCrawlingContext) {317 const lastError =318 request.errorMessages[request.errorMessages.length - 1];319 const errorMessage = lastError ? lastError.split('\n')[0] : 'no error';320 log.error(321 `Request ${request.url} failed and will not be retried anymore. Marking as failed.\nLast Error Message: ${errorMessage}`,322 );323 return this._handleResult(request, undefined, undefined, true);324 }325
326 /**327 * First of all, it initializes the state that is exposed to the user via328 * `pageFunction` context.329 *330 * Then it invokes the user provided `pageFunction` with the prescribed context331 * and saves its return value.332 *333 * Finally, it makes decisions based on the current state and post-processes334 * the data returned from the `pageFunction`.335 */336 private async _requestHandler(crawlingContext: PlaywrightCrawlingContext) {337 const { request, response, crawler } = crawlingContext;338
339 /**340 * PRE-PROCESSING341 */342 // Make sure that an object containing internal metadata343 // is present on every request.344 tools.ensureMetaData(request);345
346 // Abort the crawler if the maximum number of results was reached.347 const aborted = await this._handleMaxResultsPerCrawl(348 crawler.autoscaledPool,349 );350 if (aborted) return;351
352 const pageFunctionArguments: Dictionary = {};353
354 // We must use properties and descriptors not to trigger getters / setters.355 Object.defineProperties(356 pageFunctionArguments,357 Object.getOwnPropertyDescriptors(crawlingContext),358 );359
360 pageFunctionArguments.response = {361 status: response && response.status(),362 headers: response && response.headers(),363 };364
365 // Setup and create Context.366 const contextOptions = {367 crawlerSetup: {368 rawInput: this.rawInput,369 env: this.env,370 globalStore: this.globalStore,371 requestQueue: this.requestQueue,372 keyValueStore: this.keyValueStore,373 customData: this.input.customData,374 },375 pageFunctionArguments,376 };377 const { context, state } = createContext(contextOptions);378
379 if (this.input.closeCookieModals) {380 await sleep(500);381 await crawlingContext.page.evaluate(getInjectableScript());382 await sleep(2000);383 }384
385 if (this.input.maxScrollHeightPixels > 0) {386 await crawlingContext.infiniteScroll({387 maxScrollHeight: this.input.maxScrollHeightPixels,388 });389 }390
391 /**392 * USER FUNCTION INVOCATION393 */394 const pageFunctionResult = await this.evaledPageFunction(context);395
396 /**397 * POST-PROCESSING398 */399 // Enqueue more links if Pseudo URLs and a link selector are available,400 // unless the user invoked the `skipLinks()` context function401 // or maxCrawlingDepth would be exceeded.402 if (!state.skipLinks) await this._handleLinks(crawlingContext);403
404 // Save the `pageFunction`s result to the default dataset.405 await this._handleResult(406 request,407 response,408 pageFunctionResult as Dictionary,409 );410 }411
412 private async _handleMaxResultsPerCrawl(autoscaledPool?: AutoscaledPool) {413 if (414 !this.input.maxResultsPerCrawl ||415 this.pagesOutputted < this.input.maxResultsPerCrawl416 )417 return false;418 if (!autoscaledPool) return false;419 log.info(420 `User set limit of ${this.input.maxResultsPerCrawl} results was reached. Finishing the crawl.`,421 );422 await autoscaledPool.abort();423 return true;424 }425
426 private async _handleLinks({427 request,428 enqueueLinks,429 }: PlaywrightCrawlingContext) {430 if (!this.requestQueue) return;431 const currentDepth = (request.userData![META_KEY] as RequestMetadata)432 .depth;433 const hasReachedMaxDepth =434 this.input.maxCrawlingDepth &&435 currentDepth >= this.input.maxCrawlingDepth;436 if (hasReachedMaxDepth) {437 log.debug(438 `Request ${request.url} reached the maximum crawling depth of ${currentDepth}.`,439 );440 return;441 }442
443 const enqueueOptions: EnqueueLinksOptions = {444 globs: this.input.globs,445 pseudoUrls: this.input.pseudoUrls,446 exclude: this.input.excludes,447 transformRequestFunction: (requestOptions) => {448 requestOptions.userData ??= {};449 requestOptions.userData[META_KEY] = {450 parentRequestId: request.id || request.uniqueKey,451 depth: currentDepth + 1,452 };453
454 requestOptions.useExtendedUniqueKey = true;455 requestOptions.keepUrlFragment = this.input.keepUrlFragments;456 return requestOptions;457 },458 };459
460 if (this.input.linkSelector) {461 await enqueueLinks({462 ...enqueueOptions,463 selector: this.input.linkSelector,464 });465 }466 }467
468 private async _handleResult(469 request: Request,470 response?: Response,471 pageFunctionResult?: Dictionary,472 isError?: boolean,473 ) {474 const payload = tools.createDatasetPayload(475 request,476 response,477 pageFunctionResult,478 isError,479 );480 await this.dataset.pushData(payload);481 this.pagesOutputted++;482 }483}