Pricing
Pay per usage
Crawls websites with the headless Chromium, Chrome, or Firefox browser and Playwright library using a provided server-side Node.js code. Supports both recursive crawling and a list of URLs. Supports login to a website.
4.7 (8)
Pricing
Pay per usage
67
2.6K
538
Issues response
6 days
Last modified
2 days ago
{ "extends": "@apify/tsconfig", "compilerOptions": { "outDir": "dist", "module": "ESNext", "allowJs": true, "skipLibCheck": true }, "include": ["src"]}{ "title": "Playwright Scraper Input", "description": "Use the following form to configure Playwright Scraper. The Start URLs and Page function are required and all other fields are optional. To learn more about the different options, click on their title or on the nearby question marks. For details about the Page function visit the Actor's README in the ACTOR INFO tab.", "type": "object", "schemaVersion": 1, "properties": { "startUrls": { "sectionCaption": "Basic configuration", "title": "Start URLs", "type": "array", "description": "URLs to start with", "prefill": [{ "url": "https://crawlee.dev/js" }], "editor": "requestListSources" }, "globs": { "title": "Glob Patterns", "type": "array", "description": "Glob patterns to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Glob patterns will cause the scraper to enqueue all links matched by the Link selector.", "editor": "globs", "default": [], "prefill": [ { "glob": "https://crawlee.dev/js/*/*" } ] }, "pseudoUrls": { "title": "Pseudo-URLs", "type": "array", "description": "Pseudo-URLs to match links in the page that you want to enqueue. Combine with Link selector to tell the scraper where to find links. Omitting the Pseudo-URLs will cause the scraper to enqueue all links matched by the Link selector.", "editor": "pseudoUrls", "default": [], "prefill": [] }, "excludes": { "title": "Exclude Glob Patterns", "type": "array", "description": "Glob patterns to match links in the page that you want to exclude from being enqueued.", "editor": "globs", "default": [], "prefill": [ { "glob": "/**/*.{png,jpg,jpeg,pdf}" } ] }, "linkSelector": { "title": "Link selector", "type": "string", "description": "CSS selector matching elements with 'href' attributes that should be enqueued. To enqueue urls from <code><div class=\"my-class\" href=...></code> tags, you would enter <strong>div.my-class</strong>. Leave empty to ignore all links.", "editor": "textfield", "prefill": "a" }, "keepUrlFragments": { "title": "Keep URL fragments", "type": "boolean", "description": "URL fragments (the parts of URL after a <code>#</code>) are not considered when the scraper determines whether a URL has already been visited. This means that when adding URLs such as <code>https://example.com/#foo</code> and <code>https://example.com/#bar</code>, only the first will be visited. Turn this option on to tell the scraper to visit both.", "default": false }, "respectRobotsTxtFile": { "title": "Respect the robots.txt file", "type": "boolean", "description": "If enabled, the crawler will consult the robots.txt file for the target website before crawling each page. At the moment, the crawler does not use any specific user agent identifier. The crawl-delay directive is also not supported yet.", "default": false, "prefill": true }, "pageFunction": { "title": "Page function", "type": "string", "description": "Function executed for each request", "prefill": "async function pageFunction(context) {\n const { page, request, log } = context;\n const title = await page.title();\n log.info(`URL: ${request.url} TITLE: ${title}`);\n return {\n url: request.url,\n title\n };\n}", "editor": "javascript" }, "proxyConfiguration": { "sectionCaption": "Proxy configuration", "title": "Proxy configuration", "type": "object", "description": "Specifies proxy servers that will be used by the scraper in order to hide its origin.<br><br>For details, see <a href='https://apify.com/apify/playwright-scraper#proxy-configuration' target='_blank' rel='noopener'>Proxy configuration</a> in README.", "prefill": { "useApifyProxy": true }, "default": { "useApifyProxy": true }, "editor": "proxy" }, "proxyRotation": { "title": "Proxy rotation", "type": "string", "description": "This property indicates the strategy of proxy rotation and can only be used in conjunction with Apify Proxy. The recommended setting automatically picks the best proxies from your available pool and rotates them evenly, discarding proxies that become blocked or unresponsive. If this strategy does not work for you for any reason, you may configure the scraper to either use a new proxy for each request, or to use one proxy as long as possible, until the proxy fails. IMPORTANT: This setting will only use your available Apify Proxy pool, so if you don't have enough proxies for a given task, no rotation setting will produce satisfactory results.", "default": "RECOMMENDED", "editor": "select", "enum": ["RECOMMENDED", "PER_REQUEST", "UNTIL_FAILURE"], "enumTitles": [ "Use recommended settings", "Rotate proxy after each request", "Use one proxy until failure" ] }, "sessionPoolName": { "title": "Session pool name", "type": "string", "description": "<b>Use only english alphanumeric characters dashes and underscores.</b> A session is a representation of a user. It has it's own IP and cookies which are then used together to emulate a real user. Usage of the sessions is controlled by the Proxy rotation option. By providing a session pool name, you enable sharing of those sessions across multiple Actor runs. This is very useful when you need specific cookies for accessing the websites or when a lot of your proxies are already blocked. Instead of trying randomly, a list of working sessions will be saved and a new Actor run can reuse those sessions. Note that the IP lock on sessions expires after 24 hours, unless the session is used again in that window.", "editor": "textfield", "minLength": 3, "maxLength": 200, "pattern": "[0-9A-z-]" }, "initialCookies": { "title": "Initial cookies", "type": "array", "description": "The provided cookies will be pre-set to all pages the scraper opens.", "default": [], "prefill": [], "editor": "json" }, "launcher": { "sectionCaption": "Browser configuration", "title": "Browser Type", "description": "Choose the browser to launch.", "type": "string", "editor": "select", "enum": ["chromium", "firefox"], "enumTitles": ["Chromium", "Firefox"], "prefill": "chromium", "default": "chromium" }, "useChrome": { "title": "Use Chrome (could only be used if Chromium Browser is selected)", "type": "boolean", "description": "The scraper will use a real Chrome browser instead of a Chromium masking as Chrome. Using this option may help with bypassing certain anti-scraping protections, but risks that the scraper will be unstable or not work at all.", "default": false, "groupCaption": "Browser masking", "groupDescription": "Settings that help mask as a real user and prevent scraper detection." }, "headless": { "title": "Run browsers in headless mode", "type": "boolean", "description": "By default, browsers run in headless mode. You can toggle this off to run them in headful mode, which can help with certain rare anti-scraping protections but is slower and more costly.", "default": true }, "ignoreSslErrors": { "title": "Ignore SSL errors", "type": "boolean", "description": "Scraper will ignore SSL certificate errors.", "default": false, "groupCaption": "Security", "groupDescription": "Various options that help you disable browser security features such as HTTPS certificates and CORS." }, "ignoreCorsAndCsp": { "title": "Ignore CORS and CSP", "type": "boolean", "description": "Scraper will ignore CSP (content security policy) and CORS (cross origin resource sharing) settings of visited pages and requested domains. This enables you to freely use XHR/Fetch to make HTTP requests from the scraper.", "default": false }, "downloadMedia": { "sectionCaption": "Performance and limits", "title": "Download media", "type": "boolean", "description": "Scraper will download media such as images, fonts, videos and sounds. Disabling this may speed up the scrape, but certain websites could stop working correctly.", "default": true, "groupCaption": "Resources", "groupDescription": "Settings that help to disable downloading web page resources." }, "downloadCss": { "title": "Download CSS", "type": "boolean", "description": "Scraper will download CSS stylesheets. Disabling this may speed up the scrape, but certain websites could stop working correctly.", "default": true }, "maxRequestRetries": { "title": "Max request retries", "type": "integer", "description": "Maximum number of times the request for the page will be retried in case of an error. Setting it to 0 means that the request will be attempted once and will not be retried if it fails.", "minimum": 0, "default": 3, "unit": "retries" }, "maxPagesPerCrawl": { "title": "Max pages per run", "type": "integer", "description": "Maximum number of pages that the scraper will open. 0 means unlimited.", "minimum": 0, "default": 0, "unit": "pages" }, "maxResultsPerCrawl": { "title": "Max result records", "type": "integer", "description": "Maximum number of results that will be saved to dataset. The scraper will terminate afterwards. 0 means unlimited.", "minimum": 0, "default": 0, "unit": "results" }, "maxCrawlingDepth": { "title": "Max crawling depth", "type": "integer", "description": "Defines how many links away from the StartURLs will the scraper descend. 0 means unlimited.", "minimum": 0, "default": 0 }, "maxConcurrency": { "title": "Max concurrency", "type": "integer", "description": "Defines how many pages can be processed by the scraper in parallel. The scraper automatically increases and decreases concurrency based on available system resources. Use this option to set a hard limit.", "minimum": 1, "default": 50 }, "pageLoadTimeoutSecs": { "title": "Page load timeout", "type": "integer", "description": "Maximum time the scraper will allow a web page to load in seconds.", "minimum": 1, "default": 60, "unit": "secs" }, "pageFunctionTimeoutSecs": { "title": "Page function timeout", "type": "integer", "description": "Maximum time the scraper will wait for the page function to execute in seconds.", "minimum": 1, "default": 60, "unit": "secs" }, "waitUntil": { "title": "Navigation wait until", "description": "The scraper will wait until the selected events are triggered in the page before executing the page function. Available events are <code>domcontentloaded</code>, <code>load</code> and <code>networkidle</code> <a href=\"https://playwright.dev/docs/api/class-page#page-goto-option-wait-until\" target=\"_blank\">See Playwright docs</a>.", "type": "string", "editor": "select", "enum": ["networkidle", "load", "domcontentloaded"], "prefill": "networkidle", "default": "networkidle" }, "preNavigationHooks": { "sectionCaption": "Advanced configuration", "title": "Pre-navigation hooks", "type": "string", "description": "Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`, which are passed to the `page.goto()` function the crawler calls to navigate.", "prefill": "// We need to return array of (possibly async) functions here.\n// The functions accept two arguments: the \"crawlingContext\" object\n// and \"gotoOptions\".\n[\n async (crawlingContext, gotoOptions) => {\n const { page } = crawlingContext;\n // ...\n },\n]", "editor": "javascript" }, "postNavigationHooks": { "title": "Post-navigation hooks", "type": "string", "description": "Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful. The function accepts `crawlingContext` as the only parameter.", "prefill": "// We need to return array of (possibly async) functions here.\n// The functions accept a single argument: the \"crawlingContext\" object.\n[\n async (crawlingContext) => {\n const { page } = crawlingContext;\n // ...\n },\n]", "editor": "javascript" }, "closeCookieModals": { "title": "Dismiss cookie modals", "type": "boolean", "description": "Using the [I don't care about cookies](https://addons.mozilla.org/en-US/firefox/addon/i-dont-care-about-cookies/) browser extension. When on, the crawler will automatically try to dismiss cookie consent modals. This can be useful when crawling European websites that show cookie consent modals.", "default": false }, "maxScrollHeightPixels": { "title": "Maximum scrolling distance in pixels", "type": "integer", "description": "The crawler will scroll down the page until all content is loaded or the maximum scrolling distance is reached. Setting this to `0` disables scrolling altogether.", "default": 5000 }, "debugLog": { "title": "Debug log", "type": "boolean", "description": "Debug messages will be included in the log. Use <code>context.log.debug('message')</code> to log your own debug messages.", "default": false, "groupCaption": "Enable logs", "groupDescription": "Logs settings" }, "browserLog": { "title": "Browser log", "type": "boolean", "description": "Console messages from the Browser will be included in the log. This may result in the log being flooded by error messages, warnings and other messages of little value, especially with high concurrency.", "default": false }, "customData": { "title": "Custom data", "type": "object", "description": "This object will be available on pageFunction's context as customData.", "default": {}, "prefill": {}, "editor": "json" }, "datasetName": { "title": "Dataset name", "type": "string", "description": "Name or ID of the dataset that will be used for storing results. If left empty, the default dataset of the run will be used.", "editor": "textfield" }, "keyValueStoreName": { "title": "Key-value store name", "type": "string", "description": "Name or ID of the key-value store that will be used for storing records. If left empty, the default key-value store of the run will be used.", "editor": "textfield" }, "requestQueueName": { "title": "Request queue name", "type": "string", "description": "Name of the request queue that will be used for storing requests. If left empty, the default request queue of the run will be used.", "editor": "textfield" } }, "required": ["startUrls", "pageFunction", "proxyConfiguration"]}FROM apify/actor-node-playwright:22 AS builder
COPY package*.json ./
RUN npm install --include=dev --audit=false
COPY . ./
RUN npm run build
FROM apify/actor-node-playwright:22
COPY /home/myuser/dist ./dist
COPY package*.json ./
RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version \ && rm -r ~/.npm
COPY . ./
ENV APIFY_DISABLE_OUTDATED_WARNING=1
CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent{ "name": "actor-playwright-scraper", "version": "3.1.0", "private": true, "description": "Crawl web pages using Apify, headless browser and Playwright", "type": "module", "dependencies": { "@apify/scraper-tools": "^1.1.4", "@crawlee/core": "^3.14.1", "@crawlee/playwright": "^3.14.1", "@crawlee/utils": "^3.14.1", "apify": "^3.2.6", "idcac-playwright": "^0.1.3", "playwright": "*" }, "devDependencies": { "@apify/tsconfig": "^0.1.0", "@types/node": "^22.7.4", "tsx": "^4.19.1", "typescript": "~5.8.0" }, "scripts": { "start": "npm run start:dev", "start:prod": "node dist/main.js", "start:dev": "tsx src/main.ts", "build": "tsc" }, "repository": { "type": "git", "url": "https://github.com/apify/apify-sdk-js" }, "author": { "name": "Apify Technologies", "email": "support@apify.com", "url": "https://apify.com" }, "license": "Apache-2.0", "homepage": "https://github.com/apify/apify-sdk-js"}1# Change Log2
3## 1.0.15 (2024-10-25)4
5- Updated Crawlee version to v3.11.5 and SDK v3.2.66- Updated Node to v227
8## 1.0.14 (2024-04-09)9
10- Updated Crawlee version to v3.8.0.11- Updated to use new request queue in scraper12
13## 1.0.11 (2023-08-22)14
15- Updated Crawlee version to v3.5.2.16- Updated Node.js version to v18.17- Added new options:18 - **Dismiss cookie modals** (`closeCookieModals`): Using the [I don't care about cookies](https://addons.mozilla.org/en-US/firefox/addon/i-dont-care-about-cookies/) browser extension. When on, the crawler will automatically try to dismiss cookie consent modals. This can be useful when crawling European websites that show cookie consent modals.19 - **Maximum scrolling distance in pixels** (`maxScrollHeightPixels`): The crawler will scroll down the page until all content is loaded or the maximum scrolling distance is reached. Setting this to `0` disables scrolling altogether.20 - **Exclude Glob Patterns** (`excludes`): Glob patterns to match links in the page that you want to exclude from being enqueued.21
22## 1.023
24- Initial version built on Crawlee.25- Proxy usage is now required.# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed filesnode_modules1import { runActor } from '@apify/scraper-tools';2
3import { CrawlerSetup } from './internals/crawler_setup.js';4
5runActor(CrawlerSetup);{ "actorSpecification": 1, "name": "playwright-scraper", "version": "0.1", "buildTag": "latest", "storages": { "dataset": { "actorSpecification": 1, "fields": {}, "views": {} } }}1import type {2 GlobInput,3 ProxyConfigurationOptions,4 PseudoUrlInput,5 RegExpInput,6 RequestOptions,7 Session,8} from '@crawlee/core';9import type { Dictionary } from '@crawlee/utils';10
11export const enum ProxyRotation {12 Recommended = 'RECOMMENDED',13 PerRequest = 'PER_REQUEST',14 UntilFailure = 'UNTIL_FAILURE',15}16
17/**18 * Replicates the INPUT_SCHEMA with TypeScript types for quick reference19 * and IDE type check integration.20 */21export interface Input {22 startUrls: RequestOptions[];23 globs: GlobInput[];24 regexps: RegExpInput[];25 pseudoUrls: PseudoUrlInput[];26 excludes: GlobInput[];27 linkSelector?: string;28 keepUrlFragments: boolean;29 respectRobotsTxtFile: boolean;30 pageFunction: string;31 preNavigationHooks?: string;32 postNavigationHooks?: string;33 proxyConfiguration: ProxyConfigurationOptions;34 proxyRotation: ProxyRotation;35 sessionPoolName?: string;36 initialCookies: Parameters<Session['setCookies']>[0];37 launcher: 'chromium' | 'firefox' | 'webkit';38 useChrome: boolean;39 maxScrollHeightPixels: number;40 ignoreSslErrors: boolean;41 ignoreCorsAndCsp: boolean;42 closeCookieModals: boolean;43 downloadMedia: boolean;44 downloadCss: boolean;45 maxRequestRetries: number;46 maxPagesPerCrawl: number;47 maxResultsPerCrawl: number;48 maxCrawlingDepth: number;49 maxConcurrency: number;50 pageLoadTimeoutSecs: number;51 pageFunctionTimeoutSecs: number;52 waitUntil: 'networkidle' | 'load' | 'domcontentloaded';53 debugLog: boolean;54 browserLog: boolean;55 customData: Dictionary;56 datasetName?: string;57 keyValueStoreName?: string;58 requestQueueName?: string;59 headless: boolean;60}1import { readFile } from 'node:fs/promises';2import { dirname } from 'node:path';3import { fileURLToPath, URL } from 'node:url';4
5import type {6 AutoscaledPool,7 EnqueueLinksOptions,8 PlaywrightCrawlerOptions,9 PlaywrightCrawlingContext,10 PlaywrightLaunchContext,11 ProxyConfiguration,12 Request,13} from '@crawlee/playwright';14import {15 Dataset,16 KeyValueStore,17 log,18 PlaywrightCrawler,19 RequestList,20 RequestQueueV2,21} from '@crawlee/playwright';22import type { Awaitable, Dictionary } from '@crawlee/utils';23import { sleep } from '@crawlee/utils';24import type { ApifyEnv } from 'apify';25import { Actor } from 'apify';26import { getInjectableScript } from 'idcac-playwright';27import type { Response } from 'playwright';28import playwright from 'playwright';29
30import type {31 CrawlerSetupOptions,32 RequestMetadata,33} from '@apify/scraper-tools';34import {35 browserTools,36 constants as scraperToolsConstants,37 createContext,38 tools,39} from '@apify/scraper-tools';40
41import type { Input } from './consts.js';42import { ProxyRotation } from './consts.js';43
44const SESSION_STORE_NAME = 'APIFY-PLAYWRIGHT-SCRAPER-SESSION-STORE';45const REQUEST_QUEUE_INIT_FLAG_KEY = 'REQUEST_QUEUE_INITIALIZED';46
47const {48 META_KEY,49 DEFAULT_VIEWPORT,50 DEVTOOLS_TIMEOUT_SECS,51 SESSION_MAX_USAGE_COUNTS,52} = scraperToolsConstants;53const SCHEMA = JSON.parse(54 await readFile(new URL('../../INPUT_SCHEMA.json', import.meta.url), 'utf8'),55);56
57/**58 * Holds all the information necessary for constructing a crawler59 * instance and creating a context for a pageFunction invocation.60 */61export class CrawlerSetup implements CrawlerSetupOptions {62 name = 'Playwright Scraper';63 rawInput: string;64 env: ApifyEnv;65 /**66 * Used to store data that persist navigations67 */68 globalStore = new Map();69 requestQueue: RequestQueueV2;70 keyValueStore: KeyValueStore;71 customData: unknown;72 input: Input;73 maxSessionUsageCount: number;74 evaledPageFunction: (...args: unknown[]) => unknown;75 evaledPreNavigationHooks: ((...args: unknown[]) => Awaitable<void>)[];76 evaledPostNavigationHooks: ((...args: unknown[]) => Awaitable<void>)[];77 blockedUrlPatterns: string[] = [];78 devtools: boolean;79 datasetName?: string;80 keyValueStoreName?: string;81 requestQueueName?: string;82
83 crawler!: PlaywrightCrawler;84 dataset!: Dataset;85 pagesOutputted!: number;86 private initPromise: Promise<void>;87
88 constructor(input: Input) {89 // Set log level early to prevent missed messages.90 if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);91
92 // Keep this as string to be immutable.93 this.rawInput = JSON.stringify(input);94
95 // Attempt to load page function from disk if not present on input.96 tools.maybeLoadPageFunctionFromDisk(97 input,98 dirname(fileURLToPath(import.meta.url)),99 );100
101 // Validate INPUT if not running on Apify Cloud Platform.102 if (!Actor.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);103
104 this.input = input;105 this.env = Actor.getEnv();106
107 // Validations108 this.input.pseudoUrls.forEach((purl) => {109 if (!tools.isPlainObject(purl)) {110 throw new Error(111 'The pseudoUrls Array must only contain Objects.',112 );113 }114 if (purl.userData && !tools.isPlainObject(purl.userData)) {115 throw new Error(116 'The userData property of a pseudoUrl must be an Object.',117 );118 }119 });120
121 if (this.input.useChrome && this.input.launcher !== 'chromium') {122 throw new Error(123 'useChrome option could only be used with Chromium browser selected.',124 );125 }126
127 this.input.initialCookies.forEach((cookie) => {128 if (!tools.isPlainObject(cookie)) {129 throw new Error(130 'The initialCookies Array must only contain Objects.',131 );132 }133 });134
135 if (136 !/^(domcontentloaded|load|networkidle)$/.test(this.input.waitUntil)137 ) {138 throw new Error(139 'Navigation wait until event must be valid. See tooltip.',140 );141 }142
143 // solving proxy rotation settings144 this.maxSessionUsageCount =145 SESSION_MAX_USAGE_COUNTS[this.input.proxyRotation];146
147 // Functions need to be evaluated.148 this.evaledPageFunction = tools.evalFunctionOrThrow(149 this.input.pageFunction,150 );151
152 if (this.input.preNavigationHooks) {153 this.evaledPreNavigationHooks = tools.evalFunctionArrayOrThrow(154 this.input.preNavigationHooks,155 'preNavigationHooks',156 );157 } else {158 this.evaledPreNavigationHooks = [];159 }160
161 if (this.input.postNavigationHooks) {162 this.evaledPostNavigationHooks = tools.evalFunctionArrayOrThrow(163 this.input.postNavigationHooks,164 'postNavigationHooks',165 );166 } else {167 this.evaledPostNavigationHooks = [];168 }169
170 // Excluded resources171 this.blockedUrlPatterns = [];172 if (!this.input.downloadMedia) {173 this.blockedUrlPatterns = [174 '.jpg',175 '.jpeg',176 '.png',177 '.svg',178 '.gif',179 '.webp',180 '.webm',181 '.ico',182 '.woff',183 '.eot',184 ];185 }186
187 if (!this.input.downloadCss) {188 this.blockedUrlPatterns.push('.css');189 }190
191 // Start Chromium with Debugger any time the page function includes the keyword.192 this.devtools = this.input.pageFunction.includes('debugger;');193
194 // Named storages195 this.datasetName = this.input.datasetName;196 this.keyValueStoreName = this.input.keyValueStoreName;197 this.requestQueueName = this.input.requestQueueName;198
199 // Initialize async operations.200 this.crawler = null!;201 this.requestQueue = null!;202 this.dataset = null!;203 this.keyValueStore = null!;204 this.initPromise = this._initializeAsync();205 }206
207 private async _initializeAsync() {208 // RequestList209 const startUrls = this.input.startUrls.map((req) => {210 req.useExtendedUniqueKey = true;211 req.keepUrlFragment = this.input.keepUrlFragments;212 return req;213 });214
215 // KeyValueStore216 this.keyValueStore = await KeyValueStore.open(this.keyValueStoreName);217
218 // RequestQueue219 this.requestQueue = await RequestQueueV2.open(this.requestQueueName);220
221 if (222 !(await this.keyValueStore.recordExists(223 REQUEST_QUEUE_INIT_FLAG_KEY,224 ))225 ) {226 const requests: Request[] = [];227 for await (const request of await RequestList.open(228 null,229 startUrls,230 )) {231 if (232 this.input.maxResultsPerCrawl > 0 &&233 requests.length >= 1.5 * this.input.maxResultsPerCrawl234 ) {235 break;236 }237 requests.push(request);238 }239
240 const { waitForAllRequestsToBeAdded } =241 await this.requestQueue.addRequestsBatched(requests);242
243 void waitForAllRequestsToBeAdded.then(async () => {244 await this.keyValueStore.setValue(245 REQUEST_QUEUE_INIT_FLAG_KEY,246 '1',247 );248 });249 }250
251 // Dataset252 this.dataset = await Dataset.open(this.datasetName);253 const info = await this.dataset.getInfo();254 this.pagesOutputted = info?.itemCount ?? 0;255 }256
257 /**258 * Resolves to a `PlaywrightCrawler` instance.259 */260 async createCrawler() {261 await this.initPromise;262
263 const args = [];264 if (this.input.ignoreCorsAndCsp) args.push('--disable-web-security');265
266 const options: PlaywrightCrawlerOptions = {267 requestHandler: this._requestHandler.bind(this),268 requestQueue: this.requestQueue,269 requestHandlerTimeoutSecs: this.devtools270 ? DEVTOOLS_TIMEOUT_SECS271 : this.input.pageFunctionTimeoutSecs,272 preNavigationHooks: [],273 postNavigationHooks: [],274 failedRequestHandler: this._failedRequestHandler.bind(this),275 respectRobotsTxtFile: this.input.respectRobotsTxtFile,276 maxConcurrency: this.input.maxConcurrency,277 maxRequestRetries: this.input.maxRequestRetries,278 maxRequestsPerCrawl: this.input.maxPagesPerCrawl,279 proxyConfiguration: (await Actor.createProxyConfiguration(280 this.input.proxyConfiguration,281 )) as any as ProxyConfiguration,282 launchContext: {283 useChrome: this.input.useChrome,284 launcher: playwright[this.input.launcher],285 launchOptions: {286 ignoreHTTPSErrors: this.input.ignoreSslErrors,287 bypassCSP: this.input.ignoreCorsAndCsp,288 defaultViewport: DEFAULT_VIEWPORT,289 devtools: this.devtools,290 args,291 headless: this.input.headless,292 },293 } as PlaywrightLaunchContext,294 useSessionPool: true,295 persistCookiesPerSession: true,296 sessionPoolOptions: {297 persistStateKeyValueStoreId: this.input.sessionPoolName298 ? SESSION_STORE_NAME299 : undefined,300 persistStateKey: this.input.sessionPoolName,301 sessionOptions: {302 maxUsageCount: this.maxSessionUsageCount,303 },304 },305 experiments: {306 requestLocking: true,307 },308 };309
310 this._createNavigationHooks(options);311
312 if (this.input.proxyRotation === ProxyRotation.UntilFailure) {313 options.sessionPoolOptions!.maxPoolSize = 1;314 }315
316 this.crawler = new PlaywrightCrawler(options);317
318 return this.crawler;319 }320
321 private _createNavigationHooks(options: PlaywrightCrawlerOptions) {322 options.preNavigationHooks!.push(323 async ({ request, page, session, blockRequests }, gotoOptions) => {324 // Attach a console listener to get all logs from Browser context.325 if (this.input.browserLog) browserTools.dumpConsole(page);326
327 // Prevent download of stylesheets and media, unless selected otherwise328 if (this.blockedUrlPatterns.length) {329 await blockRequests({330 urlPatterns: this.blockedUrlPatterns,331 });332 }333
334 // Add initial cookies, if any.335 if (336 this.input.initialCookies &&337 this.input.initialCookies.length338 ) {339 const cookiesToSet = session340 ? tools.getMissingCookiesFromSession(341 session,342 this.input.initialCookies,343 request.url,344 )345 : this.input.initialCookies;346
347 if (cookiesToSet?.length) {348 // setting initial cookies that are not already in the session and page349 session?.setCookies(cookiesToSet, request.url);350 await page.context().addCookies(cookiesToSet);351 }352 }353
354 if (gotoOptions) {355 gotoOptions.timeout =356 (this.devtools357 ? DEVTOOLS_TIMEOUT_SECS358 : this.input.pageLoadTimeoutSecs) * 1000;359 gotoOptions.waitUntil = this.input.waitUntil;360 }361 },362 );363
364 options.preNavigationHooks!.push(365 ...this._runHookWithEnhancedContext(this.evaledPreNavigationHooks),366 );367 options.postNavigationHooks!.push(368 ...this._runHookWithEnhancedContext(this.evaledPostNavigationHooks),369 );370 }371
372 private _runHookWithEnhancedContext(373 hooks: ((...args: unknown[]) => Awaitable<void>)[],374 ) {375 return hooks.map((hook) => (ctx: Dictionary, ...args: unknown[]) => {376 const { customData } = this.input;377 return hook({ ...ctx, Apify: Actor, Actor, customData }, ...args);378 });379 }380
381 private async _failedRequestHandler({382 request,383 }: PlaywrightCrawlingContext) {384 const lastError =385 request.errorMessages[request.errorMessages.length - 1];386 const errorMessage = lastError ? lastError.split('\n')[0] : 'no error';387 log.error(388 `Request ${request.url} failed and will not be retried anymore. Marking as failed.\nLast Error Message: ${errorMessage}`,389 );390 return this._handleResult(request, undefined, undefined, true);391 }392
393 /**394 * First of all, it initializes the state that is exposed to the user via395 * `pageFunction` context.396 *397 * Then it invokes the user provided `pageFunction` with the prescribed context398 * and saves its return value.399 *400 * Finally, it makes decisions based on the current state and post-processes401 * the data returned from the `pageFunction`.402 */403 private async _requestHandler(crawlingContext: PlaywrightCrawlingContext) {404 const { request, response, crawler } = crawlingContext;405
406 /**407 * PRE-PROCESSING408 */409 // Make sure that an object containing internal metadata410 // is present on every request.411 tools.ensureMetaData(request);412
413 // Abort the crawler if the maximum number of results was reached.414 const aborted = await this._handleMaxResultsPerCrawl(415 crawler.autoscaledPool,416 );417 if (aborted) return;418
419 const pageFunctionArguments: Dictionary = {};420
421 // We must use properties and descriptors not to trigger getters / setters.422 Object.defineProperties(423 pageFunctionArguments,424 Object.getOwnPropertyDescriptors(crawlingContext),425 );426
427 pageFunctionArguments.response = {428 status: response && response.status(),429 headers: response && response.headers(),430 };431
432 // Setup and create Context.433 const contextOptions = {434 crawlerSetup: {435 rawInput: this.rawInput,436 env: this.env,437 globalStore: this.globalStore,438 requestQueue: this.requestQueue,439 keyValueStore: this.keyValueStore,440 customData: this.input.customData,441 },442 pageFunctionArguments,443 };444 const { context, state } = createContext(contextOptions);445
446 if (this.input.closeCookieModals) {447 await sleep(500);448 await crawlingContext.page.evaluate(getInjectableScript());449 await sleep(2000);450 }451
452 if (this.input.maxScrollHeightPixels > 0) {453 await crawlingContext.infiniteScroll({454 maxScrollHeight: this.input.maxScrollHeightPixels,455 });456 }457
458 /**459 * USER FUNCTION INVOCATION460 */461 const pageFunctionResult = await this.evaledPageFunction(context);462
463 /**464 * POST-PROCESSING465 */466 // Enqueue more links if Pseudo URLs and a link selector are available,467 // unless the user invoked the `skipLinks()` context function468 // or maxCrawlingDepth would be exceeded.469 if (!state.skipLinks) await this._handleLinks(crawlingContext);470
471 // Save the `pageFunction`s result to the default dataset.472 await this._handleResult(473 request,474 response,475 pageFunctionResult as Dictionary,476 );477 }478
479 private async _handleMaxResultsPerCrawl(autoscaledPool?: AutoscaledPool) {480 if (481 !this.input.maxResultsPerCrawl ||482 this.pagesOutputted < this.input.maxResultsPerCrawl483 )484 return false;485 if (!autoscaledPool) return false;486 log.info(487 `User set limit of ${this.input.maxResultsPerCrawl} results was reached. Finishing the crawl.`,488 );489 await autoscaledPool.abort();490 return true;491 }492
493 private async _handleLinks({494 request,495 enqueueLinks,496 }: PlaywrightCrawlingContext) {497 if (!this.requestQueue) return;498 const currentDepth = (request.userData![META_KEY] as RequestMetadata)499 .depth;500 const hasReachedMaxDepth =501 this.input.maxCrawlingDepth &&502 currentDepth >= this.input.maxCrawlingDepth;503 if (hasReachedMaxDepth) {504 log.debug(505 `Request ${request.url} reached the maximum crawling depth of ${currentDepth}.`,506 );507 return;508 }509
510 const enqueueOptions: EnqueueLinksOptions = {511 globs: this.input.globs,512 pseudoUrls: this.input.pseudoUrls,513 exclude: this.input.excludes,514 transformRequestFunction: (requestOptions) => {515 requestOptions.userData ??= {};516 requestOptions.userData[META_KEY] = {517 parentRequestId: request.id || request.uniqueKey,518 depth: currentDepth + 1,519 };520
521 requestOptions.useExtendedUniqueKey = true;522 requestOptions.keepUrlFragment = this.input.keepUrlFragments;523 return requestOptions;524 },525 };526
527 if (this.input.linkSelector) {528 await enqueueLinks({529 ...enqueueOptions,530 selector: this.input.linkSelector,531 });532 }533 }534
535 private async _handleResult(536 request: Request,537 response?: Response,538 pageFunctionResult?: Dictionary,539 isError?: boolean,540 ) {541 const payload = tools.createDatasetPayload(542 request,543 response,544 pageFunctionResult,545 isError,546 );547 await this.dataset.pushData(payload);548 this.pagesOutputted++;549 }550}