Pricing
Pay per usage
Crawls arbitrary websites using a web browser and extracts structured data from web pages using a provided JavaScript function. The Actor supports both recursive crawling and lists of URLs, and automatically manages concurrency for maximum performance.
4.8 (27)
Pricing
Pay per usage
1K
97K
5.1K
Issues response
1.7 days
Last modified
19 days ago
{ "extends": "@apify/tsconfig", "compilerOptions": { "outDir": "dist", "module": "ESNext", "allowJs": true, "skipLibCheck": true, "lib": ["dom"] }, "include": ["src"]}FROM apify/actor-node-puppeteer-chrome:22 AS builder
COPY package*.json ./
RUN npm install --include=dev --audit=false
COPY . ./
RUN npm run build
FROM apify/actor-node-puppeteer-chrome:22
COPY /home/myuser/dist ./dist
COPY package*.json ./
RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version \ && rm -r ~/.npm
COPY . ./
ENV APIFY_DISABLE_OUTDATED_WARNING=1
CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent{ "name": "actor-web-scraper", "version": "3.1.0", "private": true, "description": "Crawl web pages using Apify, headless Chrome and Puppeteer", "main": "dist/main.js", "type": "module", "dependencies": { "@apify/scraper-tools": "^1.1.4", "@crawlee/puppeteer": "^3.14.1", "apify": "^3.2.6", "content-type": "^1.0.5", "crawlee": "^3.13.2", "devtools-server": "^0.0.2", "idcac-playwright": "^0.1.3", "puppeteer": "*" }, "devDependencies": { "@apify/tsconfig": "^0.1.0", "@types/content-type": "^1.1.8", "@types/node": "^22.7.4", "tsx": "^4.19.1", "typescript": "~5.9.0" }, "scripts": { "start": "npm run start:dev", "start:prod": "node dist/main.js", "start:dev": "tsx src/main.ts", "build": "tsc" }, "repository": { "type": "git", "url": "https://github.com/apify/apify-sdk-js" }, "author": { "name": "Apify Technologies", "email": "support@apify.com", "url": "https://apify.com" }, "contributors": [ "Marek Trunkat <marek@apify.com>", "Ondra Urban <ondra@apify.com>" ], "license": "Apache-2.0", "homepage": "https://github.com/apify/apify-sdk-js"}# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed filesnode_modules1import { runActor } from '@apify/scraper-tools';2
3import { CrawlerSetup } from './internals/crawler_setup.js';4
5runActor(CrawlerSetup);1import { launchPuppeteer } from '@crawlee/puppeteer';2import type { Browser, Page } from 'puppeteer';3
4import { createBundle } from '../src/internals/bundle.browser';5
6const NAMESPACE = 'Apify';7
8describe('Bundle', () => {9 let browser: Browser;10 let page: Page;11
12 beforeAll(async () => {13 browser = await launchPuppeteer({ launchOptions: { headless: true } });14 });15
16 afterAll(async () => {17 await browser.close();18 });19
20 beforeEach(async () => {21 page = await browser.newPage();22 await page.evaluateOnNewDocument(createBundle, NAMESPACE);23 });24
25 afterEach(async () => {26 await page.close();27 });28
29 describe('Context', () => {30 const CONTEXT_OPTIONS = {31 crawlerSetup: {32 rawInput: '{}',33 },34 browserHandles: {35 apify: {},36 globalStore: {},37 log: {},38 },39 pageFunctionArguments: {40 request: {},41 },42 };43
44 beforeEach(async () => {45 await page.goto('about:chrome');46 await page.waitForFunction(47 (namespace: string) => !!window[namespace],48 {},49 NAMESPACE,50 );51 await page.evaluate(52 (namespace: string, contextOptions) => {53 window.contextInstance =54 window[namespace].createContext(contextOptions);55 },56 NAMESPACE,57 CONTEXT_OPTIONS,58 );59 });60
61 describe('waitFor', () => {62 it('should work with a number', async () => {63 const millis = await page.evaluate(async () => {64 const ctx = window.contextInstance;65 const start = Date.now();66 await ctx.waitFor(10);67 return Date.now() - start;68 });69 expect(millis).toBeGreaterThan(9);70 });71
72 it('should work with a selector', async () => {73 const millis = await page.evaluate(async () => {74 const ctx = window.contextInstance;75 const start = Date.now();76 setTimeout(() => {77 const el = document.createElement('div');78 el.id = 'very-unique-id';79 document.body.appendChild(el);80 }, 10);81 await ctx.waitFor('#very-unique-id');82 return Date.now() - start;83 });84 expect(millis).toBeGreaterThan(9);85 });86
87 it('should work with a function', async () => {88 const millis = await page.evaluate(async () => {89 const ctx = window.contextInstance;90 let done = false;91 const start = Date.now();92 setTimeout(() => {93 done = true;94 }, 10);95 await ctx.waitFor(() => done);96 return Date.now() - start;97 });98 expect(millis).toBeGreaterThan(9);99 });100 });101 });102});1/**2 * GlobalStore is a trivial storage that resembles a Map to be used from Browser contexts3 * to retain data through page navigations and browser instances. It limits Map's functionality4 * because it's currently impossible for functions and object references to cross Node-Browser threshold.5 */6export class GlobalStore<V> extends Map<string, V> {7 override get(key: string) {8 if (typeof key !== 'string') {9 throw new Error(10 'GlobalStore#get parameter "key" must be a string.',11 );12 }13 return super.get(key);14 }15
16 override set(key: string, value: V) {17 if (typeof key !== 'string') {18 throw new Error(19 'GlobalStore#set parameter "key" must be a string.',20 );21 }22 return super.set(key, value);23 }24
25 override forEach(): never {26 throw new Error(27 'GlobalStore#forEach function is not available due to underlying technology limitations.',28 );29 }30
31 override values() {32 return Array.from(super.values()) as any;33 }34
35 override keys() {36 return Array.from(super.keys()) as any;37 }38
39 override entries() {40 return Array.from(super.entries()) as any;41 }42}1import type {2 Dictionary,3 GlobInput,4 ProxyConfigurationOptions,5 PseudoUrlInput,6 RegExpInput,7 RequestOptions,8 Session,9} from '@crawlee/puppeteer';10import type { PuppeteerLifeCycleEvent } from 'puppeteer';11
12/**13 * Port where the remote debugging interface in Chrome14 * should be started.15 */16export const CHROME_DEBUGGER_PORT = 9222;17
18export const enum RunMode {19 Production = 'PRODUCTION',20 Development = 'DEVELOPMENT',21}22
23export const enum ProxyRotation {24 Recommended = 'RECOMMENDED',25 PerRequest = 'PER_REQUEST',26 UntilFailure = 'UNTIL_FAILURE',27}28
29export const enum BreakpointLocation {30 None = 'NONE',31 BeforeGoto = 'BEFORE_GOTO',32 BeforePageFunction = 'BEFORE_PAGE_FUNCTION',33 AfterPageFunction = 'AFTER_PAGE_FUNCTION',34}35
36declare global {37 interface Window {38 [K: string]: any;39 }40 // eslint-disable-next-line vars-on-top41 var window: Window & typeof globalThis;42 // eslint-disable-next-line vars-on-top43 var document: Document;44}45
46/**47 * Replicates the INPUT_SCHEMA with TypeScript types for quick reference48 * and IDE type check integration.49 */50export interface Input {51 startUrls: RequestOptions[];52 pageFunction: string;53 runMode: RunMode;54 keepUrlFragments: boolean;55 respectRobotsTxtFile: boolean;56 linkSelector?: string;57 globs: GlobInput[];58 regexps: RegExpInput[];59 pseudoUrls: PseudoUrlInput[];60 excludes: GlobInput[];61 preNavigationHooks?: string;62 postNavigationHooks?: string;63 injectJQuery: boolean;64 proxyConfiguration: ProxyConfigurationOptions;65 proxyRotation: ProxyRotation;66 sessionPoolName?: string;67 initialCookies: Parameters<Session['setCookies']>[0];68 useChrome: boolean;69 maxScrollHeightPixels: number;70 ignoreSslErrors: boolean;71 ignoreCorsAndCsp: boolean;72 closeCookieModals: boolean;73 downloadMedia: boolean;74 downloadCss: boolean;75 maxRequestRetries: number;76 maxPagesPerCrawl: number;77 maxResultsPerCrawl: number;78 maxCrawlingDepth: number;79 maxConcurrency: number;80 pageLoadTimeoutSecs: number;81 pageFunctionTimeoutSecs: number;82 waitUntil: PuppeteerLifeCycleEvent[];83 breakpointLocation: BreakpointLocation;84 debugLog: boolean;85 browserLog: boolean;86 customData: Dictionary;87 datasetName?: string;88 keyValueStoreName?: string;89 requestQueueName?: string;90 headless: boolean;91}1/* eslint-disable max-classes-per-file */2
3import type {4 Dictionary,5 KeyValueStore,6 RecordOptions,7 Request,8 RequestOptions,9 RequestQueue,10 RequestQueueOperationOptions,11} from '@crawlee/puppeteer';12import type { ApifyEnv } from 'apify';13
14import type { Log } from '@apify/log';15import type {16 constants,17 CrawlerSetupOptions,18 RequestMetadata,19} from '@apify/scraper-tools';20
21import type { Input } from './consts.ts';22import type { GlobalStore } from './global_store.ts';23
24interface PoolOptions {25 pollingIntervalMillis?: number;26 timeoutMillis?: number;27}28
29interface InternalState {30 browserHandles: Dictionary<31 | string32 | Record<33 string,34 { value: unknown; type: 'METHOD' | 'VALUE' | 'GETTER' }35 >36 >;37 requestQueue: RequestQueue | null;38 keyValueStore: KeyValueStore | null;39}40
41interface ProvidedResponse {42 status: number;43 headers: Dictionary<string>;44}45
46interface BrowserCrawlerSetup extends CrawlerSetupOptions {47 injectJQuery?: boolean;48 META_KEY: typeof constants.META_KEY;49}50
51interface ContextOptions {52 crawlerSetup: BrowserCrawlerSetup;53 browserHandles: InternalState['browserHandles'];54 pageFunctionArguments: Dictionary<unknown>;55}56
57/**58 * Command to be evaluated for Browser side code injection.59 * @param apifyNamespace60 */61export function createBundle(apifyNamespace: string) {62 (function (global, namespace) {63 if (typeof window[namespace] !== 'object') window[namespace] = {};64 /**65 * Takes a configuration object with function references66 * as an input and creates a dummy class that proxies67 * function invocations from Browser context to Node context.68 *69 * The configuration is to be provided by the70 * tools.createBrowserHandlesForObject function.71 */72 class NodeProxy {73 constructor(74 config: Dictionary<{75 value: unknown;76 type: 'METHOD' | 'VALUE' | 'GETTER';77 }>,78 ) {79 if (!config || typeof config !== 'object') {80 throw new Error(81 'NodeProxy: Parameter config of type Object must be provided.',82 );83 }84
85 Object.entries(config).forEach(([key, { value, type }]) => {86 if (type === 'METHOD') {87 // @ts-expect-error88 this[key] = (...args: unknown[]) =>89 // @ts-expect-error90 global[value](...args);91 } else if (type === 'GETTER') {92 Object.defineProperty(this, key, {93 // @ts-expect-error94 get: () => global[value](),95 });96 } else if (type === 'VALUE') {97 // @ts-expect-error98 this[key] = value;99 } else {100 throw new Error(101 `Unsupported function type: ${type} for function: ${key}.`,102 );103 }104 });105 }106 }107
108 /**109 * Exposed factory.110 * @param config111 */112 global[namespace].createNodeProxy = (113 config: Dictionary<{114 value: unknown;115 type: 'METHOD' | 'VALUE' | 'GETTER';116 }>,117 ) => new NodeProxy(config);118
119 const setup = Symbol('crawler-setup');120 const internalState = Symbol('request-internal-state');121
122 /**123 * Context represents everything that is available to the user124 * via Page Function. A class is used instead of a simple object125 * to avoid having to create new instances of functions with each126 * request.127 *128 * Some properties need to be accessible to the Context,129 * but should not be exposed to the user thus they are hidden130 * using a Symbol to prevent the user from easily accessing131 * and manipulating them.132 */133 class Context {134 [setup]: BrowserCrawlerSetup;135 [internalState]: InternalState;136 input: Input;137 env: ApifyEnv;138 customData: unknown;139 response: ProvidedResponse;140 request: Request;141 globalStore: GlobalStore<unknown>;142 log: Log;143
144 jQuery: any;145
146 constructor(options: ContextOptions) {147 const { crawlerSetup, browserHandles, pageFunctionArguments } =148 options;149
150 const createProxy = global[namespace].createNodeProxy;151
152 // Private153 this[setup] = crawlerSetup;154 this[internalState] = {155 browserHandles,156 requestQueue: browserHandles.requestQueue157 ? createProxy(browserHandles.requestQueue)158 : null,159 keyValueStore: browserHandles.keyValueStore160 ? createProxy(browserHandles.keyValueStore)161 : null,162 };163
164 // Copies of Node objects165 this.input = JSON.parse(crawlerSetup.rawInput);166 this.env = { ...crawlerSetup.env };167 this.customData = crawlerSetup.customData;168 this.response =169 pageFunctionArguments.response as ProvidedResponse;170 this.request = pageFunctionArguments.request as Request;171 // Functions are not converted so we need to add them this way172 // to not be enumerable and thus not polluting the object.173 Reflect.defineProperty(this.request, 'pushErrorMessage', {174 value(this: Request, errorOrMessage: Error) {175 // It's a simplified fake of the original function.176 const msg =177 (errorOrMessage && errorOrMessage.message) ||178 `${errorOrMessage}`;179 this.errorMessages.push(msg);180 },181 enumerable: false,182 });183
184 // Proxied Node objects185 this.globalStore = createProxy(browserHandles.globalStore);186 this.log = createProxy(browserHandles.log);187
188 // Browser side libraries189 if (this[setup].injectJQuery)190 this.jQuery = global.jQuery.noConflict(true);191
192 // Bind this to allow destructuring off context in pageFunction.193 this.getValue = this.getValue.bind(this);194 this.setValue = this.setValue.bind(this);195 this.saveSnapshot = this.saveSnapshot.bind(this);196 this.skipLinks = this.skipLinks.bind(this);197 this.enqueueRequest = this.enqueueRequest.bind(this);198 this.waitFor = this.waitFor.bind(this);199 }200
201 async getValue<T>(...args: Parameters<KeyValueStore['getValue']>) {202 return this[internalState].keyValueStore!.getValue(203 ...args,204 ) as Promise<T>;205 }206
207 async setValue<T>(...args: Parameters<KeyValueStore['setValue']>) {208 return this[internalState].keyValueStore!.setValue(209 ...(args as [210 key: string,211 value: T | null,212 options?: RecordOptions,213 ]),214 );215 }216
217 async saveSnapshot() {218 const handle = this[internalState].browserHandles219 .saveSnapshot as string;220 return global[handle]();221 }222
223 async skipLinks() {224 const handle = this[internalState].browserHandles225 .skipLinks as string;226 return global[handle]();227 }228
229 async enqueueRequest(230 requestOpts: RequestOptions = {} as RequestOptions,231 options: RequestQueueOperationOptions = {},232 ) {233 const defaultRequestOpts = {234 useExtendedUniqueKey: true,235 keepUrlFragment: this.input.keepUrlFragments,236 };237
238 const newRequest = { ...defaultRequestOpts, ...requestOpts };239
240 const metaKey = this[setup].META_KEY;241 const defaultUserData = {242 [metaKey]: {243 parentRequestId:244 this.request.id || this.request.uniqueKey,245 depth:246 (this.request.userData![metaKey] as RequestMetadata)247 .depth + 1,248 },249 };250
251 newRequest.userData = {252 ...defaultUserData,253 ...requestOpts.userData,254 };255
256 return this[internalState].requestQueue!.addRequest(257 newRequest,258 options,259 );260 }261
262 async waitFor(263 selectorOrNumberOrFunction:264 | string265 | number266 | ((...args: unknown[]) => boolean),267 options = {},268 ) {269 if (!options || typeof options !== 'object')270 throw new Error('Parameter options must be an Object');271 const type = typeof selectorOrNumberOrFunction;272 if (type === 'string') {273 return this._waitForSelector(274 selectorOrNumberOrFunction as string,275 options,276 );277 }278 if (type === 'number') {279 return this._waitForMillis(280 selectorOrNumberOrFunction as number,281 );282 }283 if (type === 'function') {284 return this._waitForFunction(285 selectorOrNumberOrFunction as (286 ...args: unknown[]287 ) => boolean,288 options,289 );290 }291 throw new Error(292 'Parameter selectorOrNumberOrFunction must be one of the said types.',293 );294 }295
296 async _waitForSelector(selector: string, options = {}) {297 try {298 await this._poll(() => {299 return !!global.document.querySelector(selector);300 }, options);301 } catch (err) {302 const casted = err as Error;303 if (/timeout of \d+ms exceeded/.test(casted.message)) {304 throw new Error(305 `Timeout Error: waiting for selector failed: ${casted.message}`,306 );307 }308 throw err;309 }310 }311
312 async _waitForMillis(millis: number) {313 return new Promise((res) => {314 setTimeout(res, millis);315 });316 }317
318 async _waitForFunction(319 predicate: () => boolean,320 options: PoolOptions = {},321 ) {322 try {323 await this._poll(predicate, options);324 } catch (err) {325 const casted = err as Error;326 if (/timeout of \d+ms exceeded/.test(casted.message)) {327 throw new Error(328 `Timeout Error: waiting for function failed: ${casted.message}`,329 );330 }331 throw err;332 }333 }334
335 async _poll(predicate: () => boolean, options: PoolOptions = {}) {336 const { pollingIntervalMillis = 50, timeoutMillis = 20000 } =337 options;338 return new Promise<void>((resolve, reject) => {339 const handler = (): void => {340 if (predicate()) {341 resolve();342 } else {343 setTimeout(handler);344 }345 };346 const pollTimeout = setTimeout(347 handler,348 pollingIntervalMillis,349 );350 setTimeout(() => {351 clearTimeout(pollTimeout);352 return reject(353 new Error(354 `timeout of ${timeoutMillis}ms exceeded.`,355 ),356 );357 }, timeoutMillis);358 });359 }360 }361
362 /**363 * Exposed factory.364 */365 global[namespace].createContext = (options: ContextOptions) => {366 return new Context(options);367 };368 })(window, apifyNamespace);369}1import { readFile } from 'node:fs/promises';2import { dirname } from 'node:path';3import { setTimeout } from 'node:timers/promises';4import { fileURLToPath, URL } from 'node:url';5
6import type {7 AutoscaledPool,8 Awaitable,9 Dictionary,10 ProxyConfiguration,11 PuppeteerCrawlerOptions,12 PuppeteerCrawlingContext,13 Request,14} from '@crawlee/puppeteer';15import {16 Dataset,17 KeyValueStore,18 log,19 PuppeteerCrawler,20 puppeteerUtils,21 RequestList,22 RequestQueueV2,23} from '@crawlee/puppeteer';24import type { ApifyEnv } from 'apify';25import { Actor } from 'apify';26import contentType from 'content-type';27// @ts-expect-error no typings28import DevToolsServer from 'devtools-server';29import { getInjectableScript } from 'idcac-playwright';30import type { HTTPResponse, Page } from 'puppeteer';31
32import type { CrawlerSetupOptions, createContext } from '@apify/scraper-tools';33import {34 browserTools,35 constants as scraperToolsConstants,36 tools,37} from '@apify/scraper-tools';38
39import { createBundle } from './bundle.browser.js';40import type { Input } from './consts.js';41import {42 BreakpointLocation,43 CHROME_DEBUGGER_PORT,44 ProxyRotation,45 RunMode,46} from './consts.js';47import { GlobalStore } from './global_store.js';48
49const SESSION_STORE_NAME = 'APIFY-WEB-SCRAPER-SESSION-STORE';50const REQUEST_QUEUE_INIT_FLAG_KEY = 'REQUEST_QUEUE_INITIALIZED';51
52const MAX_CONCURRENCY_IN_DEVELOPMENT = 1;53const {54 SESSION_MAX_USAGE_COUNTS,55 DEFAULT_VIEWPORT,56 DEVTOOLS_TIMEOUT_SECS,57 META_KEY,58} = scraperToolsConstants;59const SCHEMA = JSON.parse(60 await readFile(new URL('../../INPUT_SCHEMA.json', import.meta.url), 'utf8'),61);62
63interface PageContext {64 apifyNamespace: string;65 skipLinks: boolean;66 timers: {67 start: [number, number];68 navStart?: [number, number];69 };70
71 browserHandles?: Awaited<ReturnType<CrawlerSetup['_injectBrowserHandles']>>;72}73
74interface Output {75 pageFunctionResult?: Dictionary;76 pageFunctionError?: tools.ErrorLike;77 requestFromBrowser: Request;78}79
80/**81 * Holds all the information necessary for constructing a crawler82 * instance and creating a context for a pageFunction invocation.83 */84export class CrawlerSetup implements CrawlerSetupOptions {85 name = 'Web Scraper';86 rawInput: string;87 env: ApifyEnv;88 /**89 * Used to store data that persist navigations90 */91 globalStore = new GlobalStore<unknown>();92 requestQueue: RequestQueueV2;93 keyValueStore: KeyValueStore;94 customData: unknown;95 input: Input;96 maxSessionUsageCount: number;97 evaledPreNavigationHooks: ((...args: unknown[]) => Awaitable<void>)[];98 evaledPostNavigationHooks: ((...args: unknown[]) => Awaitable<void>)[];99
100 /**101 * Used to store page specific data.102 */103 pageContexts = new WeakMap<Page, PageContext>();104
105 blockedUrlPatterns: string[] = [];106 isDevRun: boolean;107 datasetName?: string;108 keyValueStoreName?: string;109 requestQueueName?: string;110
111 crawler!: PuppeteerCrawler;112 dataset!: Dataset;113 pagesOutputted!: number;114 private initPromise: Promise<void>;115
116 constructor(input: Input) {117 // Set log level early to prevent missed messages.118 if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);119
120 // Keep this as string to be immutable.121 this.rawInput = JSON.stringify(input);122
123 // Attempt to load page function from disk if not present on input.124 tools.maybeLoadPageFunctionFromDisk(125 input,126 dirname(fileURLToPath(import.meta.url)),127 );128
129 // Validate INPUT if not running on Apify Cloud Platform.130 if (!Actor.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);131
132 this.input = input;133 this.env = Actor.getEnv();134
135 // Validations136 this.input.pseudoUrls.forEach((purl) => {137 if (!tools.isPlainObject(purl)) {138 throw new Error(139 'The pseudoUrls Array must only contain Objects.',140 );141 }142 if (purl.userData && !tools.isPlainObject(purl.userData)) {143 throw new Error(144 'The userData property of a pseudoUrl must be an Object.',145 );146 }147 });148
149 this.input.initialCookies.forEach((cookie) => {150 if (!tools.isPlainObject(cookie)) {151 throw new Error(152 'The initialCookies Array must only contain Objects.',153 );154 }155 });156
157 this.input.waitUntil.forEach((event) => {158 if (159 !/^(domcontentloaded|load|networkidle2|networkidle0)$/.test(160 event,161 )162 ) {163 throw new Error(164 'Navigation wait until events must be valid. See tooltip.',165 );166 }167 });168
169 // solving proxy rotation settings170 this.maxSessionUsageCount =171 SESSION_MAX_USAGE_COUNTS[this.input.proxyRotation];172
173 tools.evalFunctionOrThrow(this.input.pageFunction);174
175 if (this.input.preNavigationHooks) {176 this.evaledPreNavigationHooks = tools.evalFunctionArrayOrThrow(177 this.input.preNavigationHooks,178 'preNavigationHooks',179 );180 } else {181 this.evaledPreNavigationHooks = [];182 }183
184 if (this.input.postNavigationHooks) {185 this.evaledPostNavigationHooks = tools.evalFunctionArrayOrThrow(186 this.input.postNavigationHooks,187 'postNavigationHooks',188 );189 } else {190 this.evaledPostNavigationHooks = [];191 }192
193 // Excluded resources194 if (!this.input.downloadMedia) {195 this.blockedUrlPatterns = [196 '.jpg',197 '.jpeg',198 '.png',199 '.svg',200 '.gif',201 '.webp',202 '.webm',203 '.ico',204 '.woff',205 '.eot',206 ];207 }208
209 if (!this.input.downloadCss) {210 this.blockedUrlPatterns.push('.css');211 }212
213 this.isDevRun = this.input.runMode === RunMode.Development;214
215 // Named storages216 this.datasetName = this.input.datasetName;217 this.keyValueStoreName = this.input.keyValueStoreName;218 this.requestQueueName = this.input.requestQueueName;219
220 // Initialize async operations.221 this.crawler = null!;222 this.requestQueue = null!;223 this.dataset = null!;224 this.keyValueStore = null!;225 this.initPromise = this._initializeAsync();226 }227
228 private async _initializeAsync() {229 // RequestList230 const startUrls = this.input.startUrls.map((req) => {231 req.useExtendedUniqueKey = true;232 req.keepUrlFragment = this.input.keepUrlFragments;233 return req;234 });235
236 // KeyValueStore237 this.keyValueStore = await KeyValueStore.open(this.keyValueStoreName);238
239 // RequestQueue240 this.requestQueue = await RequestQueueV2.open(this.requestQueueName);241
242 if (243 !(await this.keyValueStore.recordExists(244 REQUEST_QUEUE_INIT_FLAG_KEY,245 ))246 ) {247 const requests: Request[] = [];248 for await (const request of await RequestList.open(249 null,250 startUrls,251 )) {252 if (253 this.input.maxResultsPerCrawl > 0 &&254 requests.length >= 1.5 * this.input.maxResultsPerCrawl255 ) {256 break;257 }258 requests.push(request);259 }260
261 const { waitForAllRequestsToBeAdded } =262 await this.requestQueue.addRequestsBatched(requests);263
264 void waitForAllRequestsToBeAdded.then(async () => {265 await this.keyValueStore.setValue(266 REQUEST_QUEUE_INIT_FLAG_KEY,267 '1',268 );269 });270 }271
272 // Dataset273 this.dataset = await Dataset.open(this.datasetName);274 const info = await this.dataset.getInfo();275 this.pagesOutputted = info?.itemCount ?? 0;276 }277
278 /**279 * Resolves to a `PuppeteerCrawler` instance.280 * constructor.281 */282 async createCrawler() {283 await this.initPromise;284
285 const args = ['--disable-dev-shm-usage'];286 if (this.input.ignoreCorsAndCsp) args.push('--disable-web-security');287 if (this.isDevRun)288 args.push(`--remote-debugging-port=${CHROME_DEBUGGER_PORT}`);289
290 const options: PuppeteerCrawlerOptions = {291 requestHandler: this._requestHandler.bind(this),292 requestQueue: this.requestQueue,293 requestHandlerTimeoutSecs: this.isDevRun294 ? DEVTOOLS_TIMEOUT_SECS295 : this.input.pageFunctionTimeoutSecs,296 preNavigationHooks: [],297 postNavigationHooks: [],298 failedRequestHandler: this._failedRequestHandler.bind(this),299 respectRobotsTxtFile: this.input.respectRobotsTxtFile,300 maxConcurrency: this.isDevRun301 ? MAX_CONCURRENCY_IN_DEVELOPMENT302 : this.input.maxConcurrency,303 maxRequestRetries: this.input.maxRequestRetries,304 maxRequestsPerCrawl:305 this.input.maxPagesPerCrawl === 0306 ? undefined307 : this.input.maxPagesPerCrawl,308 proxyConfiguration: (await Actor.createProxyConfiguration(309 this.input.proxyConfiguration,310 )) as any as ProxyConfiguration,311 browserPoolOptions: {312 preLaunchHooks: [313 async () => {314 if (!this.isDevRun) {315 return;316 }317
318 const devToolsServer = new DevToolsServer({319 containerHost: new URL(320 process.env.ACTOR_WEB_SERVER_URL!,321 ).host,322 devToolsServerPort:323 process.env.ACTOR_WEB_SERVER_PORT,324 chromeRemoteDebuggingPort: CHROME_DEBUGGER_PORT,325 });326 await devToolsServer.start();327 },328 ],329 },330 launchContext: {331 useChrome: this.input.useChrome,332 launchOptions: {333 acceptInsecureCerts: this.input.ignoreSslErrors,334 defaultViewport: DEFAULT_VIEWPORT,335 args,336 headless: this.input.headless,337 },338 },339 useSessionPool: !this.isDevRun,340 persistCookiesPerSession: !this.isDevRun,341 sessionPoolOptions: {342 persistStateKeyValueStoreId: this.input.sessionPoolName343 ? SESSION_STORE_NAME344 : undefined,345 persistStateKey: this.input.sessionPoolName,346 sessionOptions: {347 maxUsageCount: this.maxSessionUsageCount,348 },349 },350 experiments: {351 requestLocking: true,352 },353 };354
355 this._createNavigationHooks(options);356
357 if (this.input.proxyRotation === ProxyRotation.UntilFailure) {358 options.sessionPoolOptions!.maxPoolSize = 1;359 }360 if (this.isDevRun) {361 options.browserPoolOptions!.retireBrowserAfterPageCount = Infinity;362 }363
364 this.crawler = new PuppeteerCrawler(options);365
366 if (this.isDevRun) logDevRunWarning();367 return this.crawler;368 }369
370 private _createNavigationHooks(options: PuppeteerCrawlerOptions) {371 options.preNavigationHooks!.push(372 async ({ request, page, session }, gotoOptions) => {373 const start = process.hrtime();374
375 // Create a new page context with a new random key for Apify namespace.376 const pageContext: PageContext = {377 apifyNamespace: await tools.createRandomHash(),378 skipLinks: false,379 timers: { start },380 };381 this.pageContexts.set(page, pageContext);382
383 // Attach a console listener to get all logs as soon as possible.384 if (this.input.browserLog) browserTools.dumpConsole(page);385
386 // Prevent download of stylesheets and media, unless selected otherwise387 if (this.blockedUrlPatterns.length) {388 await puppeteerUtils.blockRequests(page, {389 urlPatterns: this.blockedUrlPatterns,390 });391 }392
393 // Add initial cookies, if any.394 if (395 this.input.initialCookies &&396 this.input.initialCookies.length397 ) {398 const cookiesToSet = session399 ? tools.getMissingCookiesFromSession(400 session,401 this.input.initialCookies,402 request.url,403 )404 : this.input.initialCookies;405 if (cookiesToSet && cookiesToSet.length) {406 // setting initial cookies that are not already in the session and page407 // TODO: We can remove the condition when there is an option to define blocked status codes in sessionPool408 session?.setCookies(cookiesToSet, request.url);409 await page.setCookie(...cookiesToSet);410 }411 }412
413 // Disable content security policy.414 if (this.input.ignoreCorsAndCsp) await page.setBypassCSP(true);415
416 tools.logPerformance(request, 'gotoFunction INIT', start);417 const handleStart = process.hrtime();418 pageContext.browserHandles = await this._injectBrowserHandles(419 page,420 pageContext,421 );422 tools.logPerformance(423 request,424 'gotoFunction INJECTION HANDLES',425 handleStart,426 );427
428 const evalStart = process.hrtime();429 await Promise.all([430 page.evaluateOnNewDocument(431 createBundle,432 pageContext.apifyNamespace,433 ),434 page.evaluateOnNewDocument(435 browserTools.wrapPageFunction(436 this.input.pageFunction,437 pageContext.apifyNamespace,438 ),439 ),440 ]);441 tools.logPerformance(442 request,443 'gotoFunction INJECTION EVAL',444 evalStart,445 );446
447 if (this.isDevRun) {448 const cdpClient = await page.target().createCDPSession();449 await cdpClient.send('Debugger.enable');450 if (451 this.input.breakpointLocation ===452 BreakpointLocation.BeforeGoto453 ) {454 await cdpClient.send('Debugger.pause');455 }456 }457
458 pageContext.timers.navStart = process.hrtime();459 if (gotoOptions) {460 gotoOptions.timeout = this.input.pageLoadTimeoutSecs * 1000;461 gotoOptions.waitUntil = this.input.waitUntil;462 }463 },464 );465
466 options.preNavigationHooks!.push(467 ...this._runHookWithEnhancedContext(this.evaledPreNavigationHooks),468 );469 options.postNavigationHooks!.push(470 ...this._runHookWithEnhancedContext(this.evaledPostNavigationHooks),471 );472
473 options.postNavigationHooks!.push(474 async ({ request, page, response }) => {475 await this._waitForLoadEventWhenXml(page, response);476 const pageContext = this.pageContexts.get(page)!;477 tools.logPerformance(478 request,479 'gotoFunction NAVIGATION',480 pageContext.timers.navStart!,481 );482
483 const delayStart = process.hrtime();484 await this._assertNamespace(page, pageContext.apifyNamespace);485
486 // Inject selected libraries487 if (this.input.injectJQuery)488 await puppeteerUtils.injectJQuery(page);489
490 tools.logPerformance(491 request,492 'gotoFunction INJECTION DELAY',493 delayStart,494 );495 tools.logPerformance(496 request,497 'gotoFunction EXECUTION',498 pageContext.timers.start,499 );500 },501 );502 }503
504 private _runHookWithEnhancedContext(505 hooks: ((...args: unknown[]) => Awaitable<void>)[],506 ) {507 return hooks.map((hook) => (ctx: Dictionary, ...args: unknown[]) => {508 const { customData } = this.input;509 return hook({ ...ctx, customData }, ...args);510 });511 }512
513 private async _failedRequestHandler({ request }: PuppeteerCrawlingContext) {514 const lastError =515 request.errorMessages[request.errorMessages.length - 1];516 const errorMessage = lastError ? lastError.split('\n')[0] : 'no error';517 log.error(518 `Request ${request.url} failed and will not be retried anymore. Marking as failed.\nLast Error Message: ${errorMessage}`,519 );520 return this._handleResult(request, undefined, undefined, true);521 }522
523 /**524 * First of all, it initializes the state that is exposed to the user via525 * `pageFunction` context.526 *527 * Then it invokes the user provided `pageFunction` with the prescribed context528 * and saves its return value.529 *530 * Finally, it makes decisions based on the current state and post-processes531 * the data returned from the `pageFunction`.532 */533 private async _requestHandler(crawlingContext: PuppeteerCrawlingContext) {534 const { request, response, page, crawler, proxyInfo } = crawlingContext;535 const start = process.hrtime();536 const pageContext = this.pageContexts.get(page)!;537
538 /**539 * PRE-PROCESSING540 */541 // Make sure that an object containing internal metadata542 // is present on every request.543 tools.ensureMetaData(request);544
545 // Abort the crawler if the maximum number of results was reached.546 const aborted = await this._handleMaxResultsPerCrawl(547 crawler.autoscaledPool,548 );549 if (aborted) return;550
551 // Setup Context and pass the configuration down to Browser.552 const contextOptions = {553 crawlerSetup: {554 rawInput: this.rawInput,555 env: this.env,556 customData: this.input.customData,557 injectJQuery: this.input.injectJQuery,558 META_KEY,559 },560 browserHandles: pageContext.browserHandles,561 pageFunctionArguments: {562 request,563 proxyInfo,564 response: {565 status: response && response.status(),566 headers: response && response.headers(),567 },568 },569 };570
571 /**572 * USER FUNCTION EXECUTION573 */574 tools.logPerformance(request, 'requestHandler PREPROCESSING', start);575
576 if (577 this.isDevRun &&578 this.input.breakpointLocation ===579 BreakpointLocation.BeforePageFunction580 ) {581 await page.evaluate(async () => {582 // eslint-disable-next-line no-debugger -- Debugger is enabled in dev run583 debugger;584 });585 }586
587 if (this.input.closeCookieModals) {588 await setTimeout(500);589 await page.evaluate(getInjectableScript());590 await setTimeout(2000);591 }592
593 if (this.input.maxScrollHeightPixels > 0) {594 await crawlingContext.infiniteScroll({595 maxScrollHeight: this.input.maxScrollHeightPixels,596 });597 }598
599 const startUserFn = process.hrtime();600
601 const namespace = pageContext.apifyNamespace;602 const output = await page.evaluate(603 async (ctxOpts: typeof contextOptions, namespc: string) => {604 const context: ReturnType<typeof createContext>['context'] =605 window[namespc].createContext(ctxOpts);606 // eslint-disable-next-line @typescript-eslint/no-shadow607 const output = {} as Output;608 try {609 output.pageFunctionResult =610 await window[namespc].pageFunction(context);611 } catch (err) {612 const casted = err as Error;613 output.pageFunctionError = Object.getOwnPropertyNames(614 casted,615 ).reduce((memo, name) => {616 memo[name] = casted[name as keyof Error];617 return memo;618 }, {} as Dictionary);619 }620
621 // This needs to be added after pageFunction has run.622 output.requestFromBrowser = context.request as Request;623
624 /**625 * Since Dates cannot travel back to Node and Puppeteer does not use .toJSON626 * to stringify, they come back as empty objects. We could use JSON.stringify627 * ourselves, but that exposes us to overridden .toJSON in the target websites.628 * This hack is not ideal, but it avoids both problems.629 */630 function replaceAllDatesInObjectWithISOStrings(obj: Output) {631 for (const [key, value] of Object.entries(obj)) {632 if (633 value instanceof Date &&634 typeof value.toISOString === 'function'635 ) {636 Reflect.set(obj, key, value.toISOString());637 } else if (value && typeof value === 'object') {638 replaceAllDatesInObjectWithISOStrings(639 value as Output,640 );641 }642 }643 return obj;644 }645
646 return replaceAllDatesInObjectWithISOStrings(output);647 },648 contextOptions,649 namespace,650 );651
652 tools.logPerformance(653 request,654 'requestHandler USER FUNCTION',655 startUserFn,656 );657 const finishUserFn = process.hrtime();658
659 /**660 * POST-PROCESSING661 */662 const { pageFunctionResult, requestFromBrowser, pageFunctionError } =663 output;664 // Merge requestFromBrowser into request to preserve modifications that665 // may have been made in browser context.666 Object.assign(request, requestFromBrowser);667
668 // Throw error from pageFunction, if any.669 if (pageFunctionError) throw tools.createError(pageFunctionError);670
671 // Enqueue more links if a link selector is available,672 // unless the user invoked the `skipLinks()` context function673 // or maxCrawlingDepth would be exceeded.674 if (!pageContext.skipLinks) {675 await this._handleLinks(crawlingContext);676 }677
678 // Save the `pageFunction`s result (or just metadata) to the default dataset.679 await this._handleResult(request, response, pageFunctionResult);680
681 tools.logPerformance(682 request,683 'requestHandler POSTPROCESSING',684 finishUserFn,685 );686 tools.logPerformance(request, 'requestHandler EXECUTION', start);687
688 if (689 this.isDevRun &&690 this.input.breakpointLocation ===691 BreakpointLocation.AfterPageFunction692 ) {693 await page.evaluate(async () => {694 // eslint-disable-next-line no-debugger -- Debugger is enabled in dev run695 debugger;696 });697 }698 }699
700 private async _handleMaxResultsPerCrawl(autoscaledPool?: AutoscaledPool) {701 if (702 !this.input.maxResultsPerCrawl ||703 this.pagesOutputted < this.input.maxResultsPerCrawl704 )705 return false;706 if (!autoscaledPool) return false;707 log.info(708 `User set limit of ${this.input.maxResultsPerCrawl} results was reached. Finishing the crawl.`,709 );710 await autoscaledPool.abort();711 return true;712 }713
714 private async _handleLinks({715 request,716 enqueueLinks,717 }: PuppeteerCrawlingContext) {718 if (!(this.input.linkSelector && this.requestQueue)) return;719 const start = process.hrtime();720
721 const currentDepth = (722 request.userData![META_KEY] as tools.RequestMetadata723 ).depth;724 const hasReachedMaxDepth =725 this.input.maxCrawlingDepth &&726 currentDepth >= this.input.maxCrawlingDepth;727 if (hasReachedMaxDepth) {728 log.debug(729 `Request ${request.url} reached the maximum crawling depth of ${currentDepth}.`,730 );731 return;732 }733
734 await enqueueLinks({735 selector: this.input.linkSelector,736 globs: this.input.globs,737 pseudoUrls: this.input.pseudoUrls,738 exclude: this.input.excludes,739 transformRequestFunction: (requestOptions) => {740 requestOptions.userData ??= {};741 requestOptions.userData[META_KEY] = {742 parentRequestId: request.id || request.uniqueKey,743 depth: currentDepth + 1,744 };745
746 requestOptions.useExtendedUniqueKey = true;747 requestOptions.keepUrlFragment = this.input.keepUrlFragments;748 return requestOptions;749 },750 });751
752 tools.logPerformance(request, 'handleLinks EXECUTION', start);753 }754
755 private async _handleResult(756 request: Request,757 response?: HTTPResponse,758 pageFunctionResult?: Dictionary,759 isError?: boolean,760 ) {761 const start = process.hrtime();762 const payload = tools.createDatasetPayload(763 request,764 response,765 pageFunctionResult,766 isError,767 );768 await this.dataset.pushData(payload);769 this.pagesOutputted++;770 tools.logPerformance(request, 'handleResult EXECUTION', start);771 }772
773 private async _assertNamespace(page: Page, namespace: string) {774 try {775 await page.waitForFunction(776 (nmspc: string) => !!window[nmspc],777 { timeout: this.input.pageLoadTimeoutSecs * 1000 },778 namespace,779 );780 } catch (err) {781 const casted = err as Error;782 if (casted.stack!.startsWith('TimeoutError')) {783 throw new Error(784 'Injection of environment into the browser context timed out. ' +785 'If this persists even after retries, try increasing the Page load timeout input setting.',786 );787 } else {788 throw err;789 }790 }791 }792
793 private async _waitForLoadEventWhenXml(794 page: Page,795 response?: HTTPResponse,796 ) {797 // Response can sometimes be null.798 if (!response) return;799
800 const cTypeHeader = response.headers()['content-type'];801 try {802 const { type } = contentType.parse(cTypeHeader);803 if (!/^(text|application)\/xml$|\+xml$/.test(type)) return;804 } catch (err) {805 // Invalid type is not XML.806 return;807 }808
809 try {810 const timeout = this.input.pageLoadTimeoutSecs * 1000;811 await page.waitForFunction(812 () => document.readyState === 'complete',813 { timeout },814 );815 } catch (err) {816 const casted = err as Error;817 if (casted.stack!.startsWith('TimeoutError')) {818 throw new Error(819 "Parsing of XML in the page timed out. If you're expecting a large XML file, " +820 ' such as a site map, try increasing the Page load timeout input setting.',821 );822 } else {823 throw err;824 }825 }826 }827
828 private async _injectBrowserHandles(page: Page, pageContext: PageContext) {829 const saveSnapshotP = browserTools.createBrowserHandle(page, async () =>830 browserTools.saveSnapshot({ page }),831 );832 const skipLinksP = browserTools.createBrowserHandle(page, () => {833 pageContext.skipLinks = true;834 });835 const globalStoreP = browserTools.createBrowserHandlesForObject(836 page,837 this.globalStore,838 [839 'size',840 'clear',841 'delete',842 'entries',843 'get',844 'has',845 'keys',846 'set',847 'values',848 ],849 ['size'],850 );851 const logP = browserTools.createBrowserHandlesForObject(page, log, [852 'LEVELS',853 'setLevel',854 'getLevel',855 'debug',856 'info',857 'warning',858 'error',859 'exception',860 ]);861 const requestQueueP = this.requestQueue862 ? browserTools.createBrowserHandlesForObject(863 page,864 this.requestQueue,865 ['addRequest'],866 )867 : null;868 const keyValueStoreP = this.keyValueStore869 ? browserTools.createBrowserHandlesForObject(870 page,871 this.keyValueStore,872 ['getValue', 'setValue'],873 )874 : null;875
876 const [877 saveSnapshot,878 skipLinks,879 globalStore,880 logHandle,881 requestQueue,882 keyValueStore,883 ] = await Promise.all([884 saveSnapshotP,885 skipLinksP,886 globalStoreP,887 logP,888 requestQueueP,889 keyValueStoreP,890 ]);891
892 return {893 saveSnapshot,894 skipLinks,895 globalStore,896 log: logHandle,897 keyValueStore,898 requestQueue,899 };900 }901}902
903function logDevRunWarning() {904 log.warning(`905*****************************************************************906* Web Scraper is running in DEVELOPMENT MODE! *907* Concurrency is limited, sessionPool is not available, *908* timeouts are increased and debugger is enabled. *909* If you want full control and performance switch *910* Run type to PRODUCTION! *911*****************************************************************912`);913}{ "actorSpecification": 1, "name": "web-scraper", "version": "0.1", "buildTag": "latest", "storages": { "dataset": { "actorSpecification": 1, "fields": {}, "views": {} } }}