Web Scraper Experimental Debug

1const { runActor } = require('@apify/scraper-tools');
2const CrawlerSetup = require('./src/crawler_setup');
3
4runActor(CrawlerSetup);

1/* istanbul ignore next */
2/**
3 * Command to be evaluated for Browser side code injection.
4 * @param apifyNamespace
5 */
6module.exports = (apifyNamespace) => {
7    (function (global, namespace) {
8        if (typeof window[namespace] !== 'object') window[namespace] = {};
9        /**
10         * Takes a configuration object with function references
11         * as an input and creates a dummy class that proxies
12         * function invocations from Browser context to Node context.
13         *
14         * The configuration is to be provided by the
15         * tools.createBrowserHandlesForObject function.
16         */
17        class NodeProxy {
18            constructor(config) {
19                if (!config || typeof config !== 'object') {
20                    throw new Error('NodeProxy: Parameter config of type Object must be provided.');
21                }
22
23                Object.entries(config)
24                    .forEach(([key, { value, type }]) => {
25                        if (type === 'METHOD') {
26                            this[key] = (...args) => global[value](...args);
27                        } else if (type === 'GETTER') {
28                            Object.defineProperty(this, key, {
29                                get: () => global[value](),
30                            });
31                        } else if (type === 'VALUE') {
32                            this[key] = value;
33                        } else {
34                            throw new Error(`Unsupported function type: ${type} for function: ${key}.`);
35                        }
36                    });
37            }
38        }
39
40        /**
41         * Exposed factory.
42         * @param config
43         * @return {NodeProxy}
44         */
45        global[namespace].createNodeProxy = config => new NodeProxy(config);
46
47        const setup = Symbol('crawler-setup');
48        const internalState = Symbol('request-internal-state');
49
50        /**
51         * Context represents everything that is available to the user
52         * via Page Function. A class is used instead of a simple object
53         * to avoid having to create new instances of functions with each
54         * request.
55         *
56         * Some properties need to be accessible to the Context,
57         * but should not be exposed to the user thus they are hidden
58         * using a Symbol to prevent the user from easily accessing
59         * and manipulating them.
60         *
61         * @param {Object} options
62         * @param {Object} options.crawlerSetup
63         * @param {Object} options.browserHandles
64         * @param {Object} options.pageFunctionArguments
65         */
66        class Context {
67            /* eslint-disable class-methods-use-this */
68            constructor(options) {
69                const {
70                    crawlerSetup,
71                    browserHandles,
72                    pageFunctionArguments,
73                } = options;
74
75                const createProxy = global[namespace].createNodeProxy;
76
77                // Private
78                this[setup] = crawlerSetup;
79                this[internalState] = {
80                    browserHandles,
81                    requestQueue: browserHandles.requestQueue ? createProxy(browserHandles.requestQueue) : null,
82                    apify: createProxy(browserHandles.apify),
83                };
84
85                // Copies of Node objects
86                this.input = JSON.parse(crawlerSetup.rawInput);
87                this.env = Object.assign({}, crawlerSetup.env);
88                this.customData = crawlerSetup.customData;
89                this.response = pageFunctionArguments.response;
90                this.request = pageFunctionArguments.request;
91                // Functions are not converted so we need to add them this way
92                // to not be enumerable and thus not polluting the object.
93                Reflect.defineProperty(this.request, 'pushErrorMessage', {
94                    value(errorOrMessage) {
95                        // It's a simplified fake of the original function.
96                        const msg = (errorOrMessage && errorOrMessage.message) || `${errorOrMessage}`;
97                        this.errorMessages.push(msg);
98                    },
99                    enumerable: false,
100                });
101
102                // Proxied Node objects
103                this.globalStore = createProxy(browserHandles.globalStore);
104                this.log = createProxy(browserHandles.log);
105
106                // Browser side libraries
107                if (this[setup].injectJQuery) this.jQuery = global.jQuery.noConflict(true);
108                if (this[setup].injectUnderscore) this.underscoreJs = global._.noConflict();
109
110                // Bind this to allow destructuring off context in pageFunction.
111                this.getValue = this.getValue.bind(this);
112                this.setValue = this.setValue.bind(this);
113                this.saveSnapshot = this.saveSnapshot.bind(this);
114                this.skipLinks = this.skipLinks.bind(this);
115                this.enqueueRequest = this.enqueueRequest.bind(this);
116                this.waitFor = this.waitFor.bind(this);
117            }
118
119            async getValue(...args) {
120                return this[internalState].apify.getValue(...args);
121            }
122
123            async setValue(...args) {
124                return this[internalState].apify.setValue(...args);
125            }
126
127            async saveSnapshot() {
128                const handle = this[internalState].browserHandles.saveSnapshot;
129                return global[handle]();
130            }
131
132            async skipLinks() {
133                const handle = this[internalState].browserHandles.skipLinks;
134                return global[handle]();
135            }
136
137            async enqueueRequest(requestOpts = {}, options = {}) {
138                if (!this[setup].useRequestQueue) {
139                    throw new Error('Input parameter "useRequestQueue" must be set to true to be able to enqueue new requests.');
140                }
141
142                const defaultRequestOpts = {
143                    useExtendedUniqueKey: true,
144                    keepUrlFragment: this.input.keepUrlFragments,
145                };
146
147                const newRequest = { ...defaultRequestOpts, ...requestOpts };
148
149                const metaKey = this[setup].META_KEY;
150                const defaultUserData = {
151                    [metaKey]: {
152                        parentRequestId: this.request.id || this.request.uniqueKey,
153                        depth: this.request.userData[metaKey].depth + 1,
154                    },
155                };
156
157                newRequest.userData = { ...defaultUserData, ...requestOpts.userData };
158
159                return this[internalState].requestQueue.addRequest(newRequest, options);
160            }
161
162            async waitFor(selectorOrNumberOrFunction, options = {}) {
163                if (!options || typeof options !== 'object') throw new Error('Parameter options must be an Object');
164                const type = typeof selectorOrNumberOrFunction;
165                if (type === 'string') return this._waitForSelector(selectorOrNumberOrFunction, options);
166                if (type === 'number') return this._waitForMillis(selectorOrNumberOrFunction);
167                if (type === 'function') return this._waitForFunction(selectorOrNumberOrFunction, options);
168                throw new Error('Parameter selectorOrNumberOrFunction must be one of the said types.');
169            }
170
171            async _waitForSelector(selector, options = {}) {
172                try {
173                    await this._poll(() => {
174                        return !!global.document.querySelector(selector);
175                    }, options);
176                } catch (err) {
177                    if (/timeout of \d+ms exceeded/.test(err.message)) {
178                        throw new Error(`Timeout Error: waiting for selector failed: ${err.message}`);
179                    }
180                    throw err;
181                }
182            }
183
184            async _waitForMillis(millis) {
185                return new Promise(res => setTimeout(res, millis));
186            }
187
188            async _waitForFunction(predicate, options = {}) {
189                try {
190                    await this._poll(predicate, options);
191                } catch (err) {
192                    if (/timeout of \d+ms exceeded/.test(err.message)) {
193                        throw new Error(`Timeout Error: waiting for function failed: ${err.message}`);
194                    }
195                    throw err;
196                }
197            }
198
199            async _poll(predicate, options = {}) {
200                const {
201                    pollingIntervalMillis = 50,
202                    timeoutMillis = 20000,
203                } = options;
204                return new Promise((resolve, reject) => {
205                    const handler = () => {
206                        return predicate() ? resolve() : setTimeout(handler);
207                    };
208                    const pollTimeout = setTimeout(handler, pollingIntervalMillis);
209                    setTimeout(() => {
210                        clearTimeout(pollTimeout);
211                        return reject(new Error(`timeout of ${timeoutMillis}ms exceeded.`));
212                    }, timeoutMillis);
213                });
214            }
215        }
216
217        /**
218         * Exposed factory.
219         * @param {Object} options
220         * @returns {Context}
221         */
222        global[namespace].createContext = (options) => {
223            return new Context(options);
224        };
225    }(window, apifyNamespace));
226};

1const Apify = require('apify');
2const _ = require('underscore');
3const httpProxy = require('http-proxy');
4const url = require('url');
5const contentType = require('content-type');
6const {
7    tools,
8    browserTools,
9    constants: { META_KEY, DEFAULT_VIEWPORT, DEVTOOLS_TIMEOUT_SECS },
10} = require('@apify/scraper-tools');
11const rp = require('request-promise');
12
13const GlobalStore = require('./global_store');
14const createBundle = require('./bundle.browser');
15const SCHEMA = require('../INPUT_SCHEMA');
16
17const CHROME_DEBUGGER_PORT = 4322;
18
19const { utils: { log, puppeteer } } = Apify;
20
21/**
22 * Replicates the INPUT_SCHEMA with JavaScript types for quick reference
23 * and IDE type check integration.
24 *
25 * @typedef {Object} Input
26 * @property {Object[]} startUrls
27 * @property {boolean} useRequestQueue
28 * @property {Object[]} pseudoUrls
29 * @property {string} linkSelector
30 * @property {boolean} keepUrlFragments
31 * @property {string} pageFunction
32 * @property {Object} proxyConfiguration
33 * @property {boolean} debugLog
34 * @property {boolean} browserLog
35 * @property {boolean} injectJQuery
36 * @property {boolean} injectUnderscore
37 * @property {boolean} downloadMedia
38 * @property {boolean} downloadCss
39 * @property {number} maxRequestRetries
40 * @property {number} maxPagesPerCrawl
41 * @property {number} maxResultsPerCrawl
42 * @property {number} maxCrawlingDepth
43 * @property {number} maxConcurrency
44 * @property {number} pageLoadTimeoutSecs
45 * @property {number} pageFunctionTimeoutSecs
46 * @property {Object} customData
47 * @property {Array} initialCookies
48 * @property {Array} waitUntil
49 * @property {boolean} useChrome
50 * @property {boolean} useStealth
51 * @property {boolean} ignoreCorsAndCsp
52 * @property {boolean} ignoreSslErrors
53 */
54
55/**
56 * Holds all the information necessary for constructing a crawler
57 * instance and creating a context for a pageFunction invocation.
58 */
59class CrawlerSetup {
60    /* eslint-disable class-methods-use-this */
61    constructor(input) {
62        this.name = 'Web Scraper';
63        // Set log level early to prevent missed messages.
64        if (input.debugLog) log.setLevel(log.LEVELS.DEBUG);
65
66        // Keep this as string to be immutable.
67        this.rawInput = JSON.stringify(input);
68
69        // Attempt to load page function from disk if not present on input.
70        tools.maybeLoadPageFunctionFromDisk(input, __dirname);
71
72        // Validate INPUT if not running on Apify Cloud Platform.
73        if (!Apify.isAtHome()) tools.checkInputOrThrow(input, SCHEMA);
74
75        /**
76         * @type {Input}
77         */
78        this.input = input;
79        this.env = Apify.getEnv();
80
81        // Validations
82        if (this.input.pseudoUrls.length && !this.input.useRequestQueue) {
83            throw new Error('Cannot enqueue links using Pseudo URLs without using a Request Queue. '
84                + 'Either select the "Use Request Queue" option to enable Request Queue or '
85                + 'remove your Pseudo URLs.');
86        }
87        this.input.pseudoUrls.forEach((purl) => {
88            if (!tools.isPlainObject(purl)) throw new Error('The pseudoUrls Array must only contain Objects.');
89            if (purl.userData && !tools.isPlainObject(purl.userData)) throw new Error('The userData property of a pseudoUrl must be an Object.');
90        });
91        this.input.initialCookies.forEach((cookie) => {
92            if (!tools.isPlainObject(cookie)) throw new Error('The initialCookies Array must only contain Objects.');
93        });
94        this.input.waitUntil.forEach((event) => {
95            if (!/^(domcontentloaded|load|networkidle2|networkidle0)$/.test(event)) {
96                throw new Error('Navigation wait until events must be valid. See tooltip.');
97            }
98        });
99        tools.evalFunctionOrThrow(this.input.pageFunction);
100
101        // Used to store page specific data.
102        this.pageContexts = new WeakMap();
103
104        // Used to store data that persist navigations
105        this.globalStore = new GlobalStore();
106
107        // Excluded resources
108        this.blockedUrlPatterns = [];
109        if (!this.input.downloadMedia) {
110            this.blockedUrlPatterns = [...this.blockedUrlPatterns,
111                '.jpg', '.jpeg', '.png', '.svg', '.gif', '.webp', '.webm', '.ico', '.woff', '.eot',
112            ];
113        }
114        if (!this.input.downloadCss) this.blockedUrlPatterns.push('.css');
115
116        // Start Chromium with Debugger any time the page function includes the keyword.
117        this.devtools = this.input.pageFunction.includes('debugger;');
118
119        // Initialize async operations.
120        this.crawler = null;
121        this.requestList = null;
122        this.requestQueue = null;
123        this.dataset = null;
124        this.keyValueStore = null;
125        this.proxy = null;
126        this.initPromise = this._initializeAsync();
127    }
128
129    async _initializeAsync() {
130        // RequestList
131        const startUrls = this.input.startUrls.map((req) => {
132            req.useExtendedUniqueKey = true;
133            req.keepUrlFragment = this.input.keepUrlFragments;
134            return req;
135        });
136        this.requestList = await Apify.openRequestList('WEB_SCRAPER', startUrls);
137
138        // RequestQueue if selected
139        if (this.input.useRequestQueue) this.requestQueue = await Apify.openRequestQueue();
140
141        // Dataset
142        this.dataset = await Apify.openDataset();
143        const { itemsCount } = await this.dataset.getInfo();
144        this.pagesOutputted = itemsCount || 0;
145
146        // KeyValueStore
147        this.keyValueStore = await Apify.openKeyValueStore();
148    }
149
150    /**
151     * Resolves to a `PuppeteerCrawler` instance.
152     * constructor.
153     * @returns {Promise<PuppeteerCrawler>}
154     */
155    async createCrawler() {
156        await this.initPromise;
157
158        const args = [];
159        if (this.input.ignoreCorsAndCsp) args.push('--disable-web-security');
160
161        const options = {
162            handlePageFunction: this._handlePageFunction.bind(this),
163            requestList: this.requestList,
164            requestQueue: this.requestQueue,
165            handlePageTimeoutSecs: this.devtools ? DEVTOOLS_TIMEOUT_SECS : this.input.pageFunctionTimeoutSecs,
166            gotoFunction: this._gotoFunction.bind(this),
167            handleFailedRequestFunction: this._handleFailedRequestFunction.bind(this),
168            maxConcurrency: this.input.maxConcurrency,
169            maxRequestRetries: this.input.maxRequestRetries,
170            maxRequestsPerCrawl: this.input.maxPagesPerCrawl,
171            proxyUrls: this.input.proxyConfiguration.proxyUrls,
172            // launchPuppeteerFunction: use default,
173            puppeteerPoolOptions: {
174                useLiveView: true,
175                recycleDiskCache: true,
176            },
177            launchPuppeteerOptions: {
178                ...(_.omit(this.input.proxyConfiguration, 'proxyUrls')),
179                ignoreHTTPSErrors: this.input.ignoreSslErrors,
180                defaultViewport: DEFAULT_VIEWPORT,
181                devtools: this.devtools,
182                headless: false,
183                useChrome: this.input.useChrome,
184                stealth: this.input.useStealth,
185                args,
186            },
187        };
188
189        if (this.input.chromeDebugger) {
190            log.info(`Starting scraper with Chrome debugger attached:
191            - Scraper will run with concurrency 1.
192            - The whole run will be performed with a single browser (= 1 proxy IP).
193            - You can place "debugger;" to you page function to set up a break point.`);
194
195            options.puppeteerPoolOptions.useLiveView = false;
196            options.puppeteerPoolOptions.retireInstanceAfterRequestCount = 1000000;
197            options.maxConcurrency = 1;
198            options.launchPuppeteerFunction = async (launchPuppeteerOptions) => {
199                launchPuppeteerOptions.args = [`--remote-debugging-port=${CHROME_DEBUGGER_PORT}`, '--user-data-dir=remote-profile'];
200                launchPuppeteerOptions.defaultViewport = { width: 720, height: 1024 };
201
202                const browser = await Apify.launchPuppeteer(launchPuppeteerOptions);
203                const containerHost = url.parse(process.env.APIFY_CONTAINER_URL).host;
204
205                // Start HTTP proxy from live view port to Chrome debugger port.
206                if (this.proxy) proxy.close();
207                this.proxy = httpProxy.createServer({
208                    target: {
209                        host: 'localhost',
210                        port: CHROME_DEBUGGER_PORT,
211                    },
212                    ws: true,
213                }).listen(4321);
214
215                // Fetches debugger URL of page we are scraping.
216                const debuggerUrlPromise = (async () => {
217                    while (true) {
218                        const pageList = await rp({ url: `http://localhost:${CHROME_DEBUGGER_PORT}/json/list`, json: true });
219
220                        // All the internal debugger pages start with something like "devtools://"...
221                        const debuggerPage = pageList.filter(page => page.url.startsWith('http')).pop();
222                        if (debuggerPage) return debuggerPage;
223
224                        await Apify.utils.sleep(1000);
225                    }
226                })();
227
228                // Intercept requests to HTTP proxy.
229                this.proxy.on('proxyReq', async (proxyReq, req, res) => {
230                    // We need Chrome to think that it's on localhost otherwise it throws an error...
231                    proxyReq.setHeader('Host', 'localhost');
232
233                    // URL of debugger of a page we are scraping.
234                    let targetUrl = req.url;
235                    if (targetUrl === '/') {
236                        proxyReq.socket.pause(); // We need to pause it to be able to perform async operations.
237                        targetUrl = (await debuggerUrlPromise).devtoolsFrontendUrl; // This one includes ws=localhost so redirect below will be performed
238                        targetUrl = targetUrl.replace(`:${CHROME_DEBUGGER_PORT}`, '');
239                        proxyReq.socket.resume();
240                    }
241
242                    // ...but then Chrome things it's running on localhost so we need to change websocket
243                    // host in URL to be correct.
244                    if (targetUrl.includes('ws=localhost')) {
245                        const newUrl = targetUrl.replace('ws=localhost', `wss=${containerHost}`);
246                        res.writeHead(301, { Location: newUrl });
247                        res.end();
248                    }
249                });
250
251                return browser;
252            };
253        }
254
255        this.crawler = new Apify.PuppeteerCrawler(options);
256
257        if (this.input.chromeDebugger) {
258            // 🤷 this.puppeteerPool is set ip in crawler.run() which is called somewhere??
259            const interval = setInterval(() => {
260                if (this.crawler.puppeteerPool) {
261                    this.crawler.puppeteerPool.reusePages = true;
262                    clearInterval(interval);
263                }
264            }, 10);
265        }
266
267        return this.crawler;
268    }
269
270    async _gotoFunction({ request, page }) {
271        const start = process.hrtime();
272
273        // Create a new page context with a new random key for Apify namespace.
274        const pageContext = {
275            apifyNamespace: await tools.createRandomHash(),
276            skipLinks: false,
277        };
278        this.pageContexts.set(page, pageContext);
279
280        // Attach a console listener to get all logs as soon as possible.
281        if (this.input.browserLog) browserTools.dumpConsole(page);
282
283        // Prevent download of stylesheets and media, unless selected otherwise
284        if (this.blockedUrlPatterns.length) {
285            await puppeteer.blockRequests(page, {
286                urlPatterns: this.blockedUrlPatterns,
287            });
288        }
289
290        // Add initial cookies, if any.
291        if (this.input.initialCookies.length) await page.setCookie(...this.input.initialCookies);
292
293        // Disable content security policy.
294        if (this.input.ignoreCorsAndCsp) await page.setBypassCSP(true);
295
296        tools.logPerformance(request, 'gotoFunction INIT', start);
297        const handleStart = process.hrtime();
298        pageContext.browserHandles = await this._injectBrowserHandles(page, pageContext);
299        tools.logPerformance(request, 'gotoFunction INJECTION HANDLES', handleStart);
300
301        const evalStart = process.hrtime();
302        await Promise.all([
303            page.evaluateOnNewDocument(createBundle, pageContext.apifyNamespace),
304            page.evaluateOnNewDocument(browserTools.wrapPageFunction(this.input.pageFunction, pageContext.apifyNamespace)),
305        ]);
306        tools.logPerformance(request, 'gotoFunction INJECTION EVAL', evalStart);
307
308
309        // Invoke navigation.
310        const navStart = process.hrtime();
311        const response = await puppeteer.gotoExtended(page, request, {
312            timeout: (this.devtools ? DEVTOOLS_TIMEOUT_SECS : this.input.pageLoadTimeoutSecs) * 1000,
313            waitUntil: this.input.waitUntil,
314        });
315        await this._waitForLoadEventWhenXml(page, response);
316        tools.logPerformance(request, 'gotoFunction NAVIGATION', navStart);
317
318        const delayStart = process.hrtime();
319        await this._assertNamespace(page, pageContext.apifyNamespace);
320
321        // Inject selected libraries
322        if (this.input.injectJQuery) await puppeteer.injectJQuery(page);
323        if (this.input.injectUnderscore) await puppeteer.injectUnderscore(page);
324
325        tools.logPerformance(request, 'gotoFunction INJECTION DELAY', delayStart);
326        tools.logPerformance(request, 'gotoFunction EXECUTION', start);
327        return response;
328    }
329
330    _handleFailedRequestFunction({ request }) {
331        const lastError = request.errorMessages[request.errorMessages.length - 1];
332        const errorMessage = lastError ? lastError.split('\n')[0] : 'no error';
333        log.error(`Request ${request.url} failed and will not be retried anymore. Marking as failed.\nLast Error Message: ${errorMessage}`);
334        return this._handleResult(request, {}, null, true);
335    }
336
337    /**
338     * First of all, it initializes the state that is exposed to the user via
339     * `pageFunction` context.
340     *
341     * Then it invokes the user provided `pageFunction` with the prescribed context
342     * and saves its return value.
343     *
344     * Finally, it makes decisions based on the current state and post-processes
345     * the data returned from the `pageFunction`.
346     * @param {Object} environment
347     * @returns {Function}
348     */
349    async _handlePageFunction({ request, response, page, autoscaledPool }) {
350        const start = process.hrtime();
351
352        const pageContext = this.pageContexts.get(page);
353
354        /**
355         * PRE-PROCESSING
356         */
357        // Make sure that an object containing internal metadata
358        // is present on every request.
359        tools.ensureMetaData(request);
360
361        // Abort the crawler if the maximum number of results was reached.
362        const aborted = await this._handleMaxResultsPerCrawl(autoscaledPool);
363        if (aborted) return;
364
365        // Setup Context and pass the configuration down to Browser.
366        const contextOptions = {
367            crawlerSetup: Object.assign(
368                _.pick(this, ['rawInput', 'env']),
369                _.pick(this.input, ['customData', 'useRequestQueue', 'injectJQuery', 'injectUnderscore']),
370                { META_KEY },
371            ),
372            browserHandles: pageContext.browserHandles,
373            pageFunctionArguments: {
374                request,
375                response: {
376                    status: response && response.status(),
377                    headers: response && response.headers(),
378                },
379            },
380        };
381
382        /**
383         * USER FUNCTION EXECUTION
384         */
385        tools.logPerformance(request, 'handlePageFunction PREPROCESSING', start);
386        const startUserFn = process.hrtime();
387
388        const namespace = pageContext.apifyNamespace;
389        const output = await page.evaluate(async (ctxOpts, namespc) => {
390            /* eslint-disable no-shadow */
391            const context = window[namespc].createContext(ctxOpts);
392            const output = {};
393            try {
394                output.pageFunctionResult = await window[namespc].pageFunction(context);
395            } catch (err) {
396                output.pageFunctionError = Object.getOwnPropertyNames(err)
397                    .reduce((memo, name) => {
398                        memo[name] = err[name];
399                        return memo;
400                    }, {});
401            }
402            // This needs to be added after pageFunction has run.
403            output.requestFromBrowser = context.request;
404
405            /**
406             * Since Dates cannot travel back to Node and Puppeteer does not use .toJSON
407             * to stringify, they come back as empty objects. We could use JSON.stringify
408             * ourselves, but that exposes us to overridden .toJSON in the target websites.
409             * This hack is not ideal, but it avoids both problems.
410             */
411            function replaceAllDatesInObjectWithISOStrings(obj) {
412                for (const [key, value] of Object.entries(obj)) {
413                    if (value instanceof Date && typeof value.toISOString === 'function') {
414                        obj[key] = value.toISOString();
415                    } else if (value && typeof value === 'object') {
416                        replaceAllDatesInObjectWithISOStrings(value);
417                    }
418                }
419                return obj;
420            }
421
422            return replaceAllDatesInObjectWithISOStrings(output);
423        }, contextOptions, namespace);
424
425        tools.logPerformance(request, 'handlePageFunction USER FUNCTION', startUserFn);
426        const finishUserFn = process.hrtime();
427
428        /**
429         * POST-PROCESSING
430         */
431        const { pageFunctionResult, requestFromBrowser, pageFunctionError } = output;
432        // Merge requestFromBrowser into request to preserve modifications that
433        // may have been made in browser context.
434        Object.assign(request, requestFromBrowser);
435
436        // Throw error from pageFunction, if any.
437        if (pageFunctionError) throw tools.createError(pageFunctionError);
438
439        // Enqueue more links if a link selector is available,
440        // unless the user invoked the `skipLinks()` context function
441        // or maxCrawlingDepth would be exceeded.
442        if (!pageContext.skipLinks) {
443            await this._handleLinks(page, request);
444        }
445
446        // Save the `pageFunction`s result (or just metadata) to the default dataset.
447        await this._handleResult(request, response, pageFunctionResult);
448
449        tools.logPerformance(request, 'handlePageFunction POSTPROCESSING', finishUserFn);
450        tools.logPerformance(request, 'handlePageFunction EXECUTION', start);
451    }
452
453    async _handleMaxResultsPerCrawl(autoscaledPool) {
454        if (!this.input.maxResultsPerCrawl || this.pagesOutputted < this.input.maxResultsPerCrawl) return false;
455        log.info(`User set limit of ${this.input.maxResultsPerCrawl} results was reached. Finishing the crawl.`);
456        await autoscaledPool.abort();
457        return true;
458    }
459
460    async _handleLinks(page, request) {
461        if (!(this.input.linkSelector && this.requestQueue)) return;
462        const start = process.hrtime();
463
464        const currentDepth = request.userData[META_KEY].depth;
465        const hasReachedMaxDepth = this.input.maxCrawlingDepth && currentDepth >= this.input.maxCrawlingDepth;
466        if (hasReachedMaxDepth) {
467            log.debug(`Request ${request.url} reached the maximum crawling depth of ${currentDepth}.`);
468            return;
469        }
470
471        const enqueueOptions = {
472            page,
473            selector: this.input.linkSelector,
474            pseudoUrls: this.input.pseudoUrls,
475            requestQueue: this.requestQueue,
476            transformRequestFunction: (requestOptions) => {
477                requestOptions.userData = {
478                    [META_KEY]: {
479                        parentRequestId: request.id || request.uniqueKey,
480                        depth: currentDepth + 1,
481                    },
482                };
483                requestOptions.useExtendedUniqueKey = true;
484                requestOptions.keepUrlFragment = this.input.keepUrlFragments;
485                return requestOptions;
486            },
487        };
488
489        await Apify.utils.enqueueLinks(enqueueOptions);
490
491        tools.logPerformance(request, 'handleLinks EXECUTION', start);
492    }
493
494    async _handleResult(request, response, pageFunctionResult, isError) {
495        const start = process.hrtime();
496        const payload = tools.createDatasetPayload(request, response, pageFunctionResult, isError);
497        await Apify.pushData(payload);
498        this.pagesOutputted++;
499        tools.logPerformance(request, 'handleResult EXECUTION', start);
500    }
501
502    async _assertNamespace(page, namespace) {
503        try {
504            await page.waitFor(nmspc => !!window[nmspc], { timeout: this.input.pageLoadTimeoutSecs * 1000 }, namespace);
505        } catch (err) {
506            if (err.stack.startsWith('TimeoutError')) {
507                throw new Error('Injection of environment into the browser context timed out. '
508                    + 'If this persists even after retries, try increasing the Page load timeout input setting.');
509            } else {
510                throw err;
511            }
512        }
513    }
514
515    async _waitForLoadEventWhenXml(page, response) {
516        // Response can sometimes be null.
517        if (!response) return;
518
519        const cTypeHeader = response.headers()['content-type'];
520        try {
521            const { type } = contentType.parse(cTypeHeader);
522            if (!/^(text|application)\/xml$|\+xml$/.test(type)) return;
523        } catch (err) {
524            // Invalid type is not XML.
525            return;
526        }
527
528        try {
529            const timeout = this.input.pageLoadTimeoutSecs * 1000;
530            await page.waitFor(() => document.readyState === 'complete', { timeout });
531        } catch (err) {
532            if (err.stack.startsWith('TimeoutError')) {
533                throw new Error('Parsing of XML in the page timed out. If you\'re expecting a large XML file, '
534                    + ' such as a site map, try increasing the Page load timeout input setting.');
535            } else {
536                throw err;
537            }
538        }
539    }
540
541    async _injectBrowserHandles(page, pageContext) {
542        const saveSnapshotP = browserTools.createBrowserHandle(page, () => browserTools.saveSnapshot({ page }));
543        const skipLinksP = browserTools.createBrowserHandle(page, () => { pageContext.skipLinks = true; });
544        const globalStoreP = browserTools.createBrowserHandlesForObject(
545            page,
546            this.globalStore,
547            ['size', 'clear', 'delete', 'entries', 'get', 'has', 'keys', 'set', 'values'],
548        );
549        const logP = browserTools.createBrowserHandlesForObject(
550            page,
551            log,
552            ['LEVELS', 'setLevel', 'getLevel', 'debug', 'info', 'warning', 'error', 'exception'],
553        );
554        const apifyP = browserTools.createBrowserHandlesForObject(page, Apify, ['getValue', 'setValue']);
555        const requestQueueP = this.requestQueue
556            ? browserTools.createBrowserHandlesForObject(page, this.requestQueue, ['addRequest'])
557            : null;
558
559        const [
560            saveSnapshot,
561            skipLinks,
562            globalStore,
563            logHandle,
564            apify,
565            requestQueue,
566        ] = await Promise.all([saveSnapshotP, skipLinksP, globalStoreP, logP, apifyP, requestQueueP]);
567
568        const handles = {
569            saveSnapshot,
570            skipLinks,
571            globalStore,
572            log: logHandle,
573            apify,
574        };
575        if (requestQueue) handles.requestQueue = requestQueue;
576        return handles;
577    }
578}
579
580module.exports = CrawlerSetup;