FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
 &amp;&amp; npm install --only=prod --no-optional

COPY . ./

Dockerfile

{
 "title": "Český ráj (ceskyraj.com) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "type": "object",
 "schemaVersion": 1,
 "properties": {
 "mode": {
 "title": "Mode",
 "description": "",
 "type": "string",
 "editor": "select",
 "default": "TEST",
 "prefill": "TEST",
 "enumTitles": [
 "TEST",
 "FULL",
 "SINGLE"
 ],
 "enum": [
 "TEST",
 "FULL",
 "SINGLE"
 ]
 },
 "country": {
 "title": "Country",
 "description": "",
 "type": "string",
 "editor": "select",
 "default": "CZ",
 "prefill": "CZ",
 "enumTitles": [
 "CZ",
 "SK",
 "UK",
 "DE",
 "AT",
 "HU"
 ],
 "enum": [
 "CZ",
 "SK",
 "UK",
 "DE",
 "AT",
 "HU"
 ]
 },
 "debug": {
 "title": "Debug",
 "description": "Debug mode prints more logs, disables concurrency and other optimizations.",
 "type": "boolean",
 "editor": "checkbox",
 "default": false
 }
 },
 "required": [
 "mode",
 "country"
 ]
}

INPUT_SCHEMA.json

# Český ráj (ceskyraj.com) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

## Output example

* **itemId** `string` e.g. *p1390167*
* **itemName** `string` e.g. *Pedály Crankbrothers Stamp 7 orange*
* **itemUrl** `string` e.g. *https://www.ceskyraj.com/pedaly-crankbrothers-stamp-7-orange-p1141669/*
* **img** `string`
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *3896*
* **originalPrice** `number` e.g. *3999*
* **currency** `string` e.g. *CZK*

README.md

{
 "name": "cesky-raj-ceskyraj-com-scraper",
 "version": "0.1",
 "buildTag": "latest",
 "env": null,
 "template": "project_cheerio_crawler"
}

apify.json

main.js

{
 "name": "cesky-raj-ceskyraj-com-scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "type": "module",
 "scripts": {
 "start": "node ./main.js",
 "push-to-apify-platform": "npx apify push"
 },
 "dependencies": {
 "apify": "*",
 "@thi.ng/atom": "*",
 "cheerio": "*",
 "got-scraping": "*"
 },
 "apify": {
 "title": "Český ráj (ceskyraj.com) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "isPublic": true,
 "isDeprecated": false,
 "issuesEnabled": true,
 "isAnonymouslyRunnable": true,
 "notice": "",
 "pictureUrl": "",
 "seoTitle": "",
 "seoDescription": "",
 "categories": [
 "ECOMMERCE"
 ]
 }
}

package.json

{
 "actorSpecification": 1,
 "name": "cesky-raj-ceskyraj-com-scraper",
 "title": "Český ráj (ceskyraj.com) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "version": "0.1.0",
 "storages": {
 "dataset": {
 "actorSpecification": 1,
 "title": "Český ráj (ceskyraj.com) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "views": {
 "overview": {
 "title": "Overview",
 "description": "Overview of the most important fields",
 "transformation": {
 "fields": [
 "itemId",
 "itemName",
 "itemUrl",
 "img",
 "inStock",
 "currentPrice",
 "originalPrice",
 "currency"
 ]
 },
 "display": {
 "component": "table",
 "columns": [
 {
 "label": "Item ID",
 "field": "itemUrl",
 "format": "link",
 "textField": "itemId"
 },
 {
 "label": "Item Name",
 "field": "itemName",
 "format": "text"
 },
 {
 "label": "Img",
 "field": "img",
 "format": "image"
 },
 {
 "label": "In Stock",
 "field": "inStock",
 "format": "boolean"
 },
 {
 "label": "Current Price",
 "field": "currentPrice",
 "format": "number"
 },
 {
 "label": "Original Price",
 "field": "originalPrice",
 "format": "number"
 },
 {
 "label": "Currency",
 "field": "currency",
 "format": "text"
 }
 ]
 }
 }
 }
 }
 }
}

.actor/actor.json

_utils/CloudflareUnblocker.js

import { createHash } from 'crypto';

// inspired by @drobnikj
const createUniqueKeyFromUrl = (url) => {
 const hash = createHash('sha256');
 const cleanUrl = url.split('://')[1]; // Remove protocol
 hash.update(cleanUrl);
 return hash.digest('hex');
};

_utils/common.js

// inspired by hlidac-shopu
import Apify from "apify";
import { defAtom } from "@thi.ng/atom";

// TODO: make this lowes common denominator
const defaultStats = {
 urls: 0,
 items: 0,
 itemsDuplicate: 0,
 totalItems: 0,
 denied: 0,
 ok: 0
};

const inc = x => x + 1;
const dec = x => x - 1;

// TODO: stats should be in atom and updated via swap function atomically
class Stats {
 constructor(init) {
 this.stats = defAtom(init);
 this.interval = setInterval(() => this.log(), 20 * 1000);
 }

 inc(key) {
 this.stats.swapIn(key, inc);
 }

 dec(key) {
 this.stats.swapIn(key, dec);
 }

 add(key, value) {
 this.stats.swapIn(key, x => x + value);
 }

 get() {
 return this.stats.deref();
 }

 log() {
 const stats = this.stats.deref();
 Apify.utils.log.info(`stats: ${JSON.stringify(stats)}`);
 }

 /**
 * @param final {boolean} - If true, clearInterval apply
 */
 async save(final = false) {
 if (final) {
 clearInterval(this.interval);
 }
 const stats = this.stats.deref();
 await Apify.setValue("STATS", this.get());
 Apify.utils.log.info("STATS saved!");
 if (stats.ok) {
 Apify.utils.log.info(
 `Denied ratio: ${(stats.denied ?? 0 / stats.ok) * 100} %`
 );
 }
 this.log();
 }
}

/**
 *
 * @param {function} fn
 * @param {*} init
 * @returns {Promise&lt;Stats>}
 */
export async function withPersistedStats(fn, init) {
 const stats = (await Apify.getValue("STATS")) ?? init ?? defaultStats;
 const state = new Stats(fn(stats));
 const persistState = () => state.save();

 Apify.events.on("persistState", persistState);
 Apify.events.on("migrating", persistState);

 return state;
}

_utils/stats.js

FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
  && npm install --only=prod --no-optional

COPY . ./

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

Country

Debug mode prints more logs, disables concurrency and other optimizations.

Debug

Mode

Český ráj (ceskyraj.com) scraper

# Český ráj (ceskyraj.com) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

## Output example

* **itemId** `string` e.g. *p1390167*
* **itemName** `string` e.g. *Pedály Crankbrothers Stamp 7 orange*
* **itemUrl** `string` e.g. *https://www.ceskyraj.com/pedaly-crankbrothers-stamp-7-orange-p1141669/*
* **img** `string`
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *3896*
* **originalPrice** `number` e.g. *3999*
* **currency** `string` e.g. *CZK*

cesky-raj-ceskyraj-com-scraper

/**
 * TODO:
 * - consider proxies
 * - DRY price parsing
 * - different countries to input
 *
 * Beware that same product can have multiple valid urls
 * - https://www.alza.cz/iphone-13-512gb-cervena-levne-d6839524.htm
 * - https://www.alza.cz/sport/victorias-secret-st-11128877-cc-4vmq-cerna
 * - https://www.alza.cz/sport/victorias-secret-st-11156655-cc-38h2-bezova?dq=6920061
 *
 * Variants can affect price
 * - https://www.alza.cz/asus-rog-zephyrus-g14-ga401?dq=6804643 38k
 * - https://www.alza.cz/asus-rog-zephyrus-g14-ga401?dq=6771118 39k
 *
 * Pagination:
 * beware: only first next 3 pages are listed
 * -> need to use total amount of products & product per page to calculate total amount of pages
 */

import Apify from "apify";
import cheerio from "cheerio";
import { gotScraping } from "got-scraping";
import { withPersistedStats } from "./_utils/stats.js";

const { log } = Apify.utils;

var LABEL;

(function (LABEL) {
 LABEL["INDEX"] = "INDEX";
 LABEL["PRODUCTS"] = "PRODUCTS";
})(LABEL || (LABEL = {}));
var MODE;

(function (MODE) {
 MODE["TEST"] = "TEST";
 MODE["FULL"] = "FULL";
 MODE["SINGLE"] = "SINGLE";
})(MODE || (MODE = {}));
var Country;

// TODO: Maybe unify with Country enum
(function (Country) {
 Country["CZ"] = "CZ";
 Country["SK"] = "SK";
 Country["UK"] = "UK";
 Country["DE"] = "DE";
 Country["AT"] = "AT";
 Country["HU"] = "HU";
})(Country || (Country = {}));
const Countries = {
 CZ: {
 domain: "cz",
 currency: "CZK",
 },
 SK: {
 domain: "sk",
 currency: "EUR",
 },
 UK: {
 domain: "co.uk",
 currency: "GBP",
 },
 DE: {
 domain: "de",
 currency: "EUR",
 },
 AT: {
 domain: "at",
 currency: "EUR",
 },
 HU: {
 domain: "hu",
 currency: "HUF",
 },
};

async function enqueueInitialRequest(type, requestQueue, countryDef) {
 if (type === MODE.FULL) {
 await requestQueue.addRequest({
 userData: { label: LABEL.INDEX },
 url: `https://alza.${countryDef.domain}/_sitemap-categories.xml`,
 });
 } else if (type === MODE.TEST) {
 await requestQueue.addRequest({
 userData: { label: LABEL.PRODUCTS },
 url: "https://www.alza.cz/fotobrasny/18848740.htm",
 });
 await requestQueue.addRequest({
 userData: { label: LABEL.PRODUCTS },
 url: "https://www.alza.cz/objektivy-sony-fe/18879399.htm",
 });
 await requestQueue.addRequest({
 userData: { label: LABEL.PRODUCTS },
 url: "https://www.alza.cz/baterie-pro-fotoaparaty/18851400.htm",
 });
 } else if (type === MODE.SINGLE) {
 await requestQueue.addRequest({
 userData: { label: LABEL.PRODUCTS },
 url: "https://www.alza.sk/iphone-mobilne-telefony/18851638.htm", // single category, but containing multiple pages
 });
 }
}

async function handleIndex(
 { request, session = null },
 { mode, stats, requestQueue }
) {
 const response = await gotScraping({
 url: request.url,
 });
 if (![200, 404].includes(response.statusCode)) {
 session.retire();
 request.retryCount--;
 throw new Error(
 `Blocked, statusCode ${response.statusCode}, url: ${request.url}`
 );
 }
 const $ = cheerio.load(response.body, { xmlMode: true });
 const urls = $("url loc").map((i, x) => $(x).text());
 for (const url of urls) {
 await requestQueue.addRequest({
 userData: { label: LABEL.PRODUCTS },
 url,
 });
 }
}

async function handleProducts(
 { request, session = null },
 { stats, processedIds, requestQueue, countryDef }
) {
 console.log("handleProducts", request.url);

 const response = await gotScraping({
 url: request.url,
 headers: {
 Host: `www.alza.${countryDef.domain}`, // IMPORTANT to get prices in CZK, not EUR
 },
 // proxyUrl: await proxyConfiguration.newUrl(session.id),
 });

 const $ = cheerio.load(response.body);

 if (
 // no pagination info in URL -> we are on a first/initial page -> enqueue next pages
 !request.url.match(/-p\d+\.htm/)
 ) {
 // Beware: only first next 3 pages are listed, so following code is not correct
 // $('#pagerbottom a[class="pgn"]').each((i, el) => {
 // // beware: href attr does not start with `/`, as product urls do
 // const url = `https://alza.${countryDef.domain}/${($(el).attr("href"))}`
 // if (i === 0) return // skip first page, as we are already on it
 // void requestQueue.addRequest({
 // userData: { label: LABEL.PRODUCTS },
 // url,
 // })
 // })

 // Method using total amount of products and product per page to calculate total amount of pages
 const totalAmountOfProducts = parseInt($("#lblNumberItem").text());
 const productsPerPage = 24; // TODO: Maybe to constant
 const totalPages = Math.ceil(totalAmountOfProducts / productsPerPage);
 for (let i = 2; i <= totalPages; i++) {
 const url = request.url // e.g. `/iphone-mobilne-telefony/18851638.htm`
 .replace(/\.htm$/, "") // `/iphone-mobilne-telefony/18851638`
 .concat(`-p${i}.htm`); // e.g. `/iphone-mobilne-telefony/18851638-p2.htm`
 console.log("enqueue next page", url);
 void requestQueue.addRequest({
 userData: { label: LABEL.PRODUCTS },
 url,
 });
 }
 }

 $(".browsingitem").each(function () {
 const itemId = $(this).attr("data-id");
 if (!processedIds.has(itemId)) {
 stats.inc("itemsUnique");
 processedIds.add(itemId);
 const itemCode = $(this).attr("data-code"); // non-standard product property
 const img = $(this).find("a.pc.browsinglink img").data("src");
 const relativeItemUrl = $(this)
 .find("a.name.browsinglink")
 .first()
 .attr("href");
 const absoluteItemUrl = `https://alza.${countryDef.domain}${relativeItemUrl}`;
 const itemName = $(this)
 .find("a.name.browsinglink")
 .first()
 .text()
 .replace(/([\n\r])/g, "") // replace newlines and carriage returns
 .trim();

 const currentPrice = $(this)
 .find(".price .c2")
 .text()
 .replace(/,-$/, "")
 .replace(/[^\d.,]/g, "");

 const originalPrice = $(this)
 .find(".price .np2")
 .text()
 .replace(/,-$/, "")
 .replace(/[^\d.,]/g, "");

 const product = {
 itemId,
 itemName,
 itemUrl: absoluteItemUrl,
 img,
 inStock: "TODO",
 currentPrice,
 originalPrice,
 currency: countryDef.currency,
 category: "FIXME",

 // Derived
 discounted: originalPrice && originalPrice !== currentPrice,
 _discount: originalPrice
 ? // @ts-ignore
 (1 - currentPrice / originalPrice) * 100
 : undefined,
 };
 console.log(`Found product: ${itemId}`);
 Apify.pushData(product);
 } else {
 stats.inc("itemsDuplicate");
 }
 });
}

const proxyGroups = [
 "HUNGARY",
 "CZECH_LUMINATI",
 "GERMANY",
 "FRANCE",
 "ITALY",
 "SPAIN",
];

Apify.main(async () => {
 await Apify.utils.purgeLocalStorage();

 const stats = await withPersistedStats((x) => x, {
 categories: 0,
 items: 0,
 itemsUnique: 0,
 itemsDuplicate: 0,
 denied: 0,
 captchas: 0,
 });
 const processedIds = new Set();

 const input = await Apify.getInput();
 const { debug = false, country = Country.CZ, mode = MODE.TEST } = input ?? {};

 const countryDef = Countries[country];

 if (debug) Apify.utils.log.setLevel(Apify.utils.log.LEVELS.DEBUG);

 // const proxyConfiguration = await Apify.createProxyConfiguration({
 // groups: proxyGroups,
 // })

 const requestQueue = await Apify.openRequestQueue();
 await enqueueInitialRequest(mode, requestQueue, countryDef);

 const globalContext = { mode, stats, requestQueue, processedIds, countryDef };
 const crawler = new Apify.BasicCrawler({
 requestQueue,
 maxConcurrency: 1, // FIXME
 maxRequestRetries: 1,
 async handleRequestFunction(context) {
 const { request, session } = context;
 const { userData } = request;
 // @ts-ignore
 switch (userData.label) {
 case LABEL.INDEX:
 return await handleIndex(context, globalContext);
 case LABEL.PRODUCTS:
 return await handleProducts(context, globalContext);
 }
 },
 async handleFailedRequestFunction({ request }) {
 log.error(`Request ${request.url} failed multiple times`, request);
 },
 });

 await crawler.run();
 log.info("crawler finished");
});

import Apify from "apify"
import { Session } from "apify/build/session_pool/session.js"
import { BrowserPool, PlaywrightPlugin } from "browser-pool"
import playwright from "playwright"

const { utils: { log, requestAsBrowser, puppeteer } } = Apify

export default class CloudflareUnblocker {
 // unblockUrl: string
 // proxyConfiguration: any
 // browserPool: BrowserPool
 // _shouldUseCustomTLS: boolean
 // log: any
 // name: string

 constructor (options) {
 // Store options
 this.unblockUrl = options.unblockUrl
 this.proxyConfiguration = options.proxyConfiguration

 // Create browser pool
 this.browserPool = new BrowserPool({
 retireBrowserAfterPageCount: 50, // TODO: explain
 maxOpenPagesPerBrowser: 10, // TODO: explain
 browserPlugins: [
 new PlaywrightPlugin(playwright.firefox, {
 launchOptions: { headless: false },
 }),
 ],
 })

 this._shouldUseCustomTLS = false

 // Extending CrawlerExtension caused error:
 // Class extends value #<Object> is not a constructor or null
 // But all it should do is to add log method, so I'm adding it here
 this.name = this.constructor.name
 this.log = Apify.utils.log.child({ prefix: this.constructor.name })
 }

 /**
 * Main function that unblocks your session.
 */
 async unblock ({ session, request }) {
 if (this._isSessionBeingRenewed(session)) {
 request.retryCount = 0
 this._throwError("Session is being renewed")
 }

 const oldShouldUseTLS = this._shouldUseCustomTLS
 const requestOptions = this.getRequestOptions(session)
 const response = await requestAsBrowser({
 ...requestOptions,
 url: request.url,
 })

 if (this._detectChallengeFunction(response)) {
 // Browser challenge detected starting the bypassing
 try {
 this._markSessionBeingRenewed(session)
 return this._solveChallenge({
 response,
 session,
 request,
 requestOptions,
 })
 } finally {
 this._markSessionNotBeingRenewed(session)
 }
 } else if (response.statusCode === 403) {
 // Cloudflare captcha detected -> switching to slower TLS
 if (oldShouldUseTLS) {
 this.log.info("Captcha found even with the TLS hack")
 await Apify.setValue(
 `CAPTCHA-HTML-${Math.random() * 1000}`,
 response.body,
 { contentType: "text/html" },
 )
 } else {
 this.log.info(
 "Captcha found for the first time -> switching to custom TLS",
 )
 this._shouldUseCustomTLS = true
 }
 }

 this.log.info("Session OK")
 session.setCookiesFromResponse(response)
 return response
 }

 /**
 * Solves the challenge by starting the browser and saving the cookies to the session.
 * @param options
 * @param options.request
 * @param options.response
 * @param options.session
 * @param options.requestOptions
 * @return {Promise<*>}
 * @private
 */
 async _solveChallenge (options) {
 const { request, response, session } = options
 const { body, headers } = response

 const receivedCookies = headers["set-cookie"]
 log.debug(`${this.name}: received cookies: ${receivedCookies}`)
 const requestOptions = this.getRequestOptions(session)

 if (!receivedCookies) {
 await Apify.setValue(`NO-COOKIES-HTML-${Math.random() * 1000}`, body, {
 contentType: "text/html",
 })
 }

 session.setCookiesFromResponse(response)

 const cloudflareAuthReq = await this._getSolvedChallengeRequest({
 response,
 session,
 request,
 })

 const browserHeaders = cloudflareAuthReq.headers()
 const finalRequestOpts = {
 ...requestOptions,
 url: cloudflareAuthReq.url(),
 payload: cloudflareAuthReq.postData(),
 headers: {
 ...requestOptions.headers,
 "Content-Type": browserHeaders["content-type"],
 Origin: browserHeaders.origin,
 Referer: browserHeaders.referer,
 Cookie: session.getCookieString(cloudflareAuthReq.url()),
 "Content-Length": cloudflareAuthReq.postData().length,
 },
 method: "POST",
 }
 // Send the challenge response from requestAsBrowser
 const challengeResponse = await requestAsBrowser(finalRequestOpts)

 if (this._isChallengeSolvedFunction(challengeResponse)) {
 // Success
 request.retryCount = 0
 session.setCookiesFromResponse(challengeResponse)
 this.log.info("Successfully unblocked")
 return challengeResponse
 }

 // Something went wrong - challenge was not solved.
 session.retire()
 await Apify.setValue(
 `BLOCKED-HTML-${challengeResponse.statusCode}-${Math.random() * 1000}`,
 challengeResponse.body,
 { contentType: "text/html" },
 )
 this._throwError("Blocked")
 }

 /**
 * Locks session
 * @ODO: we could add this function to the Session natively
 * @param session {Session}
 * @private
 */
 _markSessionBeingRenewed (session) {
 session.userData.isBeingRenewed = true
 }

 /**
 * Unlocks session
 * @param session {Session}
 * @private
 */
 _markSessionNotBeingRenewed (session) {
 session.userData.isBeingRenewed = false
 }

 /**
 * Gets proxy URL
 * @param session {Session};
 * @return {String}
 * @private
 */
 _getProxyUrl (session) {
 return this.proxyConfiguration.newUrl(session.id)
 }

 /**
 *
 * @param session {Session}
 * @return {boolean}
 * @private
 */
 _isSessionBeingRenewed (session) {
 return session.userData.isBeingRenewed
 }

 /**
 * Throws prefixed error
 * @param message {String} - Error message
 * @private
 */
 _throwError (message) {
 throw new Error(`${this.name}: ${message}`)
 }

 /**
 * Opens new page, where solves the challenge and returns the auth request details.
 * @param response
 * @param request - Puppeteer request object
 * @return {Promise<Object>} - Auth request
 * @private
 */
 async _getSolvedChallengeRequest ({ response, request, session }) {
 const { headers, body } = response
 const page = await this.browserPool.newPage()
 let authRequest

 // Add request interceptor to get the authRequest for unlocking our session and forward other network mesures.
 await puppeteer.addInterceptRequestHandler(page, async req => {
 const reqUrl = req.url()
 const method = req.method()

 if (request.url === reqUrl && method === "GET") {
 this.log.info(`Mocking initial navigation request: ${req.url()}`)
 await req.respond({ status: 200, body, headers })
 } else if (this._detectChallengeRequestFunction(req)) {
 authRequest = req
 await req.abort()
 } else if (reqUrl.includes("transparent.gif")) {
 const imageResponse = await this._sendImageRequest(req, session)
 await req.respond({
 status: 200,
 body: imageResponse.body,
 headers: imageResponse.headers,
 })
 } else {
 await req.abort()
 }
 })

 // Add debug to page log
 // @ts-ignore
 page.on("console", msg => this.log.debug(`${this.name}: ${msg.text()}`))

 // Navigate to the unblock url.
 await page.evaluate(url => {
 // @ts-ignore
 window.location.href = url
 }, request.url)

 await this._waitUntilChallengeFinishedFunction()

 // TODO: Probably just await page.close()
 // @ts-ignore
 this.browserPool.recyclePage(page).catch(() => {
 })
 return authRequest
 }

 /**
 * Sends request with Cloudflare image like headers.
 * @param req {Request} - Puppeteer request
 * @param session {Session} - Session instance
 * @return {Promise<import("stream").Readable | import("http").IncomingMessage>}
 * @private
 */
 async _sendImageRequest (req, session) {
 const browserHeaders = req.headers()
 const requestOptions = this.getRequestOptions(session)

 const imageHeaders = {
 Referer: browserHeaders.referer,
 Connection: "keep-alive",
 Pragma: "no-cache",
 "Cache-Control": "no-cache",
 "User-Agent":
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
 "Sec-Fetch-User": "?1",
 Accept: "image/webp,image/apng,image/*,*/*;q=0.8",
 "Sec-Fetch-Site": "none",
 "Sec-Fetch-Mode": "no-cors",
 "Sec-Fetch-Dest": "image",
 "Accept-Encoding": "gzip, deflate, br",
 "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
 Cookie: session.getCookieString(req.url()),
 }

 return requestAsBrowser({
 ...requestOptions,
 url: req.url(),
 abortFunction: () => false,
 headers: imageHeaders,
 })
 }

 /**
 * Gets the browser headers;
 * @param cookieString {String} - Cookie header string.
 * @return {Object} - Browser like headers.
 * @private
 */
 _getBrowserHeaders (cookieString) {
 return {
 Connection: "close",
 Pragma: "no-cache",
 "Cache-Control": "no-cache",
 "Upgrade-Insecure-Requests": "1",
 "User-Agent":
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
 "Sec-Fetch-User": "?1",
 Accept:
 "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
 "Sec-Fetch-Site": "none",
 "Sec-Fetch-Mode": "navigate",
 "Sec-Fetch-Dest": "document",
 "Accept-Encoding": "gzip, deflate, br",
 "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
 Cookie: cookieString,
 }
 }

 /**
 * Detects the Cloudflare challenge page by 503 status code and the CDATA tag;
 * @param response
 * @return {boolean}
 * @private
 */
 _detectChallengeFunction (response) {
 return response.statusCode === 503 && response.body.includes("CDATA")
 }

 /**
 * Detects Challenge request
 * @param request {Request} - Puppeteer request
 * @return {boolean} - true if challenge request detected
 * @private
 */
 _detectChallengeRequestFunction (request) {
 return request.method() === "POST"
 }

 /**
 * Waits until the challenge is computed.
 * @return {Promise<void>}
 * @private
 */
 async _waitUntilChallengeFinishedFunction () {
 await Apify.utils.sleep(5000)
 }

 /**
 * Checks if challenge is successfully solved.
 * @param challengeResponse
 * @return {boolean}
 * @private
 */
 _isChallengeSolvedFunction (challengeResponse) {
 return challengeResponse.statusCode === 200
 }

 /**
 * Gets request options - these options are compatible with `Apify.utils.requestAsBrowser` and `@apify/http-request`
 * @param session {Session} - Session instance
 * @return {Object} - Contains request options
 */
 getRequestOptions (session) {
 const proxyUrl = this._getProxyUrl(session)

 const requestOptions = {
 headers: this._getBrowserHeaders(
 session.getCookieString(this.unblockUrl),
 ),
 proxyUrl,
 }

 if (this._shouldUseCustomTLS) {
 // @ts-ignore
 if (!requestOptions.https) requestOptions.https = {}
 requestOptions.https.ciphers = "AES256-SHA"
 }

 return requestOptions
 }

 /**
 * Creates new Session instance for the SessionPool.
 * @param sessionPool {SessionPool}.
 * @return {Promise<Session>}
 */
 async createSessionFunction (sessionPool) {
 const session = new Session({ sessionPool })
 try {
 await this.unblock({ session, request: { url: this.unblockUrl } })
 } catch (e) {
 log.warning(`${this.name}: Could not unblock session`)
 log.exception(e, 'Unblocker failed')
 }
 return session
 }

 /**
 * Gets options for the CheerioCrawler use interface.
 * @return {Object} - CheerioCrawler options
 */
 getCrawlerOptions () {
 const that = this
 return {
 useSessionPool: true,
 sessionPoolOptions: {
 maxPoolSize: 100,
 createSessionFunction: this.createSessionFunction.bind(this),
 },
 persistCookiesPerSession: true,
 useApifyProxy: true,
 async prepareRequestFunction ({ request, session }) {
 const newOptions = that.getRequestOptions(session)
 request.headers = newOptions.headers
 // @ts-ignore
 if (newOptions.ciphers) {
 // @ts-ignore
 this.requestOptions = { ciphers: newOptions.ciphers }
 }
 },
 }
 }
}

import { createHash } from 'crypto';

// inspired by @drobnikj
const createUniqueKeyFromUrl = (url) => {
  const hash = createHash('sha256');
  const cleanUrl = url.split('://')[1]; // Remove protocol
  hash.update(cleanUrl);
  return hash.digest('hex');
};


// inspired by hlidac-shopu
import Apify from "apify";
import { defAtom } from "@thi.ng/atom";

// TODO: make this lowes common denominator
const defaultStats = {
 urls: 0,
 items: 0,
 itemsDuplicate: 0,
 totalItems: 0,
 denied: 0,
 ok: 0
};

const inc = x => x + 1;
const dec = x => x - 1;

// TODO: stats should be in atom and updated via swap function atomically
class Stats {
 constructor(init) {
 this.stats = defAtom(init);
 this.interval = setInterval(() => this.log(), 20 * 1000);
 }

 inc(key) {
 this.stats.swapIn(key, inc);
 }

 dec(key) {
 this.stats.swapIn(key, dec);
 }

 add(key, value) {
 this.stats.swapIn(key, x => x + value);
 }

 get() {
 return this.stats.deref();
 }

 log() {
 const stats = this.stats.deref();
 Apify.utils.log.info(`stats: ${JSON.stringify(stats)}`);
 }

 /**
 * @param final {boolean} - If true, clearInterval apply
 */
 async save(final = false) {
 if (final) {
 clearInterval(this.interval);
 }
 const stats = this.stats.deref();
 await Apify.setValue("STATS", this.get());
 Apify.utils.log.info("STATS saved!");
 if (stats.ok) {
 Apify.utils.log.info(
 `Denied ratio: ${(stats.denied ?? 0 / stats.ok) * 100} %`
 );
 }
 this.log();
 }
}

/**
 *
 * @param {function} fn
 * @param {*} init
 * @returns {Promise<Stats>}
 */
export async function withPersistedStats(fn, init) {
 const stats = (await Apify.getValue("STATS")) ?? init ?? defaultStats;
 const state = new Stats(fn(stats));
 const persistState = () => state.save();

 Apify.events.on("persistState", persistState);
 Apify.events.on("migrating", persistState);

 return state;
}

Mode

Country

Debug

Where next?