FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
  && npm install --only=prod --no-optional

COPY . ./

Dockerfile

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

Country

Debug mode prints more logs, disables concurrency and other optimizations.

Debug

Mode

Český ráj (ceskyraj.com) scraper

INPUT_SCHEMA.json

cesky-raj-ceskyraj-com-scraper

apify.json

/**
 * TODO:
 * - consider proxies
 * - DRY price parsing
 * - different countries to input
 *
 * Beware that same product can have multiple valid urls
 * - https://www.alza.cz/iphone-13-512gb-cervena-levne-d6839524.htm
 * - https://www.alza.cz/sport/victorias-secret-st-11128877-cc-4vmq-cerna
 * - https://www.alza.cz/sport/victorias-secret-st-11156655-cc-38h2-bezova?dq=6920061
 *
 * Variants can affect price
 * - https://www.alza.cz/asus-rog-zephyrus-g14-ga401?dq=6804643 38k
 * - https://www.alza.cz/asus-rog-zephyrus-g14-ga401?dq=6771118 39k
 *
 * Pagination:
 * beware: only first next 3 pages are listed
 * -> need to use total amount of products & product per page to calculate total amount of pages
 */

import Apify from "apify";
import cheerio from "cheerio";
import { gotScraping } from "got-scraping";
import { withPersistedStats } from "./_utils/stats.js";

const { log } = Apify.utils;

var LABEL;

(function (LABEL) {
  LABEL["INDEX"] = "INDEX";
  LABEL["PRODUCTS"] = "PRODUCTS";
})(LABEL || (LABEL = {}));
var MODE;

(function (MODE) {
  MODE["TEST"] = "TEST";
  MODE["FULL"] = "FULL";
  MODE["SINGLE"] = "SINGLE";
})(MODE || (MODE = {}));
var Country;

// TODO: Maybe unify with Country enum
(function (Country) {
  Country["CZ"] = "CZ";
  Country["SK"] = "SK";
  Country["UK"] = "UK";
  Country["DE"] = "DE";
  Country["AT"] = "AT";
  Country["HU"] = "HU";
})(Country || (Country = {}));
const Countries = {
  CZ: {
    domain: "cz",
    currency: "CZK",
  },
  SK: {
    domain: "sk",
    currency: "EUR",
  },
  UK: {
    domain: "co.uk",
    currency: "GBP",
  },
  DE: {
    domain: "de",
    currency: "EUR",
  },
  AT: {
    domain: "at",
    currency: "EUR",
  },
  HU: {
    domain: "hu",
    currency: "HUF",
  },
};

async function enqueueInitialRequest(type, requestQueue, countryDef) {
  if (type === MODE.FULL) {
    await requestQueue.addRequest({
      userData: { label: LABEL.INDEX },
      url: `https://alza.${countryDef.domain}/_sitemap-categories.xml`,
    });
  } else if (type === MODE.TEST) {
    await requestQueue.addRequest({
      userData: { label: LABEL.PRODUCTS },
      url: "https://www.alza.cz/fotobrasny/18848740.htm",
    });
    await requestQueue.addRequest({
      userData: { label: LABEL.PRODUCTS },
      url: "https://www.alza.cz/objektivy-sony-fe/18879399.htm",
    });
    await requestQueue.addRequest({
      userData: { label: LABEL.PRODUCTS },
      url: "https://www.alza.cz/baterie-pro-fotoaparaty/18851400.htm",
    });
  } else if (type === MODE.SINGLE) {
    await requestQueue.addRequest({
      userData: { label: LABEL.PRODUCTS },
      url: "https://www.alza.sk/iphone-mobilne-telefony/18851638.htm", // single category, but containing multiple pages
    });
  }
}

async function handleIndex(
  { request, session = null },
  { mode, stats, requestQueue }
) {
  const response = await gotScraping({
    url: request.url,
  });
  if (![200, 404].includes(response.statusCode)) {
    session.retire();
    request.retryCount--;
    throw new Error(
      `Blocked, statusCode ${response.statusCode}, url: ${request.url}`
    );
  }
  const $ = cheerio.load(response.body, { xmlMode: true });
  const urls = $("url loc").map((i, x) => $(x).text());
  for (const url of urls) {
    await requestQueue.addRequest({
      userData: { label: LABEL.PRODUCTS },
      url,
    });
  }
}

async function handleProducts(
  { request, session = null },
  { stats, processedIds, requestQueue, countryDef }
) {
  console.log("handleProducts", request.url);

  const response = await gotScraping({
    url: request.url,
    headers: {
      Host: `www.alza.${countryDef.domain}`, // IMPORTANT to get prices in CZK, not EUR
    },
    // proxyUrl: await proxyConfiguration.newUrl(session.id),
  });

  const $ = cheerio.load(response.body);

  if (
    // no pagination info in URL -> we are on a first/initial page -> enqueue next pages
    !request.url.match(/-p\d+\.htm/)
  ) {
    // Beware: only first next 3 pages are listed, so following code is not correct
    // $('#pagerbottom a[class="pgn"]').each((i, el) => {
    //   // beware: href attr does not start with `/`, as product urls do
    //   const url = `https://alza.${countryDef.domain}/${($(el).attr("href"))}`
    //   if (i === 0) return // skip first page, as we are already on it
    //   void requestQueue.addRequest({
    //     userData: { label: LABEL.PRODUCTS },
    //     url,
    //   })
    // })

    // Method using total amount of products and product per page to calculate total amount of pages
    const totalAmountOfProducts = parseInt($("#lblNumberItem").text());
    const productsPerPage = 24; // TODO: Maybe to constant
    const totalPages = Math.ceil(totalAmountOfProducts / productsPerPage);
    for (let i = 2; i <= totalPages; i++) {
      const url = request.url // e.g. `/iphone-mobilne-telefony/18851638.htm`
        .replace(/\.htm$/, "") // `/iphone-mobilne-telefony/18851638`
        .concat(`-p${i}.htm`); // e.g. `/iphone-mobilne-telefony/18851638-p2.htm`
      console.log("enqueue next page", url);
      void requestQueue.addRequest({
        userData: { label: LABEL.PRODUCTS },
        url,
      });
    }
  }

  $(".browsingitem").each(function () {
    const itemId = $(this).attr("data-id");
    if (!processedIds.has(itemId)) {
      stats.inc("itemsUnique");
      processedIds.add(itemId);
      const itemCode = $(this).attr("data-code"); // non-standard product property
      const img = $(this).find("a.pc.browsinglink img").data("src");
      const relativeItemUrl = $(this)
        .find("a.name.browsinglink")
        .first()
        .attr("href");
      const absoluteItemUrl = `https://alza.${countryDef.domain}${relativeItemUrl}`;
      const itemName = $(this)
        .find("a.name.browsinglink")
        .first()
        .text()
        .replace(/([\n\r])/g, "") // replace newlines and carriage returns
        .trim();

      const currentPrice = $(this)
        .find(".price .c2")
        .text()
        .replace(/,-$/, "")
        .replace(/[^\d.,]/g, "");

      const originalPrice = $(this)
        .find(".price .np2")
        .text()
        .replace(/,-$/, "")
        .replace(/[^\d.,]/g, "");

      const product = {
        itemId,
        itemName,
        itemUrl: absoluteItemUrl,
        img,
        inStock: "TODO",
        currentPrice,
        originalPrice,
        currency: countryDef.currency,
        category: "FIXME",

        // Derived
        discounted: originalPrice && originalPrice !== currentPrice,
        _discount: originalPrice
          ? // @ts-ignore
            (1 - currentPrice / originalPrice) * 100
          : undefined,
      };
      console.log(`Found product: ${itemId}`);
      Apify.pushData(product);
    } else {
      stats.inc("itemsDuplicate");
    }
  });
}

const proxyGroups = [
  "HUNGARY",
  "CZECH_LUMINATI",
  "GERMANY",
  "FRANCE",
  "ITALY",
  "SPAIN",
];

Apify.main(async () => {
  await Apify.utils.purgeLocalStorage();

  const stats = await withPersistedStats((x) => x, {
    categories: 0,
    items: 0,
    itemsUnique: 0,
    itemsDuplicate: 0,
    denied: 0,
    captchas: 0,
  });
  const processedIds = new Set();

  const input = await Apify.getInput();
  const { debug = false, country = Country.CZ, mode = MODE.TEST } = input ?? {};

  const countryDef = Countries[country];

  if (debug) Apify.utils.log.setLevel(Apify.utils.log.LEVELS.DEBUG);

  // const proxyConfiguration = await Apify.createProxyConfiguration({
  //   groups: proxyGroups,
  // })

  const requestQueue = await Apify.openRequestQueue();
  await enqueueInitialRequest(mode, requestQueue, countryDef);

  const globalContext = { mode, stats, requestQueue, processedIds, countryDef };
  const crawler = new Apify.BasicCrawler({
    requestQueue,
    maxConcurrency: 1, // FIXME
    maxRequestRetries: 1,
    async handleRequestFunction(context) {
      const { request, session } = context;
      const { userData } = request;
      // @ts-ignore
      switch (userData.label) {
        case LABEL.INDEX:
          return await handleIndex(context, globalContext);
        case LABEL.PRODUCTS:
          return await handleProducts(context, globalContext);
      }
    },
    async handleFailedRequestFunction({ request }) {
      log.error(`Request ${request.url} failed multiple times`, request);
    },
  });

  await crawler.run();
  log.info("crawler finished");
});


main.js

package.json

.actor/actor.json

import Apify from "apify"
import { Session } from "apify/build/session_pool/session.js"
import { BrowserPool, PlaywrightPlugin } from "browser-pool"
import playwright from "playwright"

const { utils: { log, requestAsBrowser, puppeteer } } = Apify

export default class CloudflareUnblocker {
  // unblockUrl: string
  // proxyConfiguration: any
  // browserPool: BrowserPool
  // _shouldUseCustomTLS: boolean
  // log: any
  // name: string

  constructor (options) {
    // Store options
    this.unblockUrl = options.unblockUrl
    this.proxyConfiguration = options.proxyConfiguration

    // Create browser pool
    this.browserPool = new BrowserPool({
      retireBrowserAfterPageCount: 50, // TODO: explain
      maxOpenPagesPerBrowser: 10, // TODO: explain
      browserPlugins: [
        new PlaywrightPlugin(playwright.firefox, {
          launchOptions: { headless: false },
        }),
      ],
    })

    this._shouldUseCustomTLS = false

    // Extending CrawlerExtension caused error:
    // Class extends value #<Object> is not a constructor or null
    // But all it should do is to add log method, so I'm adding it here
    this.name = this.constructor.name
    this.log = Apify.utils.log.child({ prefix: this.constructor.name })
  }

  /**
   * Main function that unblocks your session.
   */
  async unblock ({ session, request }) {
    if (this._isSessionBeingRenewed(session)) {
      request.retryCount = 0
      this._throwError("Session is being renewed")
    }

    const oldShouldUseTLS = this._shouldUseCustomTLS
    const requestOptions = this.getRequestOptions(session)
    const response = await requestAsBrowser({
      ...requestOptions,
      url: request.url,
    })

    if (this._detectChallengeFunction(response)) {
      // Browser challenge detected starting the bypassing
      try {
        this._markSessionBeingRenewed(session)
        return this._solveChallenge({
          response,
          session,
          request,
          requestOptions,
        })
      } finally {
        this._markSessionNotBeingRenewed(session)
      }
    } else if (response.statusCode === 403) {
      // Cloudflare captcha detected -> switching to slower TLS
      if (oldShouldUseTLS) {
        this.log.info("Captcha found even with the TLS hack")
        await Apify.setValue(
          `CAPTCHA-HTML-${Math.random() * 1000}`,
          response.body,
          { contentType: "text/html" },
        )
      } else {
        this.log.info(
          "Captcha found for the first time -> switching to custom TLS",
        )
        this._shouldUseCustomTLS = true
      }
    }

    this.log.info("Session OK")
    session.setCookiesFromResponse(response)
    return response
  }

  /**
   * Solves the challenge by starting the browser and saving the cookies to the session.
   * @param options
   * @param options.request
   * @param options.response
   * @param options.session
   * @param options.requestOptions
   * @return {Promise<*>}
   * @private
   */
  async _solveChallenge (options) {
    const { request, response, session } = options
    const { body, headers } = response

    const receivedCookies = headers["set-cookie"]
    log.debug(`${this.name}: received cookies: ${receivedCookies}`)
    const requestOptions = this.getRequestOptions(session)

    if (!receivedCookies) {
      await Apify.setValue(`NO-COOKIES-HTML-${Math.random() * 1000}`, body, {
        contentType: "text/html",
      })
    }

    session.setCookiesFromResponse(response)

    const cloudflareAuthReq = await this._getSolvedChallengeRequest({
      response,
      session,
      request,
    })

    const browserHeaders = cloudflareAuthReq.headers()
    const finalRequestOpts = {
      ...requestOptions,
      url: cloudflareAuthReq.url(),
      payload: cloudflareAuthReq.postData(),
      headers: {
        ...requestOptions.headers,
        "Content-Type": browserHeaders["content-type"],
        Origin: browserHeaders.origin,
        Referer: browserHeaders.referer,
        Cookie: session.getCookieString(cloudflareAuthReq.url()),
        "Content-Length": cloudflareAuthReq.postData().length,
      },
      method: "POST",
    }
    // Send the challenge response from requestAsBrowser
    const challengeResponse = await requestAsBrowser(finalRequestOpts)

    if (this._isChallengeSolvedFunction(challengeResponse)) {
      // Success
      request.retryCount = 0
      session.setCookiesFromResponse(challengeResponse)
      this.log.info("Successfully unblocked")
      return challengeResponse
    }

    // Something went wrong - challenge was not solved.
    session.retire()
    await Apify.setValue(
      `BLOCKED-HTML-${challengeResponse.statusCode}-${Math.random() * 1000}`,
      challengeResponse.body,
      { contentType: "text/html" },
    )
    this._throwError("Blocked")
  }

  /**
   * Locks session
   * @ODO: we could add this function to the Session natively
   * @param session {Session}
   * @private
   */
  _markSessionBeingRenewed (session) {
    session.userData.isBeingRenewed = true
  }

  /**
   * Unlocks session
   * @param session {Session}
   * @private
   */
  _markSessionNotBeingRenewed (session) {
    session.userData.isBeingRenewed = false
  }

  /**
   * Gets proxy URL
   * @param session {Session};
   * @return {String}
   * @private
   */
  _getProxyUrl (session) {
    return this.proxyConfiguration.newUrl(session.id)
  }

  /**
   *
   * @param session {Session}
   * @return {boolean}
   * @private
   */
  _isSessionBeingRenewed (session) {
    return session.userData.isBeingRenewed
  }

  /**
   * Throws prefixed error
   * @param message {String} - Error message
   * @private
   */
  _throwError (message) {
    throw new Error(`${this.name}: ${message}`)
  }

  /**
   * Opens new page, where solves the challenge and returns the auth request details.
   * @param response
   * @param request - Puppeteer request object
   * @return {Promise<Object>} - Auth request
   * @private
   */
  async _getSolvedChallengeRequest ({ response, request, session }) {
    const { headers, body } = response
    const page = await this.browserPool.newPage()
    let authRequest

    // Add request interceptor to get the authRequest for unlocking our session and forward other network mesures.
    await puppeteer.addInterceptRequestHandler(page, async req => {
      const reqUrl = req.url()
      const method = req.method()

      if (request.url === reqUrl && method === "GET") {
        this.log.info(`Mocking initial navigation request: ${req.url()}`)
        await req.respond({ status: 200, body, headers })
      } else if (this._detectChallengeRequestFunction(req)) {
        authRequest = req
        await req.abort()
      } else if (reqUrl.includes("transparent.gif")) {
        const imageResponse = await this._sendImageRequest(req, session)
        await req.respond({
          status: 200,
          body: imageResponse.body,
          headers: imageResponse.headers,
        })
      } else {
        await req.abort()
      }
    })

    // Add debug to page log
    // @ts-ignore
    page.on("console", msg => this.log.debug(`${this.name}: ${msg.text()}`))

    // Navigate to the unblock url.
    await page.evaluate(url => {
      // @ts-ignore
      window.location.href = url
    }, request.url)

    await this._waitUntilChallengeFinishedFunction()

    // TODO: Probably just await page.close()
    // @ts-ignore
    this.browserPool.recyclePage(page).catch(() => {
    })
    return authRequest
  }

  /**
   * Sends request with Cloudflare image like headers.
   * @param req {Request} - Puppeteer request
   * @param session {Session} - Session instance
   * @return {Promise<import("stream").Readable | import("http").IncomingMessage>}
   * @private
   */
  async _sendImageRequest (req, session) {
    const browserHeaders = req.headers()
    const requestOptions = this.getRequestOptions(session)

    const imageHeaders = {
      Referer: browserHeaders.referer,
      Connection: "keep-alive",
      Pragma: "no-cache",
      "Cache-Control": "no-cache",
      "User-Agent":
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
      "Sec-Fetch-User": "?1",
      Accept: "image/webp,image/apng,image/*,*/*;q=0.8",
      "Sec-Fetch-Site": "none",
      "Sec-Fetch-Mode": "no-cors",
      "Sec-Fetch-Dest": "image",
      "Accept-Encoding": "gzip, deflate, br",
      "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
      Cookie: session.getCookieString(req.url()),
    }

    return requestAsBrowser({
      ...requestOptions,
      url: req.url(),
      abortFunction: () => false,
      headers: imageHeaders,
    })
  }

  /**
   * Gets the browser headers;
   * @param cookieString {String} - Cookie header string.
   * @return {Object} - Browser like headers.
   * @private
   */
  _getBrowserHeaders (cookieString) {
    return {
      Connection: "close",
      Pragma: "no-cache",
      "Cache-Control": "no-cache",
      "Upgrade-Insecure-Requests": "1",
      "User-Agent":
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
      "Sec-Fetch-User": "?1",
      Accept:
        "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
      "Sec-Fetch-Site": "none",
      "Sec-Fetch-Mode": "navigate",
      "Sec-Fetch-Dest": "document",
      "Accept-Encoding": "gzip, deflate, br",
      "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
      Cookie: cookieString,
    }
  }

  /**
   * Detects the Cloudflare challenge page by 503 status code and the CDATA tag;
   * @param response
   * @return {boolean}
   * @private
   */
  _detectChallengeFunction (response) {
    return response.statusCode === 503 && response.body.includes("CDATA")
  }

  /**
   * Detects Challenge request
   * @param request {Request} - Puppeteer request
   * @return {boolean} - true if challenge request detected
   * @private
   */
  _detectChallengeRequestFunction (request) {
    return request.method() === "POST"
  }

  /**
   * Waits until the challenge is computed.
   * @return {Promise<void>}
   * @private
   */
  async _waitUntilChallengeFinishedFunction () {
    await Apify.utils.sleep(5000)
  }

  /**
   * Checks if challenge is successfully solved.
   * @param challengeResponse
   * @return {boolean}
   * @private
   */
  _isChallengeSolvedFunction (challengeResponse) {
    return challengeResponse.statusCode === 200
  }

  /**
   * Gets request options - these options are compatible with `Apify.utils.requestAsBrowser` and `@apify/http-request`
   * @param session {Session} - Session instance
   * @return {Object} - Contains request options
   */
  getRequestOptions (session) {
    const proxyUrl = this._getProxyUrl(session)

    const requestOptions = {
      headers: this._getBrowserHeaders(
        session.getCookieString(this.unblockUrl),
      ),
      proxyUrl,
    }

    if (this._shouldUseCustomTLS) {
      // @ts-ignore
      if (!requestOptions.https) requestOptions.https = {}
      requestOptions.https.ciphers = "AES256-SHA"
    }

    return requestOptions
  }

  /**
   * Creates new Session instance for the SessionPool.
   * @param sessionPool {SessionPool}.
   * @return {Promise<Session>}
   */
  async createSessionFunction (sessionPool) {
    const session = new Session({ sessionPool })
    try {
      await this.unblock({ session, request: { url: this.unblockUrl } })
    } catch (e) {
      log.warning(`${this.name}: Could not unblock session`)
      log.exception(e, 'Unblocker failed')
    }
    return session
  }

  /**
   * Gets options for the CheerioCrawler use interface.
   * @return {Object} - CheerioCrawler options
   */
  getCrawlerOptions () {
    const that = this
    return {
      useSessionPool: true,
      sessionPoolOptions: {
        maxPoolSize: 100,
        createSessionFunction: this.createSessionFunction.bind(this),
      },
      persistCookiesPerSession: true,
      useApifyProxy: true,
      async prepareRequestFunction ({ request, session }) {
        const newOptions = that.getRequestOptions(session)
        request.headers = newOptions.headers
        // @ts-ignore
        if (newOptions.ciphers) {
          // @ts-ignore
          this.requestOptions = { ciphers: newOptions.ciphers }
        }
      },
    }
  }
}


_utils/CloudflareUnblocker.js

import { createHash } from 'crypto';

// inspired by @drobnikj
const createUniqueKeyFromUrl = (url) => {
  const hash = createHash('sha256');
  const cleanUrl = url.split('://')[1]; // Remove protocol
  hash.update(cleanUrl);
  return hash.digest('hex');
};


_utils/common.js

// inspired by hlidac-shopu
import Apify from "apify";
import { defAtom } from "@thi.ng/atom";

// TODO: make this lowes common denominator
const defaultStats = {
  urls: 0,
  items: 0,
  itemsDuplicate: 0,
  totalItems: 0,
  denied: 0,
  ok: 0
};

const inc = x => x + 1;
const dec = x => x - 1;

// TODO: stats should be in atom and updated via swap function atomically
class Stats {
  constructor(init) {
    this.stats = defAtom(init);
    this.interval = setInterval(() => this.log(), 20 * 1000);
  }

  inc(key) {
    this.stats.swapIn(key, inc);
  }

  dec(key) {
    this.stats.swapIn(key, dec);
  }

  add(key, value) {
    this.stats.swapIn(key, x => x + value);
  }

  get() {
    return this.stats.deref();
  }

  log() {
    const stats = this.stats.deref();
    Apify.utils.log.info(`stats: ${JSON.stringify(stats)}`);
  }

  /**
   * @param final {boolean} - If true, clearInterval apply
   */
  async save(final = false) {
    if (final) {
      clearInterval(this.interval);
    }
    const stats = this.stats.deref();
    await Apify.setValue("STATS", this.get());
    Apify.utils.log.info("STATS saved!");
    if (stats.ok) {
      Apify.utils.log.info(
        `Denied ratio: ${(stats.denied ?? 0 / stats.ok) * 100} %`
      );
    }
    this.log();
  }
}

/**
 *
 * @param {function} fn
 * @param {*} init
 * @returns {Promise<Stats>}
 */
export async function withPersistedStats(fn, init) {
  const stats = (await Apify.getValue("STATS")) ?? init ?? defaultStats;
  const state = new Stats(fn(stats));
  const persistState = () => state.save();

  Apify.events.on("persistState", persistState);
  Apify.events.on("migrating", persistState);

  return state;
}


Where next?