FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
 &amp;&amp; npm install --only=prod --no-optional

COPY . ./

Dockerfile

{
 "title": "Bike24 (bike24.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "type": "object",
 "schemaVersion": 1,
 "properties": {
 "mode": {
 "title": "Mode",
 "description": "",
 "type": "string",
 "editor": "select",
 "default": "FULL",
 "prefill": "FULL",
 "enumTitles": [
 "FULL",
 "SINGLE"
 ],
 "enum": [
 "FULL",
 "SINGLE"
 ]
 },
 "debug": {
 "title": "Debug",
 "description": "Debug mode prints more logs, disables concurrency and other optimizations.",
 "type": "boolean",
 "editor": "checkbox",
 "default": false
 }
 },
 "required": [
 "mode"
 ]
}

INPUT_SCHEMA.json

# Bike24 (bike24.de) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

🤖🚫 **BEWARE**: Requires access to "RESIDENTIAL" proxy group.

## Output example

* **itemId** `string` e.g. *16678*
* **itemName** `string`
* **itemUrl** `string`
* **img** `string`
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *19.95*
* **originalPrice** `number` e.g. *39.95*
* **currency** `string` e.g. *EUR*

README.md

{
 "name": "bike24-bike24-de-scraper",
 "version": "0.1",
 "buildTag": "latest",
 "env": null,
 "template": "project_cheerio_crawler"
}

apify.json

main.js

{
 "name": "bike24-bike24-de-scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "type": "module",
 "scripts": {
 "start": "node ./main.js",
 "push-to-apify-platform": "npx apify push"
 },
 "dependencies": {
 "@thi.ng/atom": "*",
 "apify": "*",
 "browser-pool": "*",
 "playwright": "*"
 },
 "apify": {
 "title": "Bike24 (bike24.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "isPublic": true,
 "isDeprecated": false,
 "issuesEnabled": true,
 "isAnonymouslyRunnable": true,
 "notice": "",
 "pictureUrl": "",
 "seoTitle": "",
 "seoDescription": "",
 "categories": [
 "ECOMMERCE"
 ]
 }
}

package.json

{
 "actorSpecification": 1,
 "name": "bike24-bike24-de-scraper",
 "title": "Bike24 (bike24.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "version": "0.1.0",
 "storages": {
 "dataset": {
 "actorSpecification": 1,
 "title": "Bike24 (bike24.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "views": {
 "overview": {
 "title": "Overview",
 "description": "Overview of the most important fields",
 "transformation": {
 "fields": [
 "itemId",
 "itemName",
 "itemUrl",
 "img",
 "inStock",
 "currentPrice",
 "originalPrice",
 "currency"
 ]
 },
 "display": {
 "component": "table",
 "columns": [
 {
 "label": "Item ID",
 "field": "itemUrl",
 "format": "link",
 "textField": "itemId"
 },
 {
 "label": "Item Name",
 "field": "itemName",
 "format": "text"
 },
 {
 "label": "Img",
 "field": "img",
 "format": "image"
 },
 {
 "label": "In Stock",
 "field": "inStock",
 "format": "boolean"
 },
 {
 "label": "Current Price",
 "field": "currentPrice",
 "format": "number"
 },
 {
 "label": "Original Price",
 "field": "originalPrice",
 "format": "number"
 },
 {
 "label": "Currency",
 "field": "currency",
 "format": "text"
 }
 ]
 }
 }
 }
 }
 }
}

.actor/actor.json

_utils/CloudflareUnblocker.js

import { createHash } from 'crypto';

// inspired by @drobnikj
const createUniqueKeyFromUrl = (url) => {
 const hash = createHash('sha256');
 const cleanUrl = url.split('://')[1]; // Remove protocol
 hash.update(cleanUrl);
 return hash.digest('hex');
};

_utils/common.js

// inspired by hlidac-shopu
import Apify from "apify";
import { defAtom } from "@thi.ng/atom";

// TODO: make this lowes common denominator
const defaultStats = {
 urls: 0,
 items: 0,
 itemsDuplicate: 0,
 totalItems: 0,
 denied: 0,
 ok: 0
};

const inc = x => x + 1;
const dec = x => x - 1;

// TODO: stats should be in atom and updated via swap function atomically
class Stats {
 constructor(init) {
 this.stats = defAtom(init);
 this.interval = setInterval(() => this.log(), 20 * 1000);
 }

 inc(key) {
 this.stats.swapIn(key, inc);
 }

 dec(key) {
 this.stats.swapIn(key, dec);
 }

 add(key, value) {
 this.stats.swapIn(key, x => x + value);
 }

 get() {
 return this.stats.deref();
 }

 log() {
 const stats = this.stats.deref();
 Apify.utils.log.info(`stats: ${JSON.stringify(stats)}`);
 }

 /**
 * @param final {boolean} - If true, clearInterval apply
 */
 async save(final = false) {
 if (final) {
 clearInterval(this.interval);
 }
 const stats = this.stats.deref();
 await Apify.setValue("STATS", this.get());
 Apify.utils.log.info("STATS saved!");
 if (stats.ok) {
 Apify.utils.log.info(
 `Denied ratio: ${(stats.denied ?? 0 / stats.ok) * 100} %`
 );
 }
 this.log();
 }
}

/**
 *
 * @param {function} fn
 * @param {*} init
 * @returns {Promise&lt;Stats>}
 */
export async function withPersistedStats(fn, init) {
 const stats = (await Apify.getValue("STATS")) ?? init ?? defaultStats;
 const state = new Stats(fn(stats));
 const persistState = () => state.save();

 Apify.events.on("persistState", persistState);
 Apify.events.on("migrating", persistState);

 return state;
}

_utils/stats.js

FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
  && npm install --only=prod --no-optional

COPY . ./

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

Debug mode prints more logs, disables concurrency and other optimizations.

Debug

Mode

Bike24 (bike24.de) scraper

# Bike24 (bike24.de) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

🤖🚫 **BEWARE**: Requires access to "RESIDENTIAL" proxy group.

## Output example

* **itemId** `string` e.g. *16678*
* **itemName** `string`
* **itemUrl** `string`
* **img** `string`
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *19.95*
* **originalPrice** `number` e.g. *39.95*
* **currency** `string` e.g. *EUR*

bike24-bike24-de-scraper

/**
 * Dev notes
 * ===
 *
 * - uses Cloudflare anti-bot protection
 *
 * Params
 * ---
 * -
 *
 * Selectors
 * ---
 * -
 */

import { URL } from "node:url";
import Apify from "apify";

// This is here only cause my bundler does not traverse dependencies
// FIXME
import "browser-pool";
import "playwright";

import { withPersistedStats } from "./_utils/stats.js";
import CloudflareUnblocker from "./_utils/CloudflareUnblocker.js";
const { log } = Apify.utils;

const LABELS = {
 INDEX: "INDEX",
 PRODUCTS: "PRODUCTS",
};

var MODE;

(function (MODE) {
 MODE["FULL"] = "FULL";
 MODE["SINGLE"] = "SINGLE";
})(MODE || (MODE = {}));

const BASE_URL = "https://www.bike24.com";

async function enqueueInitialRequest(mode, requestQueue) {
 if (mode === MODE.FULL) {
 await requestQueue.addRequest({
 userData: { label: LABELS.INDEX },
 url: "https://www.bike24.com/brands",
 });
 } else if (mode === MODE.SINGLE) {
 await requestQueue.addRequest({
 userData: { label: LABELS.PRODUCTS },
 url: "https://www.bike24.com/brands/100percent",
 });
 }
}

async function handleIndex({ $ }, { stats, requestQueue }) {
 $(".list-brands-sitemap__section-item a").each((i, el) => {
 const url = $(el).attr("href"); // urls are relative
 const fullUrl = `${BASE_URL}${url}`;
 const name = $(el).text().trim(); // there's extra space at the beginning and end
 stats.inc("categories");
 void requestQueue.addRequest({
 userData: { label: LABELS.PRODUCTS, category: name },
 url,
 });
 });
}

async function handleProducts(
 { $, request },
 { stats, processedIds, requestQueue }
) {
 console.log("handleCategory", request.url);
 if (!request.url.includes("page=")) {
 // on first page
 const totalPages = $(".page-pagination-item").last().text(); // e.g. `12`
 for (let i = 2; i <= totalPages; i++) {
 // skip first page, that is already handled
 const url = new URL(request.url);
 url.searchParams.set("page", i.toString());
 void requestQueue.addRequest({
 url: url.toString(),
 userData: {
 label: LABELS.PRODUCTS,
 category: request.userData.category, // pass category name
 },
 });
 }
 }

 $(".product-tile").each((i, el) => {
 stats.inc("items");
 const id = $(el)
 .find(".product-tile__anchor")
 .attr("href")
 .replace(/[^0-9]/g, ""); // e.g. `p2421335.html` -> `2421335
 if (!processedIds.has(id)) {
 stats.inc("itemsUnique");
 processedIds.add(id);
 const url = $(el).find(".product-tile__anchor").attr("href"); // relative url
 const fullUrl = `${BASE_URL}${url}`;
 const title = $(el).find(".product-tile__title")?.text()?.trim();
 const prices = JSON.parse($(".productPrice").attr("data-props"));
 const img = $(el).find(".product-tile__picture img").attr("src");
 const inStock = !!$(".delivery-message--success").length;
 const product = {
 itemId: id,
 itemName: title,
 itemUrl: url,
 img: img,
 inStock,
 currentPrice: prices.price,
 originalPrice: prices.oldPrice,
 currency: "EUR",

 // Derived
 _discount: prices.oldPrice
 ? (1 - prices.price / prices.oldPrice) * 100
 : undefined,
 };
 Apify.pushData(product);
 } else {
 stats.inc("itemsDuplicate");
 }
 });
}

Apify.main(async () => {
 await Apify.utils.purgeLocalStorage();

 const stats = await withPersistedStats((x) => x, {
 categories: 0,
 items: 0,
 itemsUnique: 0,
 itemsDuplicate: 0,
 });
 const processedIds = new Set();

 const input = await Apify.getInput();
 const { debug = true, mode = MODE.SINGLE } = input ?? {};

 if (debug) Apify.utils.log.setLevel(Apify.utils.log.LEVELS.DEBUG);

 const requestQueue = await Apify.openRequestQueue();
 await enqueueInitialRequest(mode, requestQueue);

 const proxyConfiguration = await Apify.createProxyConfiguration({
 groups: ["RESIDENTIAL"], // Not really sure which one to use
 // @ts-ignore
 useApifyProxy: true,
 });

 const cloudflareUnblocker = new CloudflareUnblocker({
 unblockUrl: BASE_URL,
 proxyConfiguration,
 });

 const globalContext = { mode, stats, requestQueue, processedIds };
 const crawler = new Apify.CheerioCrawler({
 requestQueue,
 maxConcurrency: debug ? 1 : 3,
 maxRequestRetries: debug ? 3 : 3,
 useSessionPool: true,
 sessionPoolOptions: {
 maxPoolSize: 100,
 createSessionFunction:
 cloudflareUnblocker.createSessionFunction.bind(cloudflareUnblocker),
 },
 proxyConfiguration,
 persistCookiesPerSession: true,
 preNavigationHooks: [
 async (crawlingContext, requestAsBrowserOptions) => {
 const session = crawlingContext.session;
 const requestOptions = cloudflareUnblocker.getRequestOptions(session);
 Object.assign(requestAsBrowserOptions, requestOptions);
 },
 ],
 async handlePageFunction(context) {
 const { request, response } = context;
 const { userData } = request;
 // @ts-ignore
 const { statusCode, body } = response;

 if (statusCode !== 200) return log.info(body.toString());

 switch (userData.label) {
 case LABELS.INDEX:
 return await handleIndex(context, globalContext);
 case LABELS.PRODUCTS:
 return await handleProducts(context, globalContext);
 }
 },
 async handleFailedRequestFunction({ request }) {
 log.error(`Request ${request.url} failed multiple times`, request);
 },
 });

 await crawler.run();
 log.info("That's all folks!");
});

import Apify from "apify"
import { Session } from "apify/build/session_pool/session.js"
import { BrowserPool, PlaywrightPlugin } from "browser-pool"
import playwright from "playwright"

const { utils: { log, requestAsBrowser, puppeteer } } = Apify

export default class CloudflareUnblocker {
 // unblockUrl: string
 // proxyConfiguration: any
 // browserPool: BrowserPool
 // _shouldUseCustomTLS: boolean
 // log: any
 // name: string

 constructor (options) {
 // Store options
 this.unblockUrl = options.unblockUrl
 this.proxyConfiguration = options.proxyConfiguration

 // Create browser pool
 this.browserPool = new BrowserPool({
 retireBrowserAfterPageCount: 50, // TODO: explain
 maxOpenPagesPerBrowser: 10, // TODO: explain
 browserPlugins: [
 new PlaywrightPlugin(playwright.firefox, {
 launchOptions: { headless: false },
 }),
 ],
 })

 this._shouldUseCustomTLS = false

 // Extending CrawlerExtension caused error:
 // Class extends value #<Object> is not a constructor or null
 // But all it should do is to add log method, so I'm adding it here
 this.name = this.constructor.name
 this.log = Apify.utils.log.child({ prefix: this.constructor.name })
 }

 /**
 * Main function that unblocks your session.
 */
 async unblock ({ session, request }) {
 if (this._isSessionBeingRenewed(session)) {
 request.retryCount = 0
 this._throwError("Session is being renewed")
 }

 const oldShouldUseTLS = this._shouldUseCustomTLS
 const requestOptions = this.getRequestOptions(session)
 const response = await requestAsBrowser({
 ...requestOptions,
 url: request.url,
 })

 if (this._detectChallengeFunction(response)) {
 // Browser challenge detected starting the bypassing
 try {
 this._markSessionBeingRenewed(session)
 return this._solveChallenge({
 response,
 session,
 request,
 requestOptions,
 })
 } finally {
 this._markSessionNotBeingRenewed(session)
 }
 } else if (response.statusCode === 403) {
 // Cloudflare captcha detected -> switching to slower TLS
 if (oldShouldUseTLS) {
 this.log.info("Captcha found even with the TLS hack")
 await Apify.setValue(
 `CAPTCHA-HTML-${Math.random() * 1000}`,
 response.body,
 { contentType: "text/html" },
 )
 } else {
 this.log.info(
 "Captcha found for the first time -> switching to custom TLS",
 )
 this._shouldUseCustomTLS = true
 }
 }

 this.log.info("Session OK")
 session.setCookiesFromResponse(response)
 return response
 }

 /**
 * Solves the challenge by starting the browser and saving the cookies to the session.
 * @param options
 * @param options.request
 * @param options.response
 * @param options.session
 * @param options.requestOptions
 * @return {Promise<*>}
 * @private
 */
 async _solveChallenge (options) {
 const { request, response, session } = options
 const { body, headers } = response

 const receivedCookies = headers["set-cookie"]
 log.debug(`${this.name}: received cookies: ${receivedCookies}`)
 const requestOptions = this.getRequestOptions(session)

 if (!receivedCookies) {
 await Apify.setValue(`NO-COOKIES-HTML-${Math.random() * 1000}`, body, {
 contentType: "text/html",
 })
 }

 session.setCookiesFromResponse(response)

 const cloudflareAuthReq = await this._getSolvedChallengeRequest({
 response,
 session,
 request,
 })

 const browserHeaders = cloudflareAuthReq.headers()
 const finalRequestOpts = {
 ...requestOptions,
 url: cloudflareAuthReq.url(),
 payload: cloudflareAuthReq.postData(),
 headers: {
 ...requestOptions.headers,
 "Content-Type": browserHeaders["content-type"],
 Origin: browserHeaders.origin,
 Referer: browserHeaders.referer,
 Cookie: session.getCookieString(cloudflareAuthReq.url()),
 "Content-Length": cloudflareAuthReq.postData().length,
 },
 method: "POST",
 }
 // Send the challenge response from requestAsBrowser
 const challengeResponse = await requestAsBrowser(finalRequestOpts)

 if (this._isChallengeSolvedFunction(challengeResponse)) {
 // Success
 request.retryCount = 0
 session.setCookiesFromResponse(challengeResponse)
 this.log.info("Successfully unblocked")
 return challengeResponse
 }

 // Something went wrong - challenge was not solved.
 session.retire()
 await Apify.setValue(
 `BLOCKED-HTML-${challengeResponse.statusCode}-${Math.random() * 1000}`,
 challengeResponse.body,
 { contentType: "text/html" },
 )
 this._throwError("Blocked")
 }

 /**
 * Locks session
 * @ODO: we could add this function to the Session natively
 * @param session {Session}
 * @private
 */
 _markSessionBeingRenewed (session) {
 session.userData.isBeingRenewed = true
 }

 /**
 * Unlocks session
 * @param session {Session}
 * @private
 */
 _markSessionNotBeingRenewed (session) {
 session.userData.isBeingRenewed = false
 }

 /**
 * Gets proxy URL
 * @param session {Session};
 * @return {String}
 * @private
 */
 _getProxyUrl (session) {
 return this.proxyConfiguration.newUrl(session.id)
 }

 /**
 *
 * @param session {Session}
 * @return {boolean}
 * @private
 */
 _isSessionBeingRenewed (session) {
 return session.userData.isBeingRenewed
 }

 /**
 * Throws prefixed error
 * @param message {String} - Error message
 * @private
 */
 _throwError (message) {
 throw new Error(`${this.name}: ${message}`)
 }

 /**
 * Opens new page, where solves the challenge and returns the auth request details.
 * @param response
 * @param request - Puppeteer request object
 * @return {Promise<Object>} - Auth request
 * @private
 */
 async _getSolvedChallengeRequest ({ response, request, session }) {
 const { headers, body } = response
 const page = await this.browserPool.newPage()
 let authRequest

 // Add request interceptor to get the authRequest for unlocking our session and forward other network mesures.
 await puppeteer.addInterceptRequestHandler(page, async req => {
 const reqUrl = req.url()
 const method = req.method()

 if (request.url === reqUrl && method === "GET") {
 this.log.info(`Mocking initial navigation request: ${req.url()}`)
 await req.respond({ status: 200, body, headers })
 } else if (this._detectChallengeRequestFunction(req)) {
 authRequest = req
 await req.abort()
 } else if (reqUrl.includes("transparent.gif")) {
 const imageResponse = await this._sendImageRequest(req, session)
 await req.respond({
 status: 200,
 body: imageResponse.body,
 headers: imageResponse.headers,
 })
 } else {
 await req.abort()
 }
 })

 // Add debug to page log
 // @ts-ignore
 page.on("console", msg => this.log.debug(`${this.name}: ${msg.text()}`))

 // Navigate to the unblock url.
 await page.evaluate(url => {
 // @ts-ignore
 window.location.href = url
 }, request.url)

 await this._waitUntilChallengeFinishedFunction()

 // TODO: Probably just await page.close()
 // @ts-ignore
 this.browserPool.recyclePage(page).catch(() => {
 })
 return authRequest
 }

 /**
 * Sends request with Cloudflare image like headers.
 * @param req {Request} - Puppeteer request
 * @param session {Session} - Session instance
 * @return {Promise<import("stream").Readable | import("http").IncomingMessage>}
 * @private
 */
 async _sendImageRequest (req, session) {
 const browserHeaders = req.headers()
 const requestOptions = this.getRequestOptions(session)

 const imageHeaders = {
 Referer: browserHeaders.referer,
 Connection: "keep-alive",
 Pragma: "no-cache",
 "Cache-Control": "no-cache",
 "User-Agent":
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
 "Sec-Fetch-User": "?1",
 Accept: "image/webp,image/apng,image/*,*/*;q=0.8",
 "Sec-Fetch-Site": "none",
 "Sec-Fetch-Mode": "no-cors",
 "Sec-Fetch-Dest": "image",
 "Accept-Encoding": "gzip, deflate, br",
 "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
 Cookie: session.getCookieString(req.url()),
 }

 return requestAsBrowser({
 ...requestOptions,
 url: req.url(),
 abortFunction: () => false,
 headers: imageHeaders,
 })
 }

 /**
 * Gets the browser headers;
 * @param cookieString {String} - Cookie header string.
 * @return {Object} - Browser like headers.
 * @private
 */
 _getBrowserHeaders (cookieString) {
 return {
 Connection: "close",
 Pragma: "no-cache",
 "Cache-Control": "no-cache",
 "Upgrade-Insecure-Requests": "1",
 "User-Agent":
 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36",
 "Sec-Fetch-User": "?1",
 Accept:
 "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
 "Sec-Fetch-Site": "none",
 "Sec-Fetch-Mode": "navigate",
 "Sec-Fetch-Dest": "document",
 "Accept-Encoding": "gzip, deflate, br",
 "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
 Cookie: cookieString,
 }
 }

 /**
 * Detects the Cloudflare challenge page by 503 status code and the CDATA tag;
 * @param response
 * @return {boolean}
 * @private
 */
 _detectChallengeFunction (response) {
 return response.statusCode === 503 && response.body.includes("CDATA")
 }

 /**
 * Detects Challenge request
 * @param request {Request} - Puppeteer request
 * @return {boolean} - true if challenge request detected
 * @private
 */
 _detectChallengeRequestFunction (request) {
 return request.method() === "POST"
 }

 /**
 * Waits until the challenge is computed.
 * @return {Promise<void>}
 * @private
 */
 async _waitUntilChallengeFinishedFunction () {
 await Apify.utils.sleep(5000)
 }

 /**
 * Checks if challenge is successfully solved.
 * @param challengeResponse
 * @return {boolean}
 * @private
 */
 _isChallengeSolvedFunction (challengeResponse) {
 return challengeResponse.statusCode === 200
 }

 /**
 * Gets request options - these options are compatible with `Apify.utils.requestAsBrowser` and `@apify/http-request`
 * @param session {Session} - Session instance
 * @return {Object} - Contains request options
 */
 getRequestOptions (session) {
 const proxyUrl = this._getProxyUrl(session)

 const requestOptions = {
 headers: this._getBrowserHeaders(
 session.getCookieString(this.unblockUrl),
 ),
 proxyUrl,
 }

 if (this._shouldUseCustomTLS) {
 // @ts-ignore
 if (!requestOptions.https) requestOptions.https = {}
 requestOptions.https.ciphers = "AES256-SHA"
 }

 return requestOptions
 }

 /**
 * Creates new Session instance for the SessionPool.
 * @param sessionPool {SessionPool}.
 * @return {Promise<Session>}
 */
 async createSessionFunction (sessionPool) {
 const session = new Session({ sessionPool })
 try {
 await this.unblock({ session, request: { url: this.unblockUrl } })
 } catch (e) {
 log.warning(`${this.name}: Could not unblock session`)
 log.exception(e, 'Unblocker failed')
 }
 return session
 }

 /**
 * Gets options for the CheerioCrawler use interface.
 * @return {Object} - CheerioCrawler options
 */
 getCrawlerOptions () {
 const that = this
 return {
 useSessionPool: true,
 sessionPoolOptions: {
 maxPoolSize: 100,
 createSessionFunction: this.createSessionFunction.bind(this),
 },
 persistCookiesPerSession: true,
 useApifyProxy: true,
 async prepareRequestFunction ({ request, session }) {
 const newOptions = that.getRequestOptions(session)
 request.headers = newOptions.headers
 // @ts-ignore
 if (newOptions.ciphers) {
 // @ts-ignore
 this.requestOptions = { ciphers: newOptions.ciphers }
 }
 },
 }
 }
}

import { createHash } from 'crypto';

// inspired by @drobnikj
const createUniqueKeyFromUrl = (url) => {
  const hash = createHash('sha256');
  const cleanUrl = url.split('://')[1]; // Remove protocol
  hash.update(cleanUrl);
  return hash.digest('hex');
};


// inspired by hlidac-shopu
import Apify from "apify";
import { defAtom } from "@thi.ng/atom";

// TODO: make this lowes common denominator
const defaultStats = {
 urls: 0,
 items: 0,
 itemsDuplicate: 0,
 totalItems: 0,
 denied: 0,
 ok: 0
};

const inc = x => x + 1;
const dec = x => x - 1;

// TODO: stats should be in atom and updated via swap function atomically
class Stats {
 constructor(init) {
 this.stats = defAtom(init);
 this.interval = setInterval(() => this.log(), 20 * 1000);
 }

 inc(key) {
 this.stats.swapIn(key, inc);
 }

 dec(key) {
 this.stats.swapIn(key, dec);
 }

 add(key, value) {
 this.stats.swapIn(key, x => x + value);
 }

 get() {
 return this.stats.deref();
 }

 log() {
 const stats = this.stats.deref();
 Apify.utils.log.info(`stats: ${JSON.stringify(stats)}`);
 }

 /**
 * @param final {boolean} - If true, clearInterval apply
 */
 async save(final = false) {
 if (final) {
 clearInterval(this.interval);
 }
 const stats = this.stats.deref();
 await Apify.setValue("STATS", this.get());
 Apify.utils.log.info("STATS saved!");
 if (stats.ok) {
 Apify.utils.log.info(
 `Denied ratio: ${(stats.denied ?? 0 / stats.ok) * 100} %`
 );
 }
 this.log();
 }
}

/**
 *
 * @param {function} fn
 * @param {*} init
 * @returns {Promise<Stats>}
 */
export async function withPersistedStats(fn, init) {
 const stats = (await Apify.getValue("STATS")) ?? init ?? defaultStats;
 const state = new Stats(fn(stats));
 const persistState = () => state.save();

 Apify.events.on("persistState", persistState);
 Apify.events.on("migrating", persistState);

 return state;
}

Bike24 (bike24.de) scraper

Output example

Industries

Where next?