FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
 &amp;&amp; npm install --only=prod --no-optional

COPY . ./

Dockerfile

{
 "title": "BikerBoarder (biker-boarder.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "type": "object",
 "schemaVersion": 1,
 "properties": {
 "mode": {
 "title": "Mode",
 "description": "",
 "type": "string",
 "editor": "select",
 "default": "FULL",
 "prefill": "FULL",
 "enumTitles": [
 "FULL",
 "SINGLE"
 ],
 "enum": [
 "FULL",
 "SINGLE"
 ]
 },
 "debug": {
 "title": "Debug",
 "description": "Debug mode prints more logs, disables concurrency and other optimizations.",
 "type": "boolean",
 "editor": "checkbox",
 "default": false
 }
 },
 "required": [
 "mode"
 ]
}

INPUT_SCHEMA.json

# BikerBoarder (biker-boarder.de) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

## Output example

* **itemId** `string` e.g. *226707*
* **itemName** `string` e.g. *Evoc Line 28l, heather ruby*
* **itemUrl** `string` e.g. *https://www.biker-boarder.de/evoc/1942902_pa.html*
* **img** `string`
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *115.90*
* **originalPrice** `number` e.g. *145.00*
* **currency** `string` e.g. *EUR*

README.md

{
 "name": "bikerboarder-biker-boarder-de-scraper",
 "version": "0.1",
 "buildTag": "latest",
 "env": null,
 "defaultRunOptions": {
 "build": "latest",
 "timeoutSecs": 3600,
 "memoryMbytes": 1024
 }
}

apify.json

main.js

{
 "name": "bikerboarder-biker-boarder-de-scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "type": "module",
 "scripts": {
 "start": "node ./main.js",
 "push-to-apify-platform": "npx apify push"
 },
 "dependencies": {
 "apify": "*",
 "@thi.ng/atom": "*"
 },
 "apify": {
 "title": "BikerBoarder (biker-boarder.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "isPublic": true,
 "isDeprecated": false,
 "issuesEnabled": true,
 "isAnonymouslyRunnable": true,
 "notice": "",
 "pictureUrl": "",
 "seoTitle": "",
 "seoDescription": "",
 "categories": [
 "ECOMMERCE"
 ]
 }
}

package.json

{
 "actorSpecification": 1,
 "name": "bikerboarder-biker-boarder-de-scraper",
 "title": "BikerBoarder (biker-boarder.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "version": "0.1.0",
 "storages": {
 "dataset": {
 "actorSpecification": 1,
 "title": "BikerBoarder (biker-boarder.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "views": {
 "overview": {
 "title": "Overview",
 "description": "Overview of the most important fields",
 "transformation": {
 "fields": [
 "itemId",
 "itemName",
 "itemUrl",
 "img",
 "inStock",
 "currentPrice",
 "originalPrice",
 "currency"
 ]
 },
 "display": {
 "component": "table",
 "columns": [
 {
 "label": "Item ID",
 "field": "itemUrl",
 "format": "link",
 "textField": "itemId"
 },
 {
 "label": "Item Name",
 "field": "itemName",
 "format": "text"
 },
 {
 "label": "Img",
 "field": "img",
 "format": "image"
 },
 {
 "label": "In Stock",
 "field": "inStock",
 "format": "boolean"
 },
 {
 "label": "Current Price",
 "field": "currentPrice",
 "format": "number"
 },
 {
 "label": "Original Price",
 "field": "originalPrice",
 "format": "number"
 },
 {
 "label": "Currency",
 "field": "currency",
 "format": "text"
 }
 ]
 }
 }
 }
 }
 }
}

.actor/actor.json

// inspired by hlidac-shopu
import Apify from "apify";
import { defAtom } from "@thi.ng/atom";

// TODO: make this lowes common denominator
const defaultStats = {
 urls: 0,
 items: 0,
 itemsDuplicate: 0,
 totalItems: 0,
 denied: 0,
 ok: 0
};

const inc = x => x + 1;
const dec = x => x - 1;

// TODO: stats should be in atom and updated via swap function atomically
class Stats {
 constructor(init) {
 this.stats = defAtom(init);
 this.interval = setInterval(() => this.log(), 20 * 1000);
 }

 inc(key) {
 this.stats.swapIn(key, inc);
 }

 dec(key) {
 this.stats.swapIn(key, dec);
 }

 add(key, value) {
 this.stats.swapIn(key, x => x + value);
 }

 get() {
 return this.stats.deref();
 }

 log() {
 const stats = this.stats.deref();
 Apify.utils.log.info(`stats: ${JSON.stringify(stats)}`);
 }

 /**
 * @param final {boolean} - If true, clearInterval apply
 */
 async save(final = false) {
 if (final) {
 clearInterval(this.interval);
 }
 const stats = this.stats.deref();
 await Apify.setValue("STATS", this.get());
 Apify.utils.log.info("STATS saved!");
 if (stats.ok) {
 Apify.utils.log.info(
 `Denied ratio: ${(stats.denied ?? 0 / stats.ok) * 100} %`
 );
 }
 this.log();
 }
}

/**
 *
 * @param {function} fn
 * @param {*} init
 * @returns {Promise&lt;Stats>}
 */
export async function withPersistedStats(fn, init) {
 const stats = (await Apify.getValue("STATS")) ?? init ?? defaultStats;
 const state = new Stats(fn(stats));
 const persistState = () => state.save();

 Apify.events.on("persistState", persistState);
 Apify.events.on("migrating", persistState);

 return state;
}

_utils/stats.js

FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
  && npm install --only=prod --no-optional

COPY . ./

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

Debug mode prints more logs, disables concurrency and other optimizations.

Debug

Mode

BikerBoarder (biker-boarder.de) scraper

# BikerBoarder (biker-boarder.de) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

## Output example

* **itemId** `string` e.g. *226707*
* **itemName** `string` e.g. *Evoc Line 28l, heather ruby*
* **itemUrl** `string` e.g. *https://www.biker-boarder.de/evoc/1942902_pa.html*
* **img** `string`
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *115.90*
* **originalPrice** `number` e.g. *145.00*
* **currency** `string` e.g. *EUR*

bikerboarder-biker-boarder-de-scraper

/**
 * Dev notes
 * ===
 *
 * original price is not available in listing page,
 * so we have to calculate it from the price and discount percentage
 *
 */

import { URL } from "node:url";
import Apify from "apify";

const { log } = Apify.utils;

const LABELS = {
 INDEX: `INDEX`,
 PRODUCTS: `PRODUCTS`,
};

var MODE;

(function (MODE) {
 MODE["FULL"] = "FULL";
 MODE["SINGLE"] = "SINGLE";
})(MODE || (MODE = {}));

const BASE_URL = `https://www.biker-boarder.de`;

async function enqueueInitialRequest(mode, requestQueue) {
 if (mode === MODE.FULL) {
 await requestQueue.addRequest({
 userData: { label: LABELS.INDEX },
 url: `${BASE_URL}/brands`,
 });
 } else if (mode === MODE.SINGLE) {
 await requestQueue.addRequest({
 userData: { label: LABELS.PRODUCTS },
 url: `${BASE_URL}/evoc`,
 });
 }
}

async function handleIndex({ $ }, { requestQueue }) {
 $(`.brand__list a.brand`).each((i, el) => {
 const relUrl = $(el).attr(`href`); // urls are relative
 const url = `${BASE_URL}${relUrl}`;
 const name = $(el).attr(`title`);
 void requestQueue.addRequest({
 userData: { label: LABELS.PRODUCTS, category: name },
 url,
 });
 });
}

async function handleProducts({ $, request }, { processedIds, requestQueue }) {
 if (!request.url.includes(`?page=`)) {
 // on first page
 const totalPages = $(`.productlist__head .pagination .page`).last().text(); // e.g. `8`
 for (let i = 2; i <= totalPages; i++) {
 // skip first page, that is already handled
 const url = new URL(request.url);
 url.searchParams.set(`page`, i.toString()); // toString() to make TS happy
 void requestQueue.addRequest({
 userData: { label: LABELS.PRODUCTS },
 url: url.toString(),
 });
 }
 }

 $(`.articlebox__list .articlebox`).each((i, el) => {
 const id = $(el).attr(`id`).replace(`pr-`, ``); // e.g. 226707
 if (!processedIds.has(id)) {
 processedIds.add(id);
 const relUrl = $(el).children(`a`).attr(`href`); // relative url
 const url = `${BASE_URL}${relUrl}`;
 const title = $(el).find(`h4`).text().trim();
 const priceRaw = $(el).find(`.price .number`).text().trim(); // e.g. `ab €28,90`
 const price = priceRaw.replace(/[^0-9,]/g, ``).replace(`,`, `.`); // keep comma as it's decimal separator
 let priceOrig = price;
 let discount = 0;
 const discountRaw = $(el).find(`.price .discount`).text().trim(); // e.g. `-42%`
 if (discountRaw) {
 discount = discountRaw.replace(/[^0-9,]/g, ``); // keep comma as it's decimal separator
 priceOrig = (price / (1 - discount / 100)).toFixed(2);
 }
 const img = $(el).find(`img.defaultimg`).attr(`src`);
 const inStock = true;
 const product = {
 itemId: id,
 itemName: title,
 itemUrl: url,
 img: img,
 inStock,
 currentPrice: price,
 originalPrice: priceOrig,
 currency: `EUR`,

 // Derived
 discounted: priceOrig && priceOrig !== price,
 _discount: discount / 100,
 };
 Apify.pushData(product);
 }
 });
}

Apify.main(async () => {
 const processedIds = new Set();

 const input = await Apify.getInput();
 const { debug = false, mode = MODE.FULL } = input ?? {};

 if (debug) Apify.utils.log.setLevel(Apify.utils.log.LEVELS.DEBUG);

 const requestQueue = await Apify.openRequestQueue();
 await enqueueInitialRequest(mode, requestQueue);

 const globalContext = { mode, requestQueue, processedIds };
 const crawler = new Apify.CheerioCrawler({
 requestQueue,
 maxConcurrency: debug ? 1 : 3,
 maxRequestRetries: debug ? 0 : 3,
 async handlePageFunction(context) {
 const { request, response } = context;
 const { userData } = request;
 // @ts-ignore
 const { statusCode, body } = response;

 if (statusCode !== 200) return log.info(body.toString());

 switch (userData.label) {
 case LABELS.INDEX:
 return await handleIndex(context, globalContext);
 case LABELS.PRODUCTS:
 return await handleProducts(context, globalContext);
 }
 },
 async handleFailedRequestFunction({ request }) {
 log.error(`Request ${request.url} failed multiple times`, request);
 },
 });

 await crawler.run();
 log.info(`crawler finished`);
});

// inspired by hlidac-shopu
import Apify from "apify";
import { defAtom } from "@thi.ng/atom";

// TODO: make this lowes common denominator
const defaultStats = {
 urls: 0,
 items: 0,
 itemsDuplicate: 0,
 totalItems: 0,
 denied: 0,
 ok: 0
};

const inc = x => x + 1;
const dec = x => x - 1;

// TODO: stats should be in atom and updated via swap function atomically
class Stats {
 constructor(init) {
 this.stats = defAtom(init);
 this.interval = setInterval(() => this.log(), 20 * 1000);
 }

 inc(key) {
 this.stats.swapIn(key, inc);
 }

 dec(key) {
 this.stats.swapIn(key, dec);
 }

 add(key, value) {
 this.stats.swapIn(key, x => x + value);
 }

 get() {
 return this.stats.deref();
 }

 log() {
 const stats = this.stats.deref();
 Apify.utils.log.info(`stats: ${JSON.stringify(stats)}`);
 }

 /**
 * @param final {boolean} - If true, clearInterval apply
 */
 async save(final = false) {
 if (final) {
 clearInterval(this.interval);
 }
 const stats = this.stats.deref();
 await Apify.setValue("STATS", this.get());
 Apify.utils.log.info("STATS saved!");
 if (stats.ok) {
 Apify.utils.log.info(
 `Denied ratio: ${(stats.denied ?? 0 / stats.ok) * 100} %`
 );
 }
 this.log();
 }
}

/**
 *
 * @param {function} fn
 * @param {*} init
 * @returns {Promise<Stats>}
 */
export async function withPersistedStats(fn, init) {
 const stats = (await Apify.getValue("STATS")) ?? init ?? defaultStats;
 const state = new Stats(fn(stats));
 const persistState = () => state.save();

 Apify.events.on("persistState", persistState);
 Apify.events.on("migrating", persistState);

 return state;
}

BikerBoarder (biker-boarder.de) scraper

Output example

Industries

E-commerce & Retail

Where next?