FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
 &amp;&amp; npm install --only=prod --no-optional

COPY . ./

Dockerfile

{
 "title": "BikerBoarder (biker-boarder.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "type": "object",
 "schemaVersion": 1,
 "properties": {
 "mode": {
 "title": "Mode",
 "description": "",
 "type": "string",
 "editor": "select",
 "default": "TEST",
 "prefill": "TEST",
 "enum": [
 "TEST",
 "FULL"
 ],
 "enumTitles": [
 "TEST mode (scrapes only \"Evoc\" &amp; \"Fox\" brands)",
 "FULL"
 ]
 },
 "APIFY_DONT_STORE_IN_DATASET": {
 "sectionCaption": "Advanced",
 "sectionDescription": "Advanced options, use only if you know what you're doing.",
 "title": "Don't store in dataset",
 "description": "If set to true, the actor will not store the results in the default dataset. Useful when using alternative storage, like own database",
 "type": "boolean",
 "default": false,
 "editor": "checkbox"
 },
 "PG_CONNECTION_STRING_NORMALIZED": {
 "title": "Postgres connection string for normalized data",
 "description": "If set, actor will store normalized data in Postgres database in PG_DATA_TABLE and PG_DATA_PRICE_TABLE tables",
 "type": "string",
 "editor": "textfield"
 },
 "PG_DATA_TABLE": {
 "title": "Postgres table name for product data",
 "description": "Table name for storing product name, url, image, ...",
 "type": "string",
 "editor": "textfield"
 },
 "PG_DATA_PRICE_TABLE": {
 "title": "Postgres table name for price data",
 "description": "Table name for storing price, original price, stock status, ...",
 "type": "string",
 "editor": "textfield"
 }
 },
 "required": [
 "mode"
 ]
}

INPUT_SCHEMA.json

# BikerBoarder (biker-boarder.de) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

## Output example

* **pid** `string` e.g. *226707*
* **name** `string` e.g. *Evoc Line 28l, heather ruby*
* **url** `string` e.g. *https://www.biker-boarder.de/evoc/1942902_pa.html*
* **img** `string`
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *115.90*
* **originalPrice** `number` e.g. *145.00*
* **currency** `string` e.g. *EUR*

README.md

{
 "name": "bikerboarder-biker-boarder-de-scraper",
 "version": "0.1",
 "buildTag": "latest",
 "env": null,
 "defaultRunOptions": {
 "build": "latest",
 "timeoutSecs": 3600,
 "memoryMbytes": 1024
 }
}

apify.json

main.js

{
 "name": "bikerboarder-biker-boarder-de-scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "type": "module",
 "scripts": {
 "start": "node ./main.js",
 "push-to-apify-platform": "npx apify push"
 },
 "dependencies": {
 "apify3": "npm:apify@^3.0.2",
 "crawlee": "*",
 "pg": "*",
 "pg-connection-string": "*",
 "dotenv": "*",
 "find-config": "*",
 "@elastic/elasticsearch": "*",
 "filenamify": "*"
 },
 "apify": {
 "title": "BikerBoarder (biker-boarder.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "isPublic": true,
 "isDeprecated": false,
 "isAnonymouslyRunnable": true,
 "notice": "",
 "pictureUrl": "",
 "seoTitle": "",
 "seoDescription": "",
 "categories": [
 "ECOMMERCE"
 ]
 }
}

package.json

{
 "actorSpecification": 1,
 "name": "bikerboarder-biker-boarder-de-scraper",
 "title": "BikerBoarder (biker-boarder.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "version": "0.1.0",
 "storages": {
 "dataset": {
 "actorSpecification": 1,
 "title": "BikerBoarder (biker-boarder.de) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
 "views": {
 "overview": {
 "title": "Overview",
 "description": "Overview of the most important fields",
 "transformation": {
 "fields": [
 "pid",
 "name",
 "url",
 "img",
 "inStock",
 "currentPrice",
 "originalPrice",
 "currency"
 ]
 },
 "display": {
 "component": "table",
 "columns": [
 {
 "label": "Pid",
 "field": "pid",
 "format": "text"
 },
 {
 "label": "Name",
 "field": "name",
 "format": "text"
 },
 {
 "label": "Url",
 "field": "url",
 "format": "link"
 },
 {
 "label": "Img",
 "field": "img",
 "format": "image"
 },
 {
 "label": "In Stock",
 "field": "inStock",
 "format": "boolean"
 },
 {
 "label": "Current Price",
 "field": "currentPrice",
 "format": "number"
 },
 {
 "label": "Original Price",
 "field": "originalPrice",
 "format": "number"
 },
 {
 "label": "Currency",
 "field": "currency",
 "format": "text"
 }
 ]
 }
 }
 }
 }
 }
}

.actor/actor.json

_utils/common.js

FROM apify/actor-node:16

COPY package.json ./

RUN npm --quiet set progress=false \
  && npm install --only=prod --no-optional

COPY . ./

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

If set to true, the actor will not store the results in the default dataset. Useful when using alternative storage, like own database

Advanced options, use only if you know what you're doing.

Don't store in dataset

If set, actor will store normalized data in Postgres database in PG_DATA_TABLE and PG_DATA_PRICE_TABLE tables

Postgres connection string for normalized data

Table name for storing price, original price, stock status, ...

Postgres table name for price data

Table name for storing product name, url, image, ...

Postgres table name for product data

Mode

BikerBoarder (biker-boarder.de) scraper

# BikerBoarder (biker-boarder.de) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details.

## Output example

* **pid** `string` e.g. *226707*
* **name** `string` e.g. *Evoc Line 28l, heather ruby*
* **url** `string` e.g. *https://www.biker-boarder.de/evoc/1942902_pa.html*
* **img** `string`
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *115.90*
* **originalPrice** `number` e.g. *145.00*
* **currency** `string` e.g. *EUR*

bikerboarder-biker-boarder-de-scraper

/**
 * Dev notes
 * ===
 * original price is not available in listing page,
 * so we have to calculate it from the price and discount percentage
 */

import { URL } from "node:url";
import { Actor } from "apify3";
import { CheerioCrawler, createCheerioRouter } from "crawlee";
import { init, save } from "./_utils/common.js";

const LABELS = {
 INDEX: `INDEX`,
 PRODUCTS: `PRODUCTS`,
};

var MODE;

(function (MODE) {
 MODE["TEST"] = "TEST";
 MODE["FULL"] = "FULL";
})(MODE || (MODE = {}));

const BASE_URL = `https://www.biker-boarder.de`;

async function enqueueInitial(mode, crawler) {
 if (mode === MODE.FULL) {
 await crawler.addRequests([
 {
 userData: { label: LABELS.INDEX },
 url: `${BASE_URL}/brands`,
 },
 ]);
 } else if (mode === MODE.TEST) {
 await crawler.addRequests([
 {
 userData: { label: LABELS.PRODUCTS },
 url: `${BASE_URL}/evoc`,
 },
 ]);
 await crawler.addRequests([
 {
 userData: { label: LABELS.PRODUCTS },
 url: `${BASE_URL}/fox`,
 },
 ]);
 }
}

async function handleIndex({ $, crawler }) {
 const requests = [];
 $(`.brand__list a.brand`).each((i, el) => {
 const relUrl = $(el).attr(`href`); // urls are relative
 const url = `${BASE_URL}${relUrl}`;
 const name = $(el).attr(`title`);
 requests.push({
 userData: { label: LABELS.PRODUCTS, category: name },
 url,
 });
 });
 await crawler.addRequests(requests);
}

async function handleProducts({ $, request, crawler }) {
 if (!request.url.includes(`?page=`)) {
 // on first page
 const totalPages = $(`.productlist__head .pagination .page`).last().text(); // e.g. `8`
 for (let i = 2; i <= totalPages; i++) {
 // skip first page, that is already handled
 const url = new URL(request.url);
 url.searchParams.set(`page`, i.toString()); // toString() to make TS happy
 void crawler.addRequests([
 {
 userData: { label: LABELS.PRODUCTS },
 url: url.toString(),
 },
 ]);
 }
 }

 const products = [];
 $(`.articlebox__list .articlebox`).each((i, el) => {
 const id = $(el).attr(`id`).replace(`pr-`, ``); // e.g. 226707
 const relUrl = $(el).children(`a`).attr(`href`); // relative url
 const url = `${BASE_URL}${relUrl}`;
 const title = $(el).find(`h4`).text().trim();
 const priceRaw = $(el).find(`.price .number`).text().trim(); // e.g. `ab €28,90`
 const price = priceRaw.replace(/[^\d,]/g, ``).replace(`,`, `.`); // keep comma as it's decimal separator
 let priceOrig = price;
 let discount = 0;
 const discountRaw = $(el).find(`.price .discount`).text().trim(); // e.g. `-42%`
 if (discountRaw) {
 discount = discountRaw.replace(/[^\d,]/g, ``); // keep comma as it's decimal separator
 priceOrig = (price / (1 - discount / 100)).toFixed(2);
 }
 const img = $(el).find(`img.defaultimg`).attr(`src`);
 const inStock = true;
 const product = {
 pid: id,
 name: title,
 url: url,
 img: img,
 inStock,
 currentPrice: price,
 originalPrice: priceOrig,
 currency: `EUR`,
 };
 products.push(product);
 });
 void save(products);
}

const router = createCheerioRouter();
router.addHandler(LABELS.INDEX, handleIndex);
router.addHandler(LABELS.PRODUCTS, handleProducts);

void Actor.main(async () => {
 const input = await Actor.getInput();
 const { mode = MODE.FULL, ...rest } = input ?? {};
 await init({ actorNameOverride: `biker-boarder-de` }, rest);
 const crawler = new CheerioCrawler({ requestHandler: router });
 await enqueueInitial(mode, crawler);
 await crawler.run();
});

import { createHash } from 'crypto'
import os from "os"
import path from "path"
// eslint-disable-next-line @apify/apify-actor/no-forbidden-node-internals
import fs from "fs"
import pg from "pg"
import pgConnectionString from 'pg-connection-string'
import { config } from 'dotenv'
import findConfig from "find-config"
import { Client as ElasticClient } from "@elastic/elasticsearch"
import filenamify from 'filenamify'
import { Dataset } from 'crawlee'

config({ path: findConfig(`.env`) })

const elasticIndexName = `actors-monorepo-shops`

const globalLogsProps = {
 __NODE_STARTED: new Date().toISOString(),
}

let actorName
let pgClient
let pgClientNormalized
let elasticClient
export async function init ({ actorNameOverride }, restInput) {
 parseEnvFromInput(restInput)

 if (os.platform() === `darwin`) {
 const filePath = process.argv[1] // ~/Projects/apify-actors-monorepo/actors/foo.ts
 const basename = path.basename(filePath) // foo.ts
 actorName = actorNameOverride ?? basename.split(`.`)[0] // foo
 const gitBranch = fs.readFileSync(path.join(process.cwd(), `..`, `.git/HEAD`), `utf8`)
 .split(` `)[1]
 .trim()
 .replace(`refs/heads/`, ``)
 const gitCommit = fs.readFileSync(path.join(process.cwd(), `..`, `.git/refs/heads/${gitBranch}`), `utf8`)
 const gitCommitShort = gitCommit.substring(0, 7)
 globalLogsProps.__GIT_COMMIT = gitCommitShort
 }

 if (process.env.APIFY_IS_AT_HOME) {
 actorName = actorNameOverride ?? process.env.APIFY_ACTOR_ID // Name would be better, but it's not in ENV
 }

 /* ELASTIC */
 /* ======= */
 if (process.env.ELASTIC_CLOUD_ID) {
 elasticClient = new ElasticClient({
 cloud: { id: process.env.ELASTIC_CLOUD_ID },
 auth: { apiKey: process.env.ELASTIC_CLOUD_API_KEY },
 })

 // const mapping = await elasticClient.indices.getMapping({ index: actorName })

 // eslint-disable-next-line no-inner-declarations
 async function enforceIndexMapping () {
 const doesIndexExist = await elasticClient.indices.exists({ index: elasticIndexName })
 if (!doesIndexExist) await elasticClient.indices.create({ index: elasticIndexName })
 await elasticClient.indices.putMapping({
 index: elasticIndexName,
 body: {
 properties: {
 _discount: { type: `float` },
 originalPrice: { type: `float` },
 currentPrice: { type: `float` },
 },
 },
 })
 }

 try {
 await enforceIndexMapping()
 } catch (err) {
 if (err.message.includes(`cannot be changed from type`)) {
 console.log(`Elastic index ${elasticIndexName} already exists with incorrect mappings. As existing mapping cannot be changed, index will be deleted and recreated.`)
 await elasticClient.indices.delete({ index: elasticIndexName })
 await enforceIndexMapping()
 }
 }
 }

 /* POSTGRESQL */
 /* ========== */
 if (process.env.PG_CONNECTION_STRING) {
 const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING)
 // const pgPool = new pg.Pool(pgConfig)

 pgClient = new pg.Client(pgConfig)
 await pgClient.connect()

 // Check if table exists and have proper columns
 const { rows: tables } = await pgClient.query(`
 SELECT table_name
 FROM information_schema.tables
 WHERE table_schema = 'public'
 `)

 // eslint-disable-next-line camelcase
 const tableExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
 if (!tableExists) {
 throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
 }

 // TODO: Handle pgClient closing
 }

 if (process.env.PG_CONNECTION_STRING_NORMALIZED) {
 const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING_NORMALIZED)

 pgClientNormalized = new pg.Client(pgConfig)
 await pgClientNormalized.connect()

 // Check if table exists and have proper columns
 const { rows: tables } = await pgClientNormalized.query(`
 SELECT table_name
 FROM information_schema.tables
 WHERE table_schema = 'public'
 `)

 // eslint-disable-next-line camelcase
 const tableMainExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
 // eslint-disable-next-line camelcase
 const tablePricesExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_PRICE_TABLE)
 if (!tableMainExists) throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
 if (!tablePricesExists) throw new Error(`Table ${process.env.PG_DATA_PRICE_TABLE} does not exist in database ${pgConfig.database}`)

 // TODO: Handle pgClient closing
 }
}

// inspired by @drobnikj
// TODO: Similar, but less obfuscated for easier debugging
export const createUniqueKeyFromUrl = (url) => {
 const hash = createHash(`sha256`)
 const cleanUrl = url.split(`://`)[1] // Remove protocol
 hash.update(cleanUrl)
 return hash.digest(`hex`)
}

/**
 *
 * @param {Date} datetime
 * @return {Promise<void>}
 */
export const sleepUntil = async (datetime) => {
 const now = new Date()
 const difference = datetime - now
 if (difference > 0) {
 return new Promise((resolve) => {
 setTimeout(resolve, difference)
 })
 }
 return Promise.resolve()
}

export function parsePrice (string) {
 let amount, currency
 const noText = string.replace(/[^\d,.]/g, ``)
 const decimals = noText.match(/([,.])(\d{2})$/)
 if (decimals) {
 const decimalSeparator = decimals[1]
 // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
 const decimalAmount = decimals[2]
 amount = parseInt(noText.split(decimalSeparator)[0])
 } {
 const justNumbers = noText.replace(/[,.]/g, ``)
 amount = parseInt(justNumbers)
 }
 return { amount, currency }
}

export function toNumberOrNull (str) {
 // TODO: Handle better, but only after adding test
 if (str === undefined) return null
 if (str === null) return null
 if (str === ``) return null
 const num = Number(str)
 if (Number.isNaN(num)) return null
 return num
}

export async function save (objs) {
 if (!Array.isArray(objs)) objs = [objs]
 if (objs.length === 0) return

 const objsExtended = objs.map((obj) => {
 const objExtended = {
 ...obj,
 actorName,
 ...globalLogsProps,
 // __NODE_VERSION: global.process.versions.node,
 // __NODE_UPTIME: global.process.uptime().toFixed(2), // seconds, 2 decimals
 }
 // if run on Apify
 if (process.env.APIFY_IS_AT_HOME) {
 objExtended.__APIFY_ACTOR_ID = process.env.APIFY_ACTOR_ID
 objExtended.__APIFY_ACTOR_RUN_ID = process.env.APIFY_ACTOR_RUN_ID
 objExtended.__APIFY_ACTOR_BUILD_ID = process.env.APIFY_ACTOR_BUILD_ID
 objExtended.__APIFY_ACTOR_BUILD_NUMBER = process.env.APIFY_ACTOR_BUILD_NUMBER
 objExtended.__APIFY_ACTOR_TASK_ID = process.env.APIFY_ACTOR_TASK_ID
 if (!process.env.APIFY_DONT_STORE_IN_DATASET) void Dataset.pushData(obj)
 }
 return objExtended
 })
 // if runs on local machine (MacOS)
 if (os.platform() === `darwin`) {
 const cwd = process.cwd() // ~/Projects/apify-actors-monorepo/actors
 const storageDir = path.join(cwd, `${actorName}.storage`) // ~/Projects/apify-actors-monorepo/actors/foo.storage
 if (!fs.existsSync(storageDir)) fs.mkdirSync(storageDir)
 const dataDir = path.join(storageDir, `data`) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data
 if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir)
 for (const objExtended of objsExtended) {
 const id = objExtended.id ?? objExtended.pid // ?? uuidv4()
 const fileName = `${filenamify(id)}.json`
 const dataFilePath = path.join(dataDir, fileName) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data/foo.json
 fs.writeFileSync(dataFilePath, JSON.stringify(objExtended, null, 2))
 }
 }

 if (pgClient) {
 const objsPg = objs.map((obj) => ({
 ...obj,
 // TODO: This is becoming not nice, and not clear
 shop: actorName,
 scrapedAt: new Date().toISOString().split(`T`)[0],
 }))

 const columns = getColumns(objsPg)
 const values = getValues(objsPg)
 const queryString = `
 INSERT INTO public."${process.env.PG_DATA_TABLE}" (${columns})
 VALUES (${values})
 `
 try {
 const { rowCount } = await pgClient.query(queryString)
 console.log(`[save] saved to database: ${JSON.stringify(rowCount)}`)
 } catch (err) {
 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
 else throw err
 }
 }

 // Only make sense for HlidacShopu
 if (pgClientNormalized) {
 const objsPgData = objs.map((obj) => ({
 shop: actorName,
 pid: obj.pid,
 name: obj.name,
 url: obj.url,
 img: obj.img,
 }))

 const objsPgDataPrice = objs.map((obj) => ({
 shop: actorName,
 pid: obj.pid,
 scrapedAt: new Date().toISOString().split(`T`)[0],
 currentPrice: obj.currentPrice,
 originalPrice: obj.originalPrice,
 inStock: obj.inStock,
 }))

 const queryString = `
 INSERT INTO public."${process.env.PG_DATA_TABLE}" (${getColumns(objsPgData)})
 VALUES (${getValues(objsPgData)})
 ON CONFLICT DO NOTHING
 `
 try {
 const { rowCount } = await pgClientNormalized.query(queryString)
 console.log(`[save] saved to database (data): ${JSON.stringify(rowCount)}`)
 } catch (err) {
 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
 else throw err
 }

 const queryStringPrice = `
 INSERT INTO public."${process.env.PG_DATA_PRICE_TABLE}" (${getColumns(objsPgDataPrice)})
 VALUES (${getValues(objsPgDataPrice)})
 ON CONFLICT DO NOTHING
 `
 try {
 const { rowCount } = await pgClientNormalized.query(queryStringPrice)
 console.log(`[save] saved to database (price): ${JSON.stringify(rowCount)}`)
 } catch (err) {
 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
 else throw err
 }
 }

 if (elasticClient) {
 // .index creates or updates the document
 // .create creates a new document if it doesn't exist, 409 if it does
 // try {
 // const res = await elasticClient.index({
 // index: `actors-monorepo-shops`, // TODO: Consider using actorName
 // id, // foo-bar
 // document: objExtended, // {...}
 // })
 // } catch (err) {
 // // https://discuss.elastic.co/t/elasticsearch-503-ok-false-message-the-requested-deployment-is-currently-unavailable/200583
 // if (err.message.includes(`requested resource is currently unavailable`)) console.log(`Elasticsearch is unavailable, skipping, but not aborting`)
 // else throw err
 // }
 }
}

function getColumns (objs) {
 return Object.keys(objs[0]).map((key) => `"${key}"`).join(`, `)
}

function getValues (objs) {
 return objs.map(objPg => Object.values(objPg).map((value) => {
 // escape strings to prevent SQL injection
 if (typeof value === `string`) return `'${value.replace(/'/g, `''`)}'`
 // convert to DB specific null
 if (typeof value === `undefined` || value === null) return `NULL`
 return value
 }).join(`, `)).join(`), (`)
}

export function parseEnvFromInput (input) {
 const env = {}
 for (const key in input) {
 if (key === key.toUpperCase()) env[key] = input[key]
 }
 console.log(`[parseEnvFromInput] ${JSON.stringify(env)}`)
 Object.assign(process.env, env)
}

Mode

Don't store in dataset

Postgres connection string for normalized data

Postgres table name for product data

Postgres table name for price data

Where next?