FROM apify/actor-node-playwright-firefox:16

COPY package.json ./

RUN npm --quiet set progress=false \
 &amp;&amp; npm install aws-crt \
 &amp;&amp; npm install --only=prod --no-optional

COPY . ./

Dockerfile

{
 "title": "r2-bike (r2-bike.com) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
 "type": "object",
 "schemaVersion": 1,
 "properties": {
 "mode": {
 "title": "Mode",
 "description": "",
 "type": "string",
 "editor": "select",
 "default": "TEST",
 "prefill": "TEST",
 "enum": [
 "TEST",
 "FULL"
 ],
 "enumTitles": [
 "TEST",
 "FULL"
 ]
 },
 "APIFY_USE_MEMORY_REQUEST_QUEUE": {
 "sectionCaption": "Advanced",
 "sectionDescription": "Advanced options, use only if you know what you're doing.",
 "title": "Use in-memory request queue instead of the native one",
 "description": "In-memory request queue can reduce costs, but it may case issues with longer runs due to non-persistence.",
 "type": "boolean",
 "default": false,
 "editor": "checkbox"
 },
 "APIFY_DONT_STORE_IN_DATASET": {
 "title": "Don't store in dataset",
 "description": "If set to true, the actor will not store the results in the default dataset. Useful when using alternative storage, like own database",
 "type": "boolean",
 "default": false,
 "editor": "checkbox"
 },
 "PG_CONNECTION_STRING_NORMALIZED": {
 "title": "Postgres connection string for normalized data",
 "description": "If set, actor will store normalized data in Postgres database in PG_DATA_TABLE and PG_DATA_PRICE_TABLE tables",
 "type": "string",
 "editor": "textfield"
 },
 "PG_DATA_TABLE": {
 "title": "Postgres table name for product data",
 "description": "Table name for storing product name, url, image, ...",
 "type": "string",
 "editor": "textfield"
 },
 "PG_DATA_PRICE_TABLE": {
 "title": "Postgres table name for price data",
 "description": "Table name for storing price, original price, stock status, ...",
 "type": "string",
 "editor": "textfield"
 }
 },
 "required": [
 "mode"
 ]
}

INPUT_SCHEMA.json

# r2-bike (r2-bike.com) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).

## Output example

* **pid** `string` e.g. *106018*
* **name** `string` e.g. *SHIMANO 105 R7000*
* **url** `string` e.g. *https://r2-bike.com/SHIMANO-105-R7000*
* **img** `string` e.g. *https://cdn.r2-bike.com/SHIMANO-105-R7000.jpg*
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *19.95*
* **originalPrice** `number` e.g. *39.95*
* **currency** `string` e.g. *EUR*

README.md

{
 "name": "r2-bike-r2-bike-com-scraper",
 "version": "0.1",
 "buildTag": "latest",
 "env": null,
 "defaultRunOptions": {
 "build": "latest",
 "timeoutSecs": 3600,
 "memoryMbytes": 4096
 }
}

apify.json

main.js

{
 "name": "r2-bike-r2-bike-com-scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
 "type": "module",
 "scripts": {
 "start": "node ./main.js",
 "push-to-apify-platform": "npx apify push"
 },
 "dependencies": {
 "apify3": "npm:apify@^3.0.2",
 "crawlee": "*",
 "@crawlee/core": "*",
 "playwright": "*",
 "pg": "*",
 "pg-connection-string": "*",
 "dotenv": "*",
 "find-config": "*",
 "@elastic/elasticsearch": "*",
 "filenamify": "*",
 "@crawlee/memory-storage": "*"
 },
 "apify": {
 "title": "r2-bike (r2-bike.com) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
 "isPublic": true,
 "isDeprecated": false,
 "isAnonymouslyRunnable": true,
 "notice": "",
 "pictureUrl": "",
 "seoTitle": "",
 "seoDescription": "",
 "categories": [
 "ECOMMERCE"
 ]
 }
}

package.json

{
 "actorSpecification": 1,
 "name": "r2-bike-r2-bike-com-scraper",
 "title": "r2-bike (r2-bike.com) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
 "version": "0.1.0",
 "storages": {
 "dataset": {
 "actorSpecification": 1,
 "title": "r2-bike (r2-bike.com) scraper",
 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
 "views": {
 "overview": {
 "title": "Overview",
 "description": "Overview of the most important fields",
 "transformation": {
 "fields": [
 "pid",
 "name",
 "url",
 "img",
 "inStock",
 "currentPrice",
 "originalPrice",
 "currency"
 ]
 },
 "display": {
 "component": "table",
 "columns": [
 {
 "label": "Pid",
 "field": "pid",
 "format": "text"
 },
 {
 "label": "Name",
 "field": "name",
 "format": "text"
 },
 {
 "label": "Url",
 "field": "url",
 "format": "link"
 },
 {
 "label": "Img",
 "field": "img",
 "format": "image"
 },
 {
 "label": "In Stock",
 "field": "inStock",
 "format": "boolean"
 },
 {
 "label": "Current Price",
 "field": "currentPrice",
 "format": "number"
 },
 {
 "label": "Original Price",
 "field": "originalPrice",
 "format": "number"
 },
 {
 "label": "Currency",
 "field": "currency",
 "format": "text"
 }
 ]
 }
 }
 }
 }
 }
}

.actor/actor.json

.actor/logo.png

_utils/common.js

FROM apify/actor-node-playwright-firefox:16

COPY package.json ./

RUN npm --quiet set progress=false \
  && npm install aws-crt \
  && npm install --only=prod --no-optional

COPY . ./

Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).

If set to true, the actor will not store the results in the default dataset. Useful when using alternative storage, like own database

Don't store in dataset

In-memory request queue can reduce costs, but it may case issues with longer runs due to non-persistence.

Advanced options, use only if you know what you're doing.

Use in-memory request queue instead of the native one

If set, actor will store normalized data in Postgres database in PG_DATA_TABLE and PG_DATA_PRICE_TABLE tables

Postgres connection string for normalized data

Table name for storing price, original price, stock status, ...

Postgres table name for price data

Table name for storing product name, url, image, ...

Postgres table name for product data

Mode

r2-bike (r2-bike.com) scraper

# r2-bike (r2-bike.com) scraper

Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).

## Output example

* **pid** `string` e.g. *106018*
* **name** `string` e.g. *SHIMANO 105 R7000*
* **url** `string` e.g. *https://r2-bike.com/SHIMANO-105-R7000*
* **img** `string` e.g. *https://cdn.r2-bike.com/SHIMANO-105-R7000.jpg*
* **inStock** `boolean` e.g. *true*
* **currentPrice** `number` e.g. *19.95*
* **originalPrice** `number` e.g. *39.95*
* **currency** `string` e.g. *EUR*

r2-bike-r2-bike-com-scraper

import { Actor } from "apify3";
import {
 CheerioCrawler,
 createCheerioRouter,
 utils as crawleeUtils,
} from "crawlee";
import { Session } from "@crawlee/core";
import playwright from "playwright";
import { init, parsePrice, save, toNumberOrNull } from "./_utils/common.js";

const LABELS = {
 INDEX: `INDEX`,
 PRODUCTS: `PRODUCTS`,
};

var MODE;

(function (MODE) {
 MODE["TEST"] = "TEST";
 MODE["FULL"] = "FULL";
})(MODE || (MODE = {}));

async function enqueueInitial(mode, crawler) {
 if (mode === MODE.FULL) {
 await crawler.addRequests([
 {
 userData: { label: LABELS.INDEX },
 url: `https://r2-bike.com/en/brands`,
 },
 ]);
 } else if (mode === MODE.TEST) {
 await crawler.addRequests([
 {
 userData: { label: LABELS.PRODUCTS },
 url: `https://r2-bike.com/en/shimano`,
 },
 ]);
 }
}

const router = createCheerioRouter();

router.addHandler(LABELS.INDEX, async ({ enqueueLinks }) => {
 await enqueueLinks({
 selector: `.vendor-index-group-wrapper li a`, // e.g. `en/shimano`
 baseUrl: `https://r2-bike.com/`, // needed for correctly absolute URLs, otherwise it would be `https://r2-bike.com/en/en/shimano`, not sure why ¯\_(ツ)_/¯
 userData: { label: LABELS.PRODUCTS },
 });
});

router.addHandler(LABELS.PRODUCTS, async ({ crawler, $, request, log }) => {
 log.info(`[PRODUCTS] ${request.url}`);

 if (!request.url.match(/_s(\d+)$/)) {
 // on first page
 const paginationText = $(`.list-pageinfo .page-current`).text().trim(); // eg. `Page 1 of 11`
 const match = paginationText.match(/(\d+) of (\d+)/);
 if (!match)
 log.error(
 `[PRODUCTS] Failed to parse pagination text: ${paginationText}`
 );
 const [, currentPage, totalPages] = match ?? [];
 if (Number(totalPages) > 1)
 log.info(`[PRODUCTS] Found ${totalPages} pages, enqueuing`);
 for (let i = 2; i <= Number(totalPages); i++) {
 // skip first page, that is already handled
 void crawler.addRequests([
 {
 url: `${request.url}_s${i}`, // eg. https://r2-bike.com/en/shimano_s2
 userData: { label: LABELS.PRODUCTS },
 },
 ]);
 }
 }

 const products = [];
 const $products = $(
 `#product-list .product-wrapper[itemprop="itemListElement"]`
 ); // itemprop to avoid selecting last fake tile, which is actually "Next page" link
 log.info(`[PRODUCTS] ${request.url} - found ${$products.length} products`);
 $products.each(async (i, el) => {
 const pid = $(`.product-cell`, el)
 .attr(`id`) // result-wrapper_buy_form_106016
 ?.replace(`result-wrapper_buy_form_`, ``); // 106016
 if (!pid)
 return log.error(
 `[PRODUCTS] Failed to parse pid from ${i + 1}th product on ${
 request.url
 }`
 );

 const url = $(`meta[itemprop="url"]`, el).attr(`content`);
 const img = $(`meta[itemprop="image"]`, el).attr(`content`);
 const name = $(`h4[itemprop="name"]`, el).text().trim();

 const priceRaw = $(`.price_wrapper .price`, el).text().trim(); // e.g. 1,98 €*
 const price = parsePrice(priceRaw).amount;

 const priceOrigRaw = $(`.price-uvp`, el).text().trim(); // e.g. MSRP: 5,95 €
 const priceOrig = parsePrice(priceOrigRaw).amount;

 const inStock = $(`.delivery-status`, el).text().includes(`available`);

 const product = {
 pid,
 name,
 url,
 img,
 inStock,
 currentPrice: toNumberOrNull(price),
 originalPrice: toNumberOrNull(priceOrig),
 currency: `EUR`,
 };
 products.push(product);
 });
 await save(products);
});

void Actor.main(async () => {
 const input = await Actor.getInput();
 const { mode = MODE.FULL, ...rest } = input ?? {};
 await init({ actorNameOverride: `r2-bike-com` }, rest);
 const crawler = new CheerioCrawler({
 requestHandler: router,
 preNavigationHooks: [
 async ({ session }, gotOptions) => {
 const userData = session.userData;
 gotOptions.headers = userData.headers; // real-like headers obtained from Firefox
 gotOptions.headers.Cookie = userData.cookies
 .map((c) => `${c.name}=${c.value}`)
 .join(`; `); // real cookies obtained from Firefox
 // gotOptions.proxyUrl = `http://127.0.0.1:9090` // NOTE: uncomment for local debugging
 },
 ],
 maxConcurrency: 1, // not brave enough for concurrency
 maxRequestRetries: 0, // not brave enough for concurrency
 sessionPoolOptions: {
 maxPoolSize: 1, // not brave enough for concurrency
 sessionOptions: {
 maxAgeSecs: 60 * 60 * 2, // 2 hours, default is 50m
 maxUsageCount: 1000, // default is 50, let's use as much as possible, until we get blocked
 },
 createSessionFunction: async (sessionPool) => {
 console.log(
 `[SESSION] Creating new session, will use Firefox to unblock (should take ~10s)`
 );
 const session = new Session({ sessionPool });
 await unblock(session);
 return session;
 },
 },
 });
 await enqueueInitial(mode, crawler);
 await crawler.run();
});

async function unblock(session) {
 const browser = await playwright.firefox.launch({
 // headless: false, // NOTE: uncomment for debugging
 // proxy: { server: `http://127.0.0.1:9090` }, // NOTE: uncomment for local debugging
 });
 const browserContext = await browser.newContext({ ignoreHTTPSErrors: true });
 await browserContext.addCookies([
 {
 name: `eu_cookie_store`,
 value: `{"b209404849c0357500f7a82a6899961a":true,"3940b498c8a17157f69d757a80ff3421":true,"1d3c65b2b03ef35e14df6b163ea3a1f6":false,"0a3fbfc21a86a28c8961999929c374f3":true,"9b88c95a15e018c3f8038a7d0160145c":true,"dd31d974a78cdd704acaa6bf15da506c":true,"d86cf69a8b82547a94ca3f6a307cf9a6":false,"d323dff6f7de41c0b9af4c35e21dc032":false,"b83d1ac867f35569c614e298f645fffe":true,"21affb15e1316adac24b26db8e421a9d":false,"2d1fc55f933c039b2e04ff9034134b4d":true,"4d60ab2c6d11d753267484006c23e54c":false,"970cfba66b8380fb97b742e4571356c6":false}`,
 domain: `r2-bike.com`,
 path: `/`,
 },
 {
 name: `r2_user_delivery_country`,
 value: `CZ`, // TODO: make it configurable
 domain: `r2-bike.com`,
 path: `/`,
 },
 {
 name: `r2_user_delivery_country_ip_backup`,
 value: `CZ`, // TODO: make it configurable
 domain: `r2-bike.com`,
 path: `/`,
 },
 {
 name: `r2_user_delivery_country_tax_1`,
 value: `21`, // TODO: make it configurable
 domain: `r2-bike.com`,
 path: `/`,
 },
 {
 name: `r2_user_delivery_country_tax_2`,
 value: `10`, // TODO: make it configurable
 domain: `r2-bike.com`,
 path: `/`,
 },
 {
 name: `ledgerCurrency`,
 value: `EUR`,
 domain: `r2-bike.com`,
 path: `/`,
 },
 ]);

 const page = await browserContext.newPage();
 // page.on(`console`, msg => console.log(`⚪️ Playwright log (${msg.type()}) ${msg.text()}`))

 let headersToSet;

 await page.route(`**/*`, (route) => {
 const request = route.request();
 const url = request.url();
 const method = request.method(); // GET, POST, etc.
 const resourceType = request.resourceType(); // document, stylesheet, image, ...
 // console.log(`🔵 Playwright route: ${method} ${url} (${resourceType})`)

 // use the first main request to store the sent headers
 if (!headersToSet) headersToSet = pickHeaders(request.headers());

 route.continue();
 });

 // Go to product listing page which sets 95 products per page to current session
 await page.goto(`https://r2-bike.com/navi.php?h=58&Sortierung=1&af=95`); // h=58 is "Shimano", Sortierung=1 is "Sort by name" – both are not important, but some values need to be set
 // Wait for some time to pass basic Cloudflare Javascript checks
 await crawleeUtils.sleep(5000); // TODO: Be smarter, 3000s is enough for r2-bike.com, but not for g2.com
 // Get all cookies and store them for subsequent requests
 const cookies = await page.context().cookies();
 session.userData = { headers: headersToSet, cookies };
}

function pickHeaders(headers) {
 // Pick just the headers that gotScraping can correctly handle (= order)
 // This seems to be needed mainly to avoid setting Host header, which when set, was at the end of the headers list, which Cloudflare did not like
 // If we skip the Host header, then gotScraping will set it automatically, and in the correct order

 // taken from https://github.com/apify/header-generator/blob/1b0fd217b6fa0beaf42b9de321e47ac5f1d4cebf/src/data_files/headers-order.json#L62
 const headersList = [
 `sec-ch-ua`,
 `sec-ch-ua-mobile`,
 `user-agent`,
 `User-Agent`,
 `accept`,
 `Accept`,
 `accept-language`,
 `Accept-Language`,
 `accept-encoding`,
 `Accept-Encoding`,
 `dnt`,
 `DNT`,
 `referer`,
 `Referer`,
 `cookie`,
 `Cookie`,
 `Connection`,
 `upgrade-insecure-requests`,
 `Upgrade-Insecure-Requests`,
 `te`,
 `sec-fetch-site`,
 `sec-fetch-mode`,
 `sec-fetch-user`,
 `sec-fetch-dest`,
 `Sec-Fetch-Mode`,
 `Sec-Fetch-Dest`,
 `Sec-Fetch-Site`,
 `Sec-Fetch-User`,
 ];
 return headersList.reduce((acc, header) => {
 if (headers[header]) acc[header] = headers[header];
 return acc;
 }, {});
}

import { createHash } from 'crypto'
import os from "os"
import path from "path"
// eslint-disable-next-line @apify/apify-actor/no-forbidden-node-internals
import fs from "fs"
import pg from "pg"
import pgConnectionString from 'pg-connection-string'
import { config } from 'dotenv'
import findConfig from "find-config"
import { Client as ElasticClient } from "@elastic/elasticsearch"
import filenamify from 'filenamify'
import { Configuration, Dataset } from 'crawlee'
import { MemoryStorage } from '@crawlee/memory-storage'

config({ path: findConfig(`.env`) })

const elasticIndexName = `actors-monorepo-shops`

const globalLogsProps = {
 __NODE_STARTED: new Date().toISOString(),
}

let actorName
let pgClient
let pgClientNormalized
let elasticClient
export async function init ({ actorNameOverride }, restInput) {
 parseEnvFromInput(restInput)

 if (os.platform() === `darwin`) {
 const filePath = process.argv[1] // ~/Projects/apify-actors-monorepo/actors/foo.ts
 const basename = path.basename(filePath) // foo.ts
 actorName = actorNameOverride ?? basename.split(`.`)[0] // foo
 const gitBranch = fs.readFileSync(path.join(process.cwd(), `..`, `.git/HEAD`), `utf8`)
 .split(` `)[1]
 .trim()
 .replace(`refs/heads/`, ``)
 const gitCommit = fs.readFileSync(path.join(process.cwd(), `..`, `.git/refs/heads/${gitBranch}`), `utf8`)
 const gitCommitShort = gitCommit.substring(0, 7)
 globalLogsProps.__GIT_COMMIT = gitCommitShort
 }

 if (process.env.APIFY_USE_MEMORY_REQUEST_QUEUE === `true`) { // dotenv -> bool-like vars are strings
 Configuration.getGlobalConfig().useStorageClient(new MemoryStorage())
 }

 if (process.env.APIFY_IS_AT_HOME) {
 actorName = actorNameOverride ?? process.env.APIFY_ACTOR_ID // Name would be better, but it's not in ENV
 }

 /* ELASTIC */
 /* ======= */
 if (process.env.ELASTIC_CLOUD_ID) {
 elasticClient = new ElasticClient({
 cloud: { id: process.env.ELASTIC_CLOUD_ID },
 auth: { apiKey: process.env.ELASTIC_CLOUD_API_KEY },
 })

 // const mapping = await elasticClient.indices.getMapping({ index: actorName })

 // eslint-disable-next-line no-inner-declarations
 async function enforceIndexMapping () {
 const doesIndexExist = await elasticClient.indices.exists({ index: elasticIndexName })
 if (!doesIndexExist) await elasticClient.indices.create({ index: elasticIndexName })
 await elasticClient.indices.putMapping({
 index: elasticIndexName,
 body: {
 properties: {
 _discount: { type: `float` },
 originalPrice: { type: `float` },
 currentPrice: { type: `float` },
 },
 },
 })
 }

 try {
 await enforceIndexMapping()
 } catch (err) {
 if (err.message.includes(`cannot be changed from type`)) {
 console.log(`Elastic index ${elasticIndexName} already exists with incorrect mappings. As existing mapping cannot be changed, index will be deleted and recreated.`)
 await elasticClient.indices.delete({ index: elasticIndexName })
 await enforceIndexMapping()
 }
 }
 }

 /* POSTGRESQL */
 /* ========== */
 if (process.env.PG_CONNECTION_STRING) {
 const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING)
 // const pgPool = new pg.Pool(pgConfig)

 pgClient = new pg.Client(pgConfig)
 await pgClient.connect()

 // Check if table exists and have proper columns
 const { rows: tables } = await pgClient.query(`
 SELECT table_name
 FROM information_schema.tables
 WHERE table_schema = 'public'
 `)

 // eslint-disable-next-line camelcase
 const tableExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
 if (!tableExists) {
 throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
 }

 // TODO: Handle pgClient closing
 }

 if (process.env.PG_CONNECTION_STRING_NORMALIZED) {
 const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING_NORMALIZED)

 pgClientNormalized = new pg.Client(pgConfig)
 await pgClientNormalized.connect()

 // Check if table exists and have proper columns
 const { rows: tables } = await pgClientNormalized.query(`
 SELECT table_name
 FROM information_schema.tables
 WHERE table_schema = 'public'
 `)

 // eslint-disable-next-line camelcase
 const tableMainExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
 // eslint-disable-next-line camelcase
 const tablePricesExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_PRICE_TABLE)
 if (!tableMainExists) throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
 if (!tablePricesExists) throw new Error(`Table ${process.env.PG_DATA_PRICE_TABLE} does not exist in database ${pgConfig.database}`)

 // TODO: Handle pgClient closing
 }
}

// inspired by @drobnikj
// TODO: Similar, but less obfuscated for easier debugging
export const createUniqueKeyFromUrl = (url) => {
 const hash = createHash(`sha256`)
 const cleanUrl = url.split(`://`)[1] // Remove protocol
 hash.update(cleanUrl)
 return hash.digest(`hex`)
}

/**
 *
 * @param {Date} datetime
 * @return {Promise<void>}
 */
export const sleepUntil = async (datetime) => {
 const now = new Date()
 const difference = datetime - now
 if (difference > 0) {
 return new Promise((resolve) => {
 setTimeout(resolve, difference)
 })
 }
 return Promise.resolve()
}

// TODO: Uff, nicer! But at least it's tested
export function parsePrice (string) {
 let amount, currency
 const noText = string.replace(/[^\d,.]/g, ``)
 const decimals = noText.match(/([,.])(\d{2})$/)
 if (decimals) {
 const decimalSeparator = decimals[1] // ?
 // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
 const decimalAmount = decimals[2] // ?
 const mainAmount = noText.split(decimalSeparator)[0].replace(/\D/g, ``)
 amount = parseFloat(mainAmount + `.` + decimalAmount) // ?
 } else {
 const justNumbers = noText.replace(/[,.]/g, ``)
 amount = parseInt(justNumbers)
 }
 return { amount, currency }
}

export function toNumberOrNull (str) {
 // TODO: Handle better, but only after adding test
 if (str === undefined) return null
 if (str === null) return null
 if (str === ``) return null
 const num = Number(str)
 if (Number.isNaN(num)) return null
 return num
}

export async function save (objs) {
 if (!Array.isArray(objs)) objs = [objs]
 if (objs.length === 0) return console.log(`No data to save.`)

 const objsExtended = objs.map(async (obj) => {
 const objExtended = {
 ...obj,
 actorName,
 ...globalLogsProps,
 // __NODE_VERSION: global.process.versions.node,
 // __NODE_UPTIME: global.process.uptime().toFixed(2), // seconds, 2 decimals
 }
 // if run on Apify
 if (process.env.APIFY_IS_AT_HOME) {
 objExtended.__APIFY_ACTOR_ID = process.env.APIFY_ACTOR_ID
 objExtended.__APIFY_ACTOR_RUN_ID = process.env.APIFY_ACTOR_RUN_ID
 objExtended.__APIFY_ACTOR_BUILD_ID = process.env.APIFY_ACTOR_BUILD_ID
 objExtended.__APIFY_ACTOR_BUILD_NUMBER = process.env.APIFY_ACTOR_BUILD_NUMBER
 objExtended.__APIFY_ACTOR_TASK_ID = process.env.APIFY_ACTOR_TASK_ID
 if (process.env.APIFY_DONT_STORE_IN_DATASET !== `true`) { // Note: dotenv is not casting vars, so they are strings
 await Dataset.pushData(obj)
 }
 }
 return objExtended
 })
 // if runs on local machine (MacOS)
 if (os.platform() === `darwin`) {
 const cwd = process.cwd() // ~/Projects/apify-actors-monorepo/actors
 const storageDir = path.join(cwd, `${actorName}.storage`) // ~/Projects/apify-actors-monorepo/actors/foo.storage
 if (!fs.existsSync(storageDir)) fs.mkdirSync(storageDir)
 const dataDir = path.join(storageDir, `data`) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data
 if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir)
 for (const objExtended of objsExtended) {
 const id = String(objExtended.id ?? objExtended.pid) // ?? uuidv4()
 const fileName = `${filenamify(id)}.json`
 const dataFilePath = path.join(dataDir, fileName) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data/foo.json
 fs.writeFileSync(dataFilePath, JSON.stringify(objExtended, null, 2))
 }
 }

 if (pgClient) {
 const objsPg = objs.map((obj) => ({
 ...obj,
 // TODO: This is becoming not nice, and not clear
 shop: actorName,
 scrapedAt: new Date().toISOString().split(`T`)[0],
 }))

 const columns = getColumns(objsPg)
 const values = getValues(objsPg)
 const queryString = `
 INSERT INTO public."${process.env.PG_DATA_TABLE}" (${columns})
 VALUES (${values})
 `
 try {
 const { rowCount } = await pgClient.query(queryString)
 console.log(`[save] saved to database: ${JSON.stringify(rowCount)}`)
 } catch (err) {
 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
 else throw err
 }
 }

 // Only make sense for HlidacShopu
 if (pgClientNormalized) {
 const objsPgData = objs.map((obj) => ({
 shop: actorName,
 pid: obj.pid,
 name: obj.name,
 url: obj.url,
 img: obj.img,
 }))

 const objsPgDataPrice = objs.map((obj) => ({
 shop: actorName,
 pid: obj.pid,
 scrapedAt: new Date().toISOString().split(`T`)[0],
 currentPrice: obj.currentPrice,
 originalPrice: obj.originalPrice,
 inStock: obj.inStock,
 }))

 const queryString = `
 INSERT INTO public."${process.env.PG_DATA_TABLE}" (${getColumns(objsPgData)})
 VALUES (${getValues(objsPgData)})
 ON CONFLICT DO NOTHING
 `
 try {
 const { rowCount } = await pgClientNormalized.query(queryString)
 console.log(`[save] saved to database (data): ${JSON.stringify(rowCount)}`)
 } catch (err) {
 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
 else throw err
 }

 const queryStringPrice = `
 INSERT INTO public."${process.env.PG_DATA_PRICE_TABLE}" (${getColumns(objsPgDataPrice)})
 VALUES (${getValues(objsPgDataPrice)})
 ON CONFLICT DO NOTHING
 `
 try {
 const { rowCount } = await pgClientNormalized.query(queryStringPrice)
 console.log(`[save] saved to database (price): ${JSON.stringify(rowCount)}`)
 } catch (err) {
 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
 else throw err
 }
 }

 if (elasticClient) {
 // .index creates or updates the document
 // .create creates a new document if it doesn't exist, 409 if it does
 // try {
 // const res = await elasticClient.index({
 // index: `actors-monorepo-shops`, // TODO: Consider using actorName
 // id, // foo-bar
 // document: objExtended, // {...}
 // })
 // } catch (err) {
 // // https://discuss.elastic.co/t/elasticsearch-503-ok-false-message-the-requested-deployment-is-currently-unavailable/200583
 // if (err.message.includes(`requested resource is currently unavailable`)) console.log(`Elasticsearch is unavailable, skipping, but not aborting`)
 // else throw err
 // }
 }
}

function getColumns (objs) {
 return Object.keys(objs[0]).map((key) => `"${key}"`).join(`, `)
}

function getValues (objs) {
 return objs.map(objPg => Object.values(objPg).map((value) => {
 // escape strings to prevent SQL injection
 if (typeof value === `string`) return `'${value.replace(/'/g, `''`)}'`
 // convert to DB specific null
 if (typeof value === `undefined` || value === null) return `NULL`
 return value
 }).join(`, `)).join(`), (`)
}

export function parseEnvFromInput (input) {
 const env = {}
 for (const key in input) {
 if (key === key.toUpperCase()) env[key] = input[key]
 }
 console.log(`[parseEnvFromInput] ${JSON.stringify(env)}`)
 Object.assign(process.env, env)
}

export const isInspect =
 process.execArgv.join().includes(`--inspect`) ||
 // @ts-ignore
 process?._preload_modules?.join(`|`)?.includes(`debug`)

r2-bike (r2-bike.com) scraper

Output example

Industries

Where next?