r2-bike (r2-bike.com) scraper

strajk/r2-bike-r2-bike-com-scraper

No credit card required

Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).

Dockerfile

FROM apify/actor-node-playwright-firefox:16

COPY package.json ./

RUN npm --quiet set progress=false \
  && npm install aws-crt \
  && npm install --only=prod --no-optional

COPY . ./

INPUT_SCHEMA.json

{
  "title": "r2-bike (r2-bike.com) scraper",
  "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
  "type": "object",
  "schemaVersion": 1,
  "properties": {
    "mode": {
      "title": "Mode",
      "description": "",
      "type": "string",
      "editor": "select",
      "default": "TEST",
      "prefill": "TEST",
      "enum": [
        "TEST",
        "FULL"
      ],
      "enumTitles": [
        "TEST",
        "FULL"
      ]
    },
    "APIFY_USE_MEMORY_REQUEST_QUEUE": {
      "sectionCaption": "Advanced",
      "sectionDescription": "Advanced options, use only if you know what you're doing.",
      "title": "Use in-memory request queue instead of the native one",
      "description": "In-memory request queue can reduce costs, but it may case issues with longer runs due to non-persistence.",
      "type": "boolean",
      "default": false,
      "editor": "checkbox"
    },
    "APIFY_DONT_STORE_IN_DATASET": {
      "title": "Don't store in dataset",
      "description": "If set to true, the actor will not store the results in the default dataset. Useful when using alternative storage, like own database",
      "type": "boolean",
      "default": false,
      "editor": "checkbox"
    },
    "PG_CONNECTION_STRING_NORMALIZED": {
      "title": "Postgres connection string for normalized data",
      "description": "If set, actor will store normalized data in Postgres database in PG_DATA_TABLE and PG_DATA_PRICE_TABLE tables",
      "type": "string",
      "editor": "textfield"
    },
    "PG_DATA_TABLE": {
      "title": "Postgres table name for product data",
      "description": "Table name for storing product name, url, image, ...",
      "type": "string",
      "editor": "textfield"
    },
    "PG_DATA_PRICE_TABLE": {
      "title": "Postgres table name for price data",
      "description": "Table name for storing price, original price, stock status, ...",
      "type": "string",
      "editor": "textfield"
    }
  },
  "required": [
    "mode"
  ]
}

apify.json

{
  "name": "r2-bike-r2-bike-com-scraper",
  "version": "0.1",
  "buildTag": "latest",
  "env": null,
  "defaultRunOptions": {
    "build": "latest",
    "timeoutSecs": 3600,
    "memoryMbytes": 4096
  }
}

main.js

import { Actor } from "apify3";
import {
  CheerioCrawler,
  createCheerioRouter,
  utils as crawleeUtils,
} from "crawlee";
import { Session } from "@crawlee/core";
import playwright from "playwright";
import { init, parsePrice, save, toNumberOrNull } from "./_utils/common.js";

const LABELS = {
  INDEX: `INDEX`,
  PRODUCTS: `PRODUCTS`,
};

var MODE;

(function (MODE) {
  MODE["TEST"] = "TEST";
  MODE["FULL"] = "FULL";
})(MODE || (MODE = {}));

async function enqueueInitial(mode, crawler) {
  if (mode === MODE.FULL) {
    await crawler.addRequests([
      {
        userData: { label: LABELS.INDEX },
        url: `https://r2-bike.com/en/brands`,
      },
    ]);
  } else if (mode === MODE.TEST) {
    await crawler.addRequests([
      {
        userData: { label: LABELS.PRODUCTS },
        url: `https://r2-bike.com/en/shimano`,
      },
    ]);
  }
}

const router = createCheerioRouter();

router.addHandler(LABELS.INDEX, async ({ enqueueLinks }) => {
  await enqueueLinks({
    selector: `.vendor-index-group-wrapper li a`, // e.g. `en/shimano`
    baseUrl: `https://r2-bike.com/`, // needed for correctly absolute URLs, otherwise it would be `https://r2-bike.com/en/en/shimano`, not sure why ¯\_(ツ)_/¯
    userData: { label: LABELS.PRODUCTS },
  });
});

router.addHandler(LABELS.PRODUCTS, async ({ crawler, $, request, log }) => {
  log.info(`[PRODUCTS] ${request.url}`);

  if (!request.url.match(/_s(\d+)$/)) {
    // on first page
    const paginationText = $(`.list-pageinfo .page-current`).text().trim(); // eg. `Page 1 of 11`
    const match = paginationText.match(/(\d+) of (\d+)/);
    if (!match)
      log.error(
        `[PRODUCTS] Failed to parse pagination text: ${paginationText}`
      );
    const [, currentPage, totalPages] = match ?? [];
    if (Number(totalPages) > 1)
      log.info(`[PRODUCTS] Found ${totalPages} pages, enqueuing`);
    for (let i = 2; i <= Number(totalPages); i++) {
      // skip first page, that is already handled
      void crawler.addRequests([
        {
          url: `${request.url}_s${i}`, // eg. https://r2-bike.com/en/shimano_s2
          userData: { label: LABELS.PRODUCTS },
        },
      ]);
    }
  }

  const products = [];
  const $products = $(
    `#product-list .product-wrapper[itemprop="itemListElement"]`
  ); // itemprop to avoid selecting last fake tile, which is actually "Next page" link
  log.info(`[PRODUCTS] ${request.url} - found ${$products.length} products`);
  $products.each(async (i, el) => {
    const pid = $(`.product-cell`, el)
      .attr(`id`) // result-wrapper_buy_form_106016
      ?.replace(`result-wrapper_buy_form_`, ``); // 106016
    if (!pid)
      return log.error(
        `[PRODUCTS] Failed to parse pid from ${i + 1}th product on ${
          request.url
        }`
      );

    const url = $(`meta[itemprop="url"]`, el).attr(`content`);
    const img = $(`meta[itemprop="image"]`, el).attr(`content`);
    const name = $(`h4[itemprop="name"]`, el).text().trim();

    const priceRaw = $(`.price_wrapper .price`, el).text().trim(); // e.g. 1,98 €*
    const price = parsePrice(priceRaw).amount;

    const priceOrigRaw = $(`.price-uvp`, el).text().trim(); // e.g. MSRP: 5,95 €
    const priceOrig = parsePrice(priceOrigRaw).amount;

    const inStock = $(`.delivery-status`, el).text().includes(`available`);

    const product = {
      pid,
      name,
      url,
      img,
      inStock,
      currentPrice: toNumberOrNull(price),
      originalPrice: toNumberOrNull(priceOrig),
      currency: `EUR`,
    };
    products.push(product);
  });
  await save(products);
});

void Actor.main(async () => {
  const input = await Actor.getInput();
  const { mode = MODE.FULL, ...rest } = input ?? {};
  await init({ actorNameOverride: `r2-bike-com` }, rest);
  const crawler = new CheerioCrawler({
    requestHandler: router,
    preNavigationHooks: [
      async ({ session }, gotOptions) => {
        const userData = session.userData;
        gotOptions.headers = userData.headers; // real-like headers obtained from Firefox
        gotOptions.headers.Cookie = userData.cookies
          .map((c) => `${c.name}=${c.value}`)
          .join(`; `); // real cookies obtained from Firefox
        // gotOptions.proxyUrl = `http://127.0.0.1:9090` // NOTE: uncomment for local debugging
      },
    ],
    maxConcurrency: 1, // not brave enough for concurrency
    maxRequestRetries: 0, // not brave enough for concurrency
    sessionPoolOptions: {
      maxPoolSize: 1, // not brave enough for concurrency
      sessionOptions: {
        maxAgeSecs: 60 * 60 * 2, // 2 hours, default is 50m
        maxUsageCount: 1000, // default is 50, let's use as much as possible, until we get blocked
      },
      createSessionFunction: async (sessionPool) => {
        console.log(
          `[SESSION] Creating new session, will use Firefox to unblock (should take ~10s)`
        );
        const session = new Session({ sessionPool });
        await unblock(session);
        return session;
      },
    },
  });
  await enqueueInitial(mode, crawler);
  await crawler.run();
});

async function unblock(session) {
  const browser = await playwright.firefox.launch({
    // headless: false, // NOTE: uncomment for debugging
    // proxy: { server: `http://127.0.0.1:9090` }, // NOTE: uncomment for local debugging
  });
  const browserContext = await browser.newContext({ ignoreHTTPSErrors: true });
  await browserContext.addCookies([
    {
      name: `eu_cookie_store`,
      value: `{"b209404849c0357500f7a82a6899961a":true,"3940b498c8a17157f69d757a80ff3421":true,"1d3c65b2b03ef35e14df6b163ea3a1f6":false,"0a3fbfc21a86a28c8961999929c374f3":true,"9b88c95a15e018c3f8038a7d0160145c":true,"dd31d974a78cdd704acaa6bf15da506c":true,"d86cf69a8b82547a94ca3f6a307cf9a6":false,"d323dff6f7de41c0b9af4c35e21dc032":false,"b83d1ac867f35569c614e298f645fffe":true,"21affb15e1316adac24b26db8e421a9d":false,"2d1fc55f933c039b2e04ff9034134b4d":true,"4d60ab2c6d11d753267484006c23e54c":false,"970cfba66b8380fb97b742e4571356c6":false}`,
      domain: `r2-bike.com`,
      path: `/`,
    },
    {
      name: `r2_user_delivery_country`,
      value: `CZ`, // TODO: make it configurable
      domain: `r2-bike.com`,
      path: `/`,
    },
    {
      name: `r2_user_delivery_country_ip_backup`,
      value: `CZ`, // TODO: make it configurable
      domain: `r2-bike.com`,
      path: `/`,
    },
    {
      name: `r2_user_delivery_country_tax_1`,
      value: `21`, // TODO: make it configurable
      domain: `r2-bike.com`,
      path: `/`,
    },
    {
      name: `r2_user_delivery_country_tax_2`,
      value: `10`, // TODO: make it configurable
      domain: `r2-bike.com`,
      path: `/`,
    },
    {
      name: `ledgerCurrency`,
      value: `EUR`,
      domain: `r2-bike.com`,
      path: `/`,
    },
  ]);

  const page = await browserContext.newPage();
  // page.on(`console`, msg => console.log(`⚪️ Playwright log (${msg.type()}) ${msg.text()}`))

  let headersToSet;

  await page.route(`**/*`, (route) => {
    const request = route.request();
    const url = request.url();
    const method = request.method(); // GET, POST, etc.
    const resourceType = request.resourceType(); // document, stylesheet, image, ...
    // console.log(`🔵 Playwright route: ${method} ${url} (${resourceType})`)

    // use the first main request to store the sent headers
    if (!headersToSet) headersToSet = pickHeaders(request.headers());

    route.continue();
  });

  // Go to product listing page which sets 95 products per page to current session
  await page.goto(`https://r2-bike.com/navi.php?h=58&Sortierung=1&af=95`); // h=58 is "Shimano", Sortierung=1 is "Sort by name" – both are not important, but some values need to be set
  // Wait for some time to pass basic Cloudflare Javascript checks
  await crawleeUtils.sleep(5000); // TODO: Be smarter, 3000s is enough for r2-bike.com, but not for g2.com
  // Get all cookies and store them for subsequent requests
  const cookies = await page.context().cookies();
  session.userData = { headers: headersToSet, cookies };
}

function pickHeaders(headers) {
  // Pick just the headers that gotScraping can correctly handle (= order)
  // This seems to be needed mainly to avoid setting Host header, which when set, was at the end of the headers list, which Cloudflare did not like
  //   If we skip the Host header, then gotScraping will set it automatically, and in the correct order

  // taken from https://github.com/apify/header-generator/blob/1b0fd217b6fa0beaf42b9de321e47ac5f1d4cebf/src/data_files/headers-order.json#L62
  const headersList = [
    `sec-ch-ua`,
    `sec-ch-ua-mobile`,
    `user-agent`,
    `User-Agent`,
    `accept`,
    `Accept`,
    `accept-language`,
    `Accept-Language`,
    `accept-encoding`,
    `Accept-Encoding`,
    `dnt`,
    `DNT`,
    `referer`,
    `Referer`,
    `cookie`,
    `Cookie`,
    `Connection`,
    `upgrade-insecure-requests`,
    `Upgrade-Insecure-Requests`,
    `te`,
    `sec-fetch-site`,
    `sec-fetch-mode`,
    `sec-fetch-user`,
    `sec-fetch-dest`,
    `Sec-Fetch-Mode`,
    `Sec-Fetch-Dest`,
    `Sec-Fetch-Site`,
    `Sec-Fetch-User`,
  ];
  return headersList.reduce((acc, header) => {
    if (headers[header]) acc[header] = headers[header];
    return acc;
  }, {});
}

package.json

{
  "name": "r2-bike-r2-bike-com-scraper",
  "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
  "type": "module",
  "scripts": {
    "start": "node ./main.js",
    "push-to-apify-platform": "npx apify push"
  },
  "dependencies": {
    "apify3": "npm:apify@^3.0.2",
    "crawlee": "*",
    "@crawlee/core": "*",
    "playwright": "*",
    "pg": "*",
    "pg-connection-string": "*",
    "dotenv": "*",
    "find-config": "*",
    "@elastic/elasticsearch": "*",
    "filenamify": "*",
    "@crawlee/memory-storage": "*"
  },
  "apify": {
    "title": "r2-bike (r2-bike.com) scraper",
    "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
    "isPublic": true,
    "isDeprecated": false,
    "isAnonymouslyRunnable": true,
    "notice": "",
    "pictureUrl": "",
    "seoTitle": "",
    "seoDescription": "",
    "categories": [
      "ECOMMERCE"
    ]
  }
}

.actor/actor.json

{
  "actorSpecification": 1,
  "name": "r2-bike-r2-bike-com-scraper",
  "title": "r2-bike (r2-bike.com) scraper",
  "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
  "version": "0.1.0",
  "storages": {
    "dataset": {
      "actorSpecification": 1,
      "title": "r2-bike (r2-bike.com) scraper",
      "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).",
      "views": {
        "overview": {
          "title": "Overview",
          "description": "Overview of the most important fields",
          "transformation": {
            "fields": [
              "pid",
              "name",
              "url",
              "img",
              "inStock",
              "currentPrice",
              "originalPrice",
              "currency"
            ]
          },
          "display": {
            "component": "table",
            "columns": [
              {
                "label": "Pid",
                "field": "pid",
                "format": "text"
              },
              {
                "label": "Name",
                "field": "name",
                "format": "text"
              },
              {
                "label": "Url",
                "field": "url",
                "format": "link"
              },
              {
                "label": "Img",
                "field": "img",
                "format": "image"
              },
              {
                "label": "In Stock",
                "field": "inStock",
                "format": "boolean"
              },
              {
                "label": "Current Price",
                "field": "currentPrice",
                "format": "number"
              },
              {
                "label": "Original Price",
                "field": "originalPrice",
                "format": "number"
              },
              {
                "label": "Currency",
                "field": "currency",
                "format": "text"
              }
            ]
          }
        }
      }
    }
  }
}

.actor/logo.png

_utils/common.js

import { createHash } from 'crypto'
import os from "os"
import path from "path"
// eslint-disable-next-line @apify/apify-actor/no-forbidden-node-internals
import fs from "fs"
import pg from "pg"
import pgConnectionString from 'pg-connection-string'
import { config } from 'dotenv'
import findConfig from "find-config"
import { Client as ElasticClient } from "@elastic/elasticsearch"
import filenamify from 'filenamify'
import { Configuration, Dataset } from 'crawlee'
import { MemoryStorage } from '@crawlee/memory-storage'

config({ path: findConfig(`.env`) })

const elasticIndexName = `actors-monorepo-shops`

const globalLogsProps = {
  __NODE_STARTED: new Date().toISOString(),
}

let actorName
let pgClient
let pgClientNormalized
let elasticClient
export async function init ({ actorNameOverride }, restInput) {
  parseEnvFromInput(restInput)

  if (os.platform() === `darwin`) {
    const filePath = process.argv[1] // ~/Projects/apify-actors-monorepo/actors/foo.ts
    const basename = path.basename(filePath) // foo.ts
    actorName = actorNameOverride ?? basename.split(`.`)[0] // foo
    const gitBranch = fs.readFileSync(path.join(process.cwd(), `..`, `.git/HEAD`), `utf8`)
      .split(` `)[1]
      .trim()
      .replace(`refs/heads/`, ``)
    const gitCommit = fs.readFileSync(path.join(process.cwd(), `..`, `.git/refs/heads/${gitBranch}`), `utf8`)
    const gitCommitShort = gitCommit.substring(0, 7)
    globalLogsProps.__GIT_COMMIT = gitCommitShort
  }

  if (process.env.APIFY_USE_MEMORY_REQUEST_QUEUE === `true`) { // dotenv -> bool-like vars are strings
    Configuration.getGlobalConfig().useStorageClient(new MemoryStorage())
  }

  if (process.env.APIFY_IS_AT_HOME) {
    actorName = actorNameOverride ?? process.env.APIFY_ACTOR_ID // Name would be better, but it's not in ENV
  }

  /* ELASTIC */
  /* ======= */
  if (process.env.ELASTIC_CLOUD_ID) {
    elasticClient = new ElasticClient({
      cloud: { id: process.env.ELASTIC_CLOUD_ID },
      auth: { apiKey: process.env.ELASTIC_CLOUD_API_KEY },
    })

    // const mapping = await elasticClient.indices.getMapping({ index: actorName })

    // eslint-disable-next-line no-inner-declarations
    async function enforceIndexMapping () {
      const doesIndexExist = await elasticClient.indices.exists({ index: elasticIndexName })
      if (!doesIndexExist) await elasticClient.indices.create({ index: elasticIndexName })
      await elasticClient.indices.putMapping({
        index: elasticIndexName,
        body: {
          properties: {
            _discount: { type: `float` },
            originalPrice: { type: `float` },
            currentPrice: { type: `float` },
          },
        },
      })
    }

    try {
      await enforceIndexMapping()
    } catch (err) {
      if (err.message.includes(`cannot be changed from type`)) {
        console.log(`Elastic index ${elasticIndexName} already exists with incorrect mappings. As existing mapping cannot be changed, index will be deleted and recreated.`)
        await elasticClient.indices.delete({ index: elasticIndexName })
        await enforceIndexMapping()
      }
    }
  }

  /* POSTGRESQL */
  /* ========== */
  if (process.env.PG_CONNECTION_STRING) {
    const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING)
    // const pgPool = new pg.Pool(pgConfig)

    pgClient = new pg.Client(pgConfig)
    await pgClient.connect()

    // Check if table exists and have proper columns
    const { rows: tables } = await pgClient.query(`
    SELECT table_name
    FROM information_schema.tables
    WHERE table_schema = 'public'
  `)

    // eslint-disable-next-line camelcase
    const tableExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
    if (!tableExists) {
      throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
    }

  // TODO: Handle pgClient closing
  }

  if (process.env.PG_CONNECTION_STRING_NORMALIZED) {
    const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING_NORMALIZED)

    pgClientNormalized = new pg.Client(pgConfig)
    await pgClientNormalized.connect()

    // Check if table exists and have proper columns
    const { rows: tables } = await pgClientNormalized.query(`
    SELECT table_name
    FROM information_schema.tables
    WHERE table_schema = 'public'
  `)

    // eslint-disable-next-line camelcase
    const tableMainExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
    // eslint-disable-next-line camelcase
    const tablePricesExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_PRICE_TABLE)
    if (!tableMainExists) throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
    if (!tablePricesExists) throw new Error(`Table ${process.env.PG_DATA_PRICE_TABLE} does not exist in database ${pgConfig.database}`)

  // TODO: Handle pgClient closing
  }
}

// inspired by @drobnikj
// TODO: Similar, but less obfuscated for easier debugging
export const createUniqueKeyFromUrl = (url) => {
  const hash = createHash(`sha256`)
  const cleanUrl = url.split(`://`)[1] // Remove protocol
  hash.update(cleanUrl)
  return hash.digest(`hex`)
}

/**
 *
 * @param {Date} datetime
 * @return {Promise<void>}
 */
export const sleepUntil = async (datetime) => {
  const now = new Date()
  const difference = datetime - now
  if (difference > 0) {
    return new Promise((resolve) => {
      setTimeout(resolve, difference)
    })
  }
  return Promise.resolve()
}

// TODO: Uff, nicer! But at least it's tested
export function parsePrice (string) {
  let amount, currency
  const noText = string.replace(/[^\d,.]/g, ``)
  const decimals = noText.match(/([,.])(\d{2})$/)
  if (decimals) {
    const decimalSeparator = decimals[1] // ?
    // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
    const decimalAmount = decimals[2] // ?
    const mainAmount = noText.split(decimalSeparator)[0].replace(/\D/g, ``)
    amount = parseFloat(mainAmount + `.` + decimalAmount) // ?
  } else {
    const justNumbers = noText.replace(/[,.]/g, ``)
    amount = parseInt(justNumbers)
  }
  return { amount, currency }
}

export function toNumberOrNull (str) {
  // TODO: Handle better, but only after adding test
  if (str === undefined) return null
  if (str === null) return null
  if (str === ``) return null
  const num = Number(str)
  if (Number.isNaN(num)) return null
  return num
}

export async function save (objs) {
  if (!Array.isArray(objs)) objs = [objs]
  if (objs.length === 0) return console.log(`No data to save.`)

  const objsExtended = objs.map(async (obj) => {
    const objExtended = {
      ...obj,
      actorName,
      ...globalLogsProps,
      // __NODE_VERSION: global.process.versions.node,
      // __NODE_UPTIME: global.process.uptime().toFixed(2), // seconds, 2 decimals
    }
    // if run on Apify
    if (process.env.APIFY_IS_AT_HOME) {
      objExtended.__APIFY_ACTOR_ID = process.env.APIFY_ACTOR_ID
      objExtended.__APIFY_ACTOR_RUN_ID = process.env.APIFY_ACTOR_RUN_ID
      objExtended.__APIFY_ACTOR_BUILD_ID = process.env.APIFY_ACTOR_BUILD_ID
      objExtended.__APIFY_ACTOR_BUILD_NUMBER = process.env.APIFY_ACTOR_BUILD_NUMBER
      objExtended.__APIFY_ACTOR_TASK_ID = process.env.APIFY_ACTOR_TASK_ID
      if (process.env.APIFY_DONT_STORE_IN_DATASET !== `true`) { // Note: dotenv is not casting vars, so they are strings
        await Dataset.pushData(obj)
      }
    }
    return objExtended
  })
  // if runs on local machine (MacOS)
  if (os.platform() === `darwin`) {
    const cwd = process.cwd() // ~/Projects/apify-actors-monorepo/actors
    const storageDir = path.join(cwd, `${actorName}.storage`) // ~/Projects/apify-actors-monorepo/actors/foo.storage
    if (!fs.existsSync(storageDir)) fs.mkdirSync(storageDir)
    const dataDir = path.join(storageDir, `data`) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data
    if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir)
    for (const objExtended of objsExtended) {
      const id = String(objExtended.id ?? objExtended.pid) // ?? uuidv4()
      const fileName = `${filenamify(id)}.json`
      const dataFilePath = path.join(dataDir, fileName) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data/foo.json
      fs.writeFileSync(dataFilePath, JSON.stringify(objExtended, null, 2))
    }
  }

  if (pgClient) {
    const objsPg = objs.map((obj) => ({
      ...obj,
      // TODO: This is becoming not nice, and not clear
      shop: actorName,
      scrapedAt: new Date().toISOString().split(`T`)[0],
    }))

    const columns = getColumns(objsPg)
    const values = getValues(objsPg)
    const queryString = `
        INSERT INTO public."${process.env.PG_DATA_TABLE}" (${columns})
        VALUES (${values})
    `
    try {
      const { rowCount } = await pgClient.query(queryString)
      console.log(`[save] saved to database: ${JSON.stringify(rowCount)}`)
    } catch (err) {
      if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
      else throw err
    }
  }

  // Only make sense for HlidacShopu
  if (pgClientNormalized) {
    const objsPgData = objs.map((obj) => ({
      shop: actorName,
      pid: obj.pid,
      name: obj.name,
      url: obj.url,
      img: obj.img,
    }))

    const objsPgDataPrice = objs.map((obj) => ({
      shop: actorName,
      pid: obj.pid,
      scrapedAt: new Date().toISOString().split(`T`)[0],
      currentPrice: obj.currentPrice,
      originalPrice: obj.originalPrice,
      inStock: obj.inStock,
    }))

    const queryString = `
        INSERT INTO public."${process.env.PG_DATA_TABLE}" (${getColumns(objsPgData)})
        VALUES (${getValues(objsPgData)})
        ON CONFLICT DO NOTHING
    `
    try {
      const { rowCount } = await pgClientNormalized.query(queryString)
      console.log(`[save] saved to database (data): ${JSON.stringify(rowCount)}`)
    } catch (err) {
      if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
      else throw err
    }

    const queryStringPrice = `
        INSERT INTO public."${process.env.PG_DATA_PRICE_TABLE}" (${getColumns(objsPgDataPrice)})
        VALUES (${getValues(objsPgDataPrice)})
        ON CONFLICT DO NOTHING
    `
    try {
      const { rowCount } = await pgClientNormalized.query(queryStringPrice)
      console.log(`[save] saved to database (price): ${JSON.stringify(rowCount)}`)
    } catch (err) {
      if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
      else throw err
    }
  }

  if (elasticClient) {
    // .index creates or updates the document
    // .create creates a new document if it doesn't exist, 409 if it does
    // try {
    //   const res = await elasticClient.index({
    //     index: `actors-monorepo-shops`, // TODO: Consider using actorName
    //     id, // foo-bar
    //     document: objExtended, // {...}
    //   })
    // } catch (err) {
    //   // https://discuss.elastic.co/t/elasticsearch-503-ok-false-message-the-requested-deployment-is-currently-unavailable/200583
    //   if (err.message.includes(`requested resource is currently unavailable`)) console.log(`Elasticsearch is unavailable, skipping, but not aborting`)
    //   else throw err
    // }
  }
}

function getColumns (objs) {
  return Object.keys(objs[0]).map((key) => `"${key}"`).join(`, `)
}

function getValues (objs) {
  return objs.map(objPg => Object.values(objPg).map((value) => {
    // escape strings to prevent SQL injection
    if (typeof value === `string`) return `'${value.replace(/'/g, `''`)}'`
    // convert to DB specific null
    if (typeof value === `undefined` || value === null) return `NULL`
    return value
  }).join(`, `)).join(`), (`)
}

export function parseEnvFromInput (input) {
  const env = {}
  for (const key in input) {
    if (key === key.toUpperCase()) env[key] = input[key]
  }
  console.log(`[parseEnvFromInput] ${JSON.stringify(env)}`)
  Object.assign(process.env, env)
}

export const isInspect =
  process.execArgv.join().includes(`--inspect`) ||
  // @ts-ignore
  process?._preload_modules?.join(`|`)?.includes(`debug`)