FROM apify/actor-node:18

COPY package.json ./

RUN npm --quiet set progress=false \
  && npm install --only=prod --no-optional

COPY . ./

Dockerfile

Scrapes products titles, prices, images and availability. Does NOT scrape product details. Uses Crawlee (Apify v3).

If set to true, the actor will not store the results in the default dataset. Useful when using alternative storage, like own database

Don't store in dataset

In-memory request queue can reduce costs, but it may case issues with longer runs due to non-persistence.

Advanced options, use only if you know what you're doing.

Use in-memory request queue instead of the native one

If set, actor will store normalized data in Postgres database in PG_DATA_TABLE and PG_DATA_PRICE_TABLE tables

Postgres connection string for normalized data

Table name for storing price, original price, stock status, ...

Postgres table name for price data

Table name for storing product name, url, image, ...

Postgres table name for product data

Mode

Bike Components (bike-components.de) scraper

INPUT_SCHEMA.json

bike-components-bike-components-de-scraper

apify.json

import { Actor } from "apify3";
import { CheerioCrawler, createCheerioRouter } from "crawlee";
import { init, parsePrice, save } from "./_utils/common.js";

const LABELS = {
  INDEX: `INDEX`,
  PRODUCTS: `PRODUCTS`,
};

var MODE;

(function (MODE) {
  MODE["TEST"] = "TEST";
  MODE["FULL"] = "FULL";
})(MODE || (MODE = {}));

const BASE_URL = `https://www.bike-components.de`;

async function enqueueInitial(mode, crawler) {
  if (mode === MODE.FULL) {
    await crawler.addRequests([
      {
        userData: { label: LABELS.INDEX },
        url: `https://www.bike-components.de/en/brands/`,
      },
    ]);
  } else if (mode === MODE.TEST) {
    await crawler.addRequests([
      {
        userData: { label: LABELS.PRODUCTS },
        url: `https://www.bike-components.de/en/100-/`,
      },
    ]);
  }
}

const router = createCheerioRouter();

router.addHandler(LABELS.INDEX, async ({ enqueueLinks }) => {
  await enqueueLinks({
    selector: `.container-manufacturer-list-for-letter .site-link`,
    userData: { label: LABELS.PRODUCTS },
  });
});

router.addHandler(LABELS.PRODUCTS, async ({ crawler, $, request, log }) => {
  // Get brand id from HTML, it's needed for the API
  const brandId = $(`body`)
    .text()
    .match(/"manufacturerId":(\d+)}/)[1]; // https://share.cleanshot.com/3YvVXs
  log.info(`[PRODUCTS] ${request.url}, brandId: ${brandId}`);

  // Paginate products via API
  let hasMorePages = true;
  let page = 0;
  while (hasMorePages) {
    const res = await fetch(
      `https://www.bike-components.de/en/api/v1/catalog/DE/property/?m%5B0%5D=${brandId}&page=${page}&productsPerPage=72`,
      {
        headers: {
          accept: `application/json`, // maybe not needed
          "cache-control": `no-cache`, // maybe not needed
        },
      }
    );

    if (!res.ok)
      throw new Error(
        `[PRODUCTS] ${request.url}: API returned ${res.status} ${res.statusText}`
      );

    const resJson = await res.json();

    log.info(
      `[PRODUCTS] ${request.url}: page: ${page}, products: ${resJson.initialData.products.length}`
    );

    // Parsing!
    const products = [];
    for (const el of resJson.initialData.products) {
      const currentPriceRaw = el.data.price; // `124.99€` or ` <span>from</span>  120.99€`
      const originalPriceRaw = el.data.strikeThroughPrice; // `118.99€`
      const product = {
        pid: el.data.productId.toString(),
        name: el.data.name,
        url: BASE_URL + el.data.link,
        img: BASE_URL + el.data.imageMedium.path, // jpeg
        inStock: el.data.stockQuantity > 0,
        currentPrice: parsePrice(currentPriceRaw)?.amount || null,
        originalPrice: parsePrice(originalPriceRaw)?.amount || null,
        currency: `EUR`,
      };
      products.push(product);
    }
    await save(products);

    // Pagination logic
    if (resJson.initialData.paging.last > resJson.initialData.paging.current) {
      page++;
    } else {
      hasMorePages = false;
    }
  }
});

void Actor.main(async () => {
  const input = await Actor.getInput();
  const { mode = MODE.FULL, ...rest } = input ?? {};
  await init({ actorNameOverride: `bike-components-de` }, rest);
  const crawler = new CheerioCrawler({ requestHandler: router });
  await enqueueInitial(mode, crawler);
  await crawler.run();
});


main.js

package.json

.actor/actor.json

iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAYAAAD0eNT6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAACyGSURBVHgB7d1NjJz3fR/wZ3f2jUtySUqkpJiMqJJJEAlIoqBpZBSIfImNFnYv0qm3uCefYqBFe6pP6SkFAjin5FIHLZCT5baIgQJWgUIqClioW+vQSo4DsZZM2pJIieSS3Pdd9vk9Mw85nJ3Zndmd3Z3Z3+cDLEhRu7Pz8rx8/2+//8SVK1ceFgBAKpMFAJCOAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQAAAACQkAAJCQADBEV0+sFa+cWS4WprYKABhlUwV7NjXxsGhMFMV8Y6s4WX49N7tZnJ3eKC5uTJYhYLJY3GhU37e4IWcBMFoEgH2Yn3xYvHhqtXj92Xvl1/1t///PPzpX3fy/c+NMAQCjRAAY0IWZzWK2vPFfmV8rFspWfwSAyyc2un7vSydXi9WtiWLx2ckqCNxYmSoWNxvF9RVvOwBHa+LKlSsPC/pWj/F//eLiowDQj+urU8X3PjldvHd/pnjzs5MFABwlTdFdnJveLE5Oxo1+rbrxRwCIHoCLsxvFbKP/7BRhIX72ubIHIR7n9nqjWNqcKN57MGuOAACHTgDYxVNTm8X58qb95aeXiktzG9VNfC+a4WGluHJivRwyWC8+WJ4ubq2VwwGr0wIAAIdOAOghlvRdOblRfHFhufh75Xj/pbnN4kxjs9ivCALx2OenN4q1h825AREC3v78RDVfAAAOgwDQQ0z2i0l8r5xdLl48uVYMSwwfzM40exXCS6dmqlDwzp05AQCAQ2MSYEvcmF8sb/jRzf+Vsrs/uumfn1tv3rAnD+4tipt+tVKg7Am4tzlZvPHJ6WqVgImCABwkPQBt6pt9tM7Plq3yw6joV//O+F2LVQGhreKMSoIAHLDUPQAvL6wWsxNbxTcv36luwi+fXilGyfsPmksG37w1X60WAIBhSdkDEGV7T5RfF2Y2ipkyAMxOPKy+Rs3MRLNn4Nz0VtkrsVGsbTVXC1g1AMB+pewB+Mav3ilePbdc9gCsjOSNv9MHSzPFrfVG1SNQlxYWAgDYjxQ9ANG9H6390zG+Xrb8Yy1+FOYZlxcfBYcWtpqbDS1MbRaXZteLxanJ4vrKdAEAe5EiAMS6+y+fX6qK+EQxnnETN/xitnhUdjh2GYyVAjYZAmCvjm0AiJtmNbFvYbW4PLde/F5544/yvcfBy6eiouBk1YuxuDlZvH9/pri5PlUOFegRAKA/xzYAXJzbqCbQxTa9sawvegGOiwg1IeYxtG8yJAAA0K9jMwkwKvctNDaLV59aqZb2/f7Z5qY9v1aO98+01tkfR1FE6NpyTA6cqLYbfu/+bHF9bbqqLGiiIAC9HJsegLj5R0v/i+U4/+nWrn0Z1BUMK2eK6rUvPNiqhgUEAAB6GfsA8Oq5peJLZVf4lfn1aq38pWMyzr9Xv3t6tdq6OMREwVqUGAaA2tgHgOj6jxteLO2LpX7Z1ZsMxZ4GANDL2AWAGMv/+sW71U59f/j0UkF3rz1zb9u/xRyB7+oJAKAYswAQXfwLrU16pg1vDyTes9Wyd+DS3HqxsjVRlRWudyIEIJ+xWgXw17/9y5HctGfc1JsMxUqBH949UQCQz1j0AFyY3miWw21sVUv62J/YZCh6Aj6Yma56BpY2J4qNh3oCADIZiwDwtWceVDequhQu+3N1fr36elhmqZtrscnQbFVHAIA8xiIARJf/hdbsdgBg/8YkAKxa1gYAQ2QufWJPTW9VQwGz5lUApCMAJDbf2KqGVgQAgHyO7W6Af3Lt6WKzvK/99MFMz+/509+4aWjhEMROjLF641tXPxvo56JGwfdvnqomKr59e77n973+7L0qxHztwv1H//b+/dnivfKzf/fe3BO7JMa+CSfL4PPPX7j96N8WNxpVkaT/cutk8aPFua6/IwpP/esuzz+eVzy/WFYZzzeeQ4SqKFE9iHfunii+/eG54qDUJbOfnd0se352PuZ7vXfdvHRqrXpPowT1lfm1vh7349Wp4vP1yeKjleniweaTbZD58nO8XB4v8bjxuXYTe1x8473nev6eeK3xGXz56Qc9NwH7zi/OFm/e2n5MfevKZ8WJxsPihT53D10qn/+Hy9PVttxRejsm1MbOnN00Jh4Wv1G+R3HN+eKZlapq527VS6+X79E7d+d2fNx4jX/x0sfb/v3d8lj+oHxu9THarvN4/uXadHFnbbL4979cqF7PTt/bj17n1JfPL1WrutrP1d3Ov26fyVvla3r78xOPjqFvXLrTqgq7t4nivd7neL6xz0yvY7FdXAPi2Iyfr+49S7PFqDu2ASA2w4kLclzEOFqnpx9WOzS+cmaw+g1xMsXnt9v6hEutrZ87Hz8uyu/ff/IGEN8XF4r2742L48LUTHlR6V0T4fRU9+d/Y3X6UQ9K/BnPJfajGPS13lg52FNx0JLZ3d67buLiGK/55YWVMgjsftOMx43zcuNh3BC3//+piebOnVfKi32v9/D6Lu/VhZmt6jnFz/cKAG983H1ScbxHsdy43xtJHKPz5c2pvsHu9DlOTTSPvzg+4vdcLP+M5bg7ie+/UQamnR53ocexGc9t9eFE1x6+zuP5g6Wt4tZMo3otu31vP3qdU5dm1x99Nrt9b63bZxLHwLtTs4+OofiezvN6EL3e53i+C32+/uZqqmYQHJcCa8c2ADD+4sT74+dvF9fLE/NiedE4isJFdenpXptMvfHxqUfPaRS3nI7WW7RiYnfMfi+O8X3x9dXzD4pb643imz95ZlsLMi7i0Sray+N+sDRTPW700i3enykGFTe0+L1xsY2L7lFa6Nh59I3ZU0XcQuubQbu4QUUxs0FEqIqv+nGvlT0yN9f7u2x/+eml6ivOm+sro3upv1D1hCwXP15sNtq6vXeH7Stl71GEij/6wt2+z+v43vDKmRPV+fLNn4x+41MAaBMHX7QLNnZIb/u9yEfBnc2HT/7OTlPlBS4ut5Fuo0V0nHW+H91aK3OtVuEoFYEapIxy52vs+j3FwbQYykZd9d7NdhxH3Z575/EWx2H8XHwm8dX+MzOTW/t+3LnyMTofdxgWpjaHGsZ6fX7djtX4t26/uyph3tjq6/F3etwoiFbs3GmwzUyXz28Ydnq8tYfN3ofNh/3fyGcn9359XY3S5g+3nnhOva6f3Z73ZkchtJken2M/x3f7OROf6ygXWRMAWurhgmbXZ+9WSbRI9+N2OfZ5bfnx40dZ3s60WxU9KlN/jBFe7XMsclzFeHB0vdWiK7nzphLvQ6TrxfK922kuwGF6u+yujJbK9dXpXb83xlSjxbvb9xyEq2WXf90yaRfj8asdF6bO4+1y2T0dX1GHI1o077T1vlyY3uz6uHFsbzueO7pv68f9ndOrVVXKd4bcqxNj/3vtCu6m8xittbf8ay+fjq7oxeq4aO/diF6kXje3zuNjp8f9i5+fqcarBxFbpcf1bdjvc7fPuhbHy7Xydd1e7//m9+q55WruR+d714/3H0yXn1HjiWtHr+tnt2O/s4cr5su81jHu32tIuXNYLa5h8Xhx3oxCL9VOUgeAD1eak3fubTTHJa+VXZMxoeTGDidYTBQJMX43NRmTlrYepb1eovstEvHNtcnyhJgqLyiNJ57DUsdEqNvRUigf7mx5kb1V/kxzrPB4TVaMVBwliOPC2h64Ymw40vQwL+D7FeO056efvHgvbjaq46SfVtVa+VrvlZ/xp+Xnvtbj+392AAEgbjjzje5j3XHcRRhtF8MsVdntXY7nnXy4PFXdMNvFhfji3Pq2xz3Z6kXYixiSuTkZVSyLA3evPE7jHI7jtb2leLrsaYjXcPnE3s7NuDnE1/8rP/v2Lvp43PDSyeGE/2fK68i98v16pxiuuMnFV+f7Ej7faBSflMdBnCf9qsq9b20VZxqbA5cov1U+j3vl73y/rZel1/H8oOwtWC7Pw+vLUwM/326Nw2aLf6s4WT7/RlsAidcw6uXrUgeAP/vwqWpy2iDJ+LufnHo0Lly31HeazBMn+Pc+Pb2vrXj/bbVaYfdZqOOk7gl5o3w/3+jyvnzwB9eKURCf8WvP3N/27+/cnive+PRUPw9RXtwbVU9PvM7DGouNC17Mzn+ux4S/P/vw7LaWZMyej9fbz+S0Xv7DLxa2zdOI1n+M83Y+7gvzzRAds6cHEc/x9efuVxfjw+gRilndcZx2jk1HSz1e27eufF7sRRwTce2J1QjvtbUS43Hj/fr2b35aDMOrTy2XrdLV8ngd7lbgMWkvPoN3781WXfD7FRPuivJt+M2yFyBuo/G4/QaAv7lZn4uPz8lex/PfLU1Xn+Ogq24i7MS8lU7f+NU7xavl84yWfx0f4vyL3oy47v/objGyUgeAO+VNqLGHsdd6okqEh5tlq+43Tq5XLbyLrRmjt8vuvBj7+b/lCR6t++rCsT74CdKcMbxenG4og3wUovvwYkfPS1w4Yub/4lb/LZtREs9/betwN3+Kraeji/up6YM9jqvVIOW5MmtbiyecK3sUYqglQkVclz5fH+1j99fKLvVoSMd1c3X05tUeK6kDwF67XSMA/KBtDfEXYyb02ZXWuuO14uO1qepC+5c/P7uvpYhx848Z3JdOqFVwFKLl1Dn0EkMWb342X9xYHs8AcLM8Nm9VAfXw7pJxDkQ9gfPTB3scx4qHmLl9fsb50u651gqWWFoZn8WoB4DorYh5C9+/1V8PG3tnEuAQfBCTYcqxo+i2utGaV7AScwqWZ3ad/b2TaH3GMp6Lsy5oR+EPIgB0vPfRbR5d1tdXnTpHJdbTX23dzOLPCNu9lsZF0In5DjFX57nk51E0Ju61amuMsuhJvTA9UQWWhcZ4Bu1x4So2BPVkmGixT3T8+34sTDXHcTkaV+fWtvUA3NuY6FmRjcMRE61iYuH59Y3qz3Czx5SFjdaW1wuNifQBICYV7veadBjqiaF2gD14AsAQ/bhM1jGZJ5Y4zTf2PnhVl5LtthyIg1eXkm2foR4Xzv9+e7744d3jW1nyXrVkaaZavjU78fh1Vstjj7gwS0z2i5nef7hDed/YNfSrz9yvlihSVEORMR/pchlko8JfNCbOTY3eJb8qBVz2qEVAOzc1up9d9DzFe1jPA6rFqpc4b9qrCC5tNRsKt0d8uEUAGKI6XcekvY19dP3XJWXPjGBluQzqUrLtYt1wdPsf9Y3wIEWXerfXVy9XO0oxb2G3YZcIbZcMlz3yoPW5NSd8NgvbLG6M3jUlJvrF8zw/PdqzNxutgmSdS3ljuWLnOVKfSw82R/s1CQAt1dKiZ+9Vf0bxlNg05bmZ9XL8cKZaLfC3SzPVB/2dG2d2fay/K7+30eNzj4R7uXz8mGAWs11/fX7tUW9BrCSIpWJx83ntmXt26TsisVFJFPEYxdK+Bynmsly/sX1i7CiEnr99MF1t7PVPn3t8YsXN/usXF6temQ9ubB+WieW3cSGOQi0ZRbGqqFvyr174/NGxHNeUOLZjG/BRET1O37mxUPzRxbtPFNR5/dn7R1ZT/2ex10DZ8xXLPOviQs3l39uPt7eqnsHtS8nHYT8AAaBolemcarb6YjeymHT3XPn3WEO9WY7qn5xsVC3AKqW2DtBba73fukjcnT0AdTGKC62iKHHxqn7P7MajkzMeP55DzJbOdvMZBbN12dwuBWpinXO9hO64GqS88WGLc6p+fvFVh+P4nGJ3x9mqfPaTJ91iq8BXVlHULHolVzpWfMQ1aJSWFtet5aXWRlH1ZxtDOfXn1zjkxtDSZqMacuoUz+np6eb1of14HNfjTAAoHpfp7Fba9FE51IVmZboXWxNpYoOUQcRJFz0MsVTpD5/uvlVsr+fA4Yix//h8uq26iIqM/fT+cLB+8mC2HA7YrFqxtXPleRP/fc6ksSd8tNxsxd5r68GpW7ELjdFrYHxU9gScbM1VqIqsnTq6CdBvfz5XjuFPF79THlf1ZMSq9yRKlZfvXTTwjmJzsmFLHQCqMpGNh1WVqH5PiPi+1bJ/Pw7Oei/w3VxqtfibJWVdpNo1e14ed0fGvt9TXTaYidC1uHk0XdFRve/m2t5PlbpnYaEqcfrka4hlog92eF1RarbR58sepHTqcRDDbHE+1j14nZtF3azKw05WkwMzi+tUtLDrXq14z7pt+zuoupTzQvlYq5NPXj93O667ubvZ3Er3ygjsf9Ls8XtYTaSM60/7nKAoqhTXrOtzU8Wl1fVqEuO4zg1KHQC+9syDnqVee4kb/5XygPhWedBHGcxupSHbNecW3K8CwCC/J4tqr++27vYoAtK5GdDN9UY1nvv+ESy/ay/lvFcXy2GjeE3xWk93DC3stlnIr59arypO9mOQ0qnHwcl6k6HyvY1Z5J2TZr9/82R1If/qhUPYMGCERSMlbvixqqgqET2klnVdyrlbF3h0n3+wPNj5Gi3q2K+gLh19lKoJp+XXf/701KNtyWsxfyK+6j1aqlLOG6O74c9OUgeASJrP7OFAi+Ug0ZI/bZx+z+JCFO9hzKw90zYe+VzZK1NPoIwiLrFnwI3yAhY3yX73Qd/rcwnPtHqFatVnXR4j+0n4C62lTXEBvjL/5LhizCWJlxuFpNr3CYjJonFxfeXM0q7zQW6Xj7F5zEunPjW1Wd0Uonrn3apV/3gIYLZVF2D2gBphcfOM331rLTaWefz5vdTaB6RdbDITKxZGKYh9UL5n8XSGvaz4ytxa1asVRZY6B81iY7U4pgeZh1HPsYlz4lZ5Pp4rx9obR7wd+rWl6aoHL86t6GFq310wrhkxZFiftxF4xm0uQOoAENtP7mWXvTgor86vlSf68V0SdtCa3bY7d/XFOFucVLHGtr308kE9l6td/l/1WZ9YL9b2sZwnbh7xFcdMp2hpxO+PNcPtAaCeHPr1Lyzu2hqqN6k5zqVTf2Vus9gs1ov/UbYS6xU7tdOtlTunD2hcO/YwuNraUndx4/FnEZX1OocOPy+vCVH2eHmEln/9eHG2ek5fH/L8ouit6yVWIMTxHDfQm1v93Wai1yBCfnXTjfH26ceb6xyVt1qbTdXzEtoDQN0TEBEl7iOxgkYAGBH1CXtvh5bb1D4/q2gZxAFxuxwD6lVfOw7kKOc7s49ZrNVY5i7bVMZF8ML08VoD3Vnydaf3+aCfw6gX9Mgiyvp23nTrC3PdIxc77DW3+B5OQI9wEVU5owW90naBj9/ZGIPrfRStubX+sCpNHvNrnp8b7WWRscwugnHcdGenRmMpdJT/Pl1tzrZRreJ6sW2r5tj171IZUCPwRK/Hm7cOfofKYTm2ASBaTXHjOL9D62m/3Ut1SdJqg42idwDYbznfmGRya5cbULySC8PfUv5IdZZ83el9PujncO6YhatxdbvLRkbVEtu28zzmA8TFuH1J2X5Ux+AYrzCoh6+iIRHvx6gHgOpGOmJLOKOHruqpexDzGh4+EQAutpZ0RxCI5y0AjIDXyi7CGIf76oXeE+8OY6LJRmtsNor9XN7jifdJeeL+tBxTe+vz+WpCXDdfOrdU/MsX9rYv+VGoy3/GidU8qZqitTVXXqRea+vijbW3MZGyuQnP0aScupRqZxnQftQlWT8sL2xLHT1S0aMR70GvSYbxmud2uYnF+9esRJZjAuAwzqlhi2P5nTsnqmVhUXxnFGvuv/HJqar3pH3+xH58WPYoRMnb6LLvvFn/+N5s9Rkt7WEo5KOylyd6Vt5vLfkclf1Q4jVGobb3Ts5U16+YrNgeBKJMdajD5/dvjv6Q3LENAKNSEnSzVUCo2EdD5EGr1GRUzLre40bx0smjXzoziLr8Z9z42mf3z7UKJrWrSyMvHGGd8F5lQPtRl2T9uEsp4bqEaK/WTtxYdmvF1i2mLIZxTg1bPJ/otr6x0uh5jh61eF7DbPQstQr4xOvuPH6rZbt7PCbr5YPx86NUDTVeY7yHMfExXvPFjvBZ33PiWjUu56NCQPtQb625ecDHaCzjiYk8DzaP30X+4/JC0b49ab0kLiZo1sMnC61CLzGRKb5iHPggxuTfvjNftWai7Gy819+68tmjC+ZuZWd38lF50YgenP/06ekyBGx/3nFh6TyG6nLSOy0RrMXP1pXyMojXGUseo9u125K2v/n05KMNjIZxw4senAhvPy2PjaW2c/B/Lz7+bD5ZjV662ZH+DOI9iaGM+PN0x4S2vfibmyer8yUer/N1D+N9iB6LuAaM2qZodcns2Pjt/QfLVQGx9nLTUcwtemrjmh33iEGXQx4mAWCPFluTjPo50GN2a2MPy0Tr9bV1K7Fb0IhJPXGjaIxSc2gAnTeu+u+d72uzlHKz5OvUkF9rPId4b++sNQuRdGupx3scQxEn9zDTPLqs4/XUm4b0+5zqn2O7eB/PdczirT/Hta2JobbA4qYf53DncdE+bHN7DIrBxLHUXGYXIXT/59Dja9PE0CZctqvnLIya+roc72NcL6Ixsjq78UR56nr+yeqI7+ciAOxBHPR/9YszfRWHiYP4j3/yTJVi//q3B0uy0eUfY8D/rRz7f69HS3C+Vdjj8ghUzxpXUWsgZkj/x09PVdvO1tpvvnHjjzKg15adMkctPpdoXa10ZLHoKYvJsnc3h9s79OPFmWrst15uOc7i+f+7GwvVNWPUKyT20/t1lOIYjK+40Uf4rMoET4xXQ8zVbECR+GJJ3o2yG3qQiT5LZUKOyVpRPvPyiZ3nJ9Slb+N3xNfiDhe02CQjEucwSnvypM4SqqEqP9u2EchxEiHn+bIrMy5oc20Xsii+s7R19FsCt4vncnt96onx9pgvEeeNXpOd1RvvjIt4vtHTeZjXuLp8e6dBav/HdWLU144IAAOKsrBxQH73k8FmeH5YjpX9mw+ersaLvvGrOxfkqEvfxuzw6AHYSWyeEctPvmAf9KGLcfu48T2x8cx0c1byIAVOxsXfL1swcWxebO1bUYuNr+LG+s6IbHwSN/hoHUYQj/OkFvND4nneGsEZ+KOifu/GabfR6vk2DndzoG8+f+eJlUghwuaX/ufzff18hOi4Lkcv8Y9GeH83AaAlToxrXSZrxPhWe/GPZrnVwVsYsZ1wXJji579XhocvzG0+Ubs8KojVWwhHKcz4vmjR9BKz0p+a3qpmovbawY79qZc3tQeAuBC9VO0IWbY2c24zPzI691H4dHWymnS1qTMM+iIAtMSF/s3PthdwqFsVtb22guLxYzZofMX60K88/aDaHKYWY0l1sNhtg5gQ859iL4MXd9hemP1568582Q24/kTZ2YtzG1UJ2Kivfm35mFVeGjNxnvxgjIquwKgRAEr/texmryo4delu76ciVdQKP9e2Rj2K1ez2MzGpr72ozedVhbPm3/tpwVQlhssb0QXbCx+YD5enis2OntK67GyUmr6xstbXZw0wigSA4nEltff2uN1s7IbVXio0NrTY7aaw32Ih1RjTybVtddEZnljec7KxvShRlJ09P90sT9zPZw0wigSAIfpH5x8Uv3dmpfjLn5+thg4OahlLtPwvVxtSrFazY8dZDIO8Xtzb/u8j8tqGUXb2udkIaWvFPz5/v+dmRjFZ6IeHMMkudhm8MLNRfO3C/W2z+n/39GpzA5Z9rF1+9anlapikXVRGiwlVwuqTLkxvVaVkYwlZ+3DgK+U1ZBwm6b1Qng8RfXfaUTWKBMW18DiI8+L1Z7dfq/7h2eXmeTOG12IBYIheOLFetcqrluEBzkSOcelfaW0XO+7iNXS7gIzKaxtG2dnoIVqYmihvvps9lzId1hK7upBRt3KlEQz2+77Xm3C1i+VUo1Kae5TMtJbwRihrD7vx2YxiAZxOJ6cetrbS7n3MvH//+IS+uiR5pxgCHtdrceoAUM/o/86NM8VeRIvmyvxa2epfLX5vYeVROch/cfnz4sHWZNUTEBfZKCKyH9HyjOVn/+SZB9WY/++XiXMc02a7c2Xr5+WpleK3TjUruPXjk9WpQym93C4+v+9E4ZQy2LUvQ/rSuZXi6vxG8ac/e2rXm/fl8riIZXW/dWql52uNiaG7LfkcRLNc7lx1YXr33mp1k3lu5vHF67Vntrdket10Yr5KZ9Grxc1mGezOx/1q2cvR75BIXbI3g5jk27mjYN0b03lcdPscPqlKZs8WdzZGZ4njTp91c/Or6eJHd8fz8/3ZSvNaE+d8fa2NcynK/Hbq9nk1rxt7u68cptQBYHVzolqet5/WV3z4JyebKXi29TDRyptvDC8VRusgfs/pRnOuQfW7xjwAxGt6XMm1v9dyVPXuq3KnHbua1a23flpq/bzWucnhtyDqYkXxtdHxvg1ybK5tdS9J3O1xZ7ts5tT7cScOrefjqNVleNsNcg7UZaE3RqihudNnXZfCbYzpdWqzx7VmkPNmHI5tQwClb16+3Xdii5K+cRDExjC7Faf44+dvb6sk109xn/pxYzghxvujZ+HCTN4u1Nh563tlL8o7R9CaiItAt6WfV8uen6tF63hobFats1EUvVzf/vBsNXZ5acA5DH/x8zNVaeRuw1k3lqerx/1iOV79ytmVgY7RWG77fscqmOMujqG4Dvz5R+equh39Lt2N9z7qk8TGOPvtSaR/b96ar3pt/ll5nY+NkwbZkCg+Y7sBjoGY2DVbJr24MMb+zl8sP+SNYqLafrfTqVaa/c1Tzd3p4ibdb2nK9gtv3FDubSwXKz0S5pnycU+Ujxu1/eOiGr9ndnK8x9GWYrOQiWYp5EFFYZe6LHKvksjx/+5NbT7x+Nfj+8ufXXm4ffvdaJm0f290V1YbvexQHrUu5bzt3zuOlXgu0VMx6Gv92QHtGBYtx8XWjmQRoGbKpzu7y2S81fJ9WHtYVDee5iZU24/T6MxubgU7XSzcb5ZGvrk+2dfjvn8/ilx1Xz5ZP+6t9akn3sObq42eG2LtJlbcxGqOh22t0Zs9JmN2+976WBrG0FMcH9F7FzeVfj6LKApWVZ1c2/lS3e14u7nDjpmdx/ON1mvs9pn0OvZ30uucimJoEx3P94Ol5nG2+rD78bPXzyQ2KIqfrTYtajtP+ykXvVqVvn5YTWCsel/72Io8yivH82leq8YjAExcuXJl5Pto3voHH+0403SYehXhqbelHZY46G91nKBxoLVXnTtocUOIlkm0LA5jD/NBt/Wsx7H70eu9qy6e61N7/t7abr090QNQ74i2l7KlcWE8jG1Do4bB+V22yI3x/kFb58N+3JhMeKXLBlft73M/Bv2sux2je3k/+hGTeS/ucl3r57gY5vHcbcOj/ZTh7ef82+017vUz6XUM1QYp6tbPtWvQY3MUGALoEAfn+S7FdYa9JG1h6uB/x6gZtEZ7v5MDm9/b/fG7tSoG+d7H/29ix+ff3kre7Xu7WTykyV1VL8ouG0cu7eEiNuzHXd3q/h5uDnBMhEE/627fu3RAF/V43N2Ok36Oi2Eez91a1Hs5nnd6Dp3Pd7fXuNfPpNcxtBf9PM6gx+Yo0AOQ2GH3AAAwOsaiv+LjsqsnxnwYrmoMd0UlO4CMxiMAxCSwVS3UYRMAAPIai7tqLBuKsflBJ5HRXYSpd+6cKN6+faKa9LO0KQAAZDMWASBme85OqCM+LM3lW1PVxJYshVgAeNJYTAKsq01VtcobW8Xrz90vrsyvF6+e7a+YBs1NOaK7/wdlb8onq43ip0uzj6rEAZDPWPQA1DepmAgYa/Gj1bq2e10G2tSlSKPVH0U6tPwBchuLHoBuopxmlMmNQhqx01j2crm9/NUvzlQ3+29/eK4AgNrYTq2PAhNRcjEqL91rlSBd3SqHCWY3x36jnP2ou/VjJ66Y3BeVuMatOhUAB29sA0DUaI6v2leeflC8eGqt2q5xdipvAIhhkigx/CfXni7ev3/wpWUBGE/HZnF9VRd6YqL4/s2TxdzkwzIMrFdbtl49sVYcZ83NhSaLu5vNNf3XYo+Bcpzf0j4AdjK2cwB2E1v8xoTB6BE4zurNi5rbDM/vuAEIANSO7Z3inTtz1Xa9pxub1Z9PTW8WF2c3jsWeArHN5oOyxf/uvZlqI433H8y0tsydOrCNSwA4Xo5tAPjh3RNV/YAHZVd4bAt5dX69qiR4XAJA7PUdm/j02r4YAHZyrPuKY9vJuDleW37YKoQzXa0ceHlhpVo2OG7evj1fje9/t7zxRwCIv2+M4RaUABy9Yx0AYn/mxY3HN8hLs43i+txU8eKYVr+L2f1Rwje6/BXyAWA/ju0kwG5iSCC+YnXA3ORW8fqz90d6omBU7Iv1/G/dPlEtefw/92aL2xuq+AGwf6mmiz9Z+370b6QxhFEt81ufbJbwdfMHYEhS9QDsJHoCvn7xblVW+LVn7xVHIW72MVchZvN/95NT1U3/g2XFfAAYPgvG20TrenFqsvpzqhwmaJT/1ig7DKYOobRw/M7V1mY9d1t/3yhM8APgYAgALXED/s6NM9Uywfj7YW8y9L1PT1eV/OI5AMBBEwA61Ovqf7k6XVyb2ijuLUQYaBRX5teGusnQjdWpqohPTO5b2yqKdxdni8XNRgEAh0EA6BAB4Ae35h/9d73J0GuNzaEWEYrSvRE0ophPtPwB4DC58+wiJuFVcwPKr4XGVvHK2eVqKeHLp1cHepwY07+5NlWO8U+Wjzld/K/FE8WHy1Nm9QNwJASAXVTbDhfTVWnh8M2J5oqBvQSA6Pa3aQ8Ao8AdaEBvfnayKiQUN/LYY+B3yyAQ8wO6lRa+VvYexI0/CvnUcwviz1jmt7ip5Q/A0REABhQ3/hCT9mJOQAwHnJ7a7BoAYoe+uOG//fkJm/YAMFIUAtqnGA64PLdePF8GgBdObFTbDv/tg+lqu953WsMG91rj/KtjugcBAMePHoB9ipb97fXJYr7RKM6UYSCKBkUN//j3KOoDAKNIDwAAJGQmGgAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEICAAAkJAAAQEL/H6pltxi4izdYAAAAAElFTkSuQmCC

.actor/logo.png

import { createHash } from 'crypto'
import os from "os"
import path from "path"
// eslint-disable-next-line @apify/apify-actor/no-forbidden-node-internals
import fs from "fs"
import pg from "pg"
import pgConnectionString from 'pg-connection-string'
import { config } from 'dotenv'
import findConfig from "find-config"
import { Client as ElasticClient } from "@elastic/elasticsearch"
import filenamify from 'filenamify'
import { Configuration, Dataset } from 'crawlee'
import { MemoryStorage } from '@crawlee/memory-storage'

config({ path: findConfig(`.env`) })

const elasticIndexName = `actors-monorepo-shops`

const globalLogsProps = {
  __NODE_STARTED: new Date().toISOString(),
}

let actorName
let pgClient
let pgClientNormalized
let elasticClient
export async function init ({ actorNameOverride }, restInput) {
  parseEnvFromInput(restInput)

  if (os.platform() === `darwin`) {
    const filePath = process.argv[1] // ~/Projects/apify-actors-monorepo/actors/foo.ts
    const basename = path.basename(filePath) // foo.ts
    actorName = actorNameOverride ?? basename.split(`.`)[0] // foo
    const gitBranch = fs.readFileSync(path.join(process.cwd(), `..`, `.git/HEAD`), `utf8`)
      .split(` `)[1]
      .trim()
      .replace(`refs/heads/`, ``)
    const gitCommit = fs.readFileSync(path.join(process.cwd(), `..`, `.git/refs/heads/${gitBranch}`), `utf8`)
    const gitCommitShort = gitCommit.substring(0, 7)
    globalLogsProps.__GIT_COMMIT = gitCommitShort
  }

  if (process.env.APIFY_USE_MEMORY_REQUEST_QUEUE === `true`) { // dotenv -> bool-like vars are strings
    Configuration.getGlobalConfig().useStorageClient(new MemoryStorage())
  }

  if (process.env.APIFY_IS_AT_HOME) {
    actorName = actorNameOverride ?? process.env.APIFY_ACTOR_ID // Name would be better, but it's not in ENV
  }

  /* ELASTIC */
  /* ======= */
  if (process.env.ELASTIC_CLOUD_ID) {
    elasticClient = new ElasticClient({
      cloud: { id: process.env.ELASTIC_CLOUD_ID },
      auth: { apiKey: process.env.ELASTIC_CLOUD_API_KEY },
    })

    // const mapping = await elasticClient.indices.getMapping({ index: actorName })

    // eslint-disable-next-line no-inner-declarations
    async function enforceIndexMapping () {
      const doesIndexExist = await elasticClient.indices.exists({ index: elasticIndexName })
      if (!doesIndexExist) await elasticClient.indices.create({ index: elasticIndexName })
      await elasticClient.indices.putMapping({
        index: elasticIndexName,
        body: {
          properties: {
            _discount: { type: `float` },
            originalPrice: { type: `float` },
            currentPrice: { type: `float` },
          },
        },
      })
    }

    try {
      await enforceIndexMapping()
    } catch (err) {
      if (err.message.includes(`cannot be changed from type`)) {
        console.log(`Elastic index ${elasticIndexName} already exists with incorrect mappings. As existing mapping cannot be changed, index will be deleted and recreated.`)
        await elasticClient.indices.delete({ index: elasticIndexName })
        await enforceIndexMapping()
      }
    }
  }

  /* POSTGRESQL */
  /* ========== */
  if (process.env.PG_CONNECTION_STRING) {
    const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING)
    // const pgPool = new pg.Pool(pgConfig)

    pgClient = new pg.Client(pgConfig)
    await pgClient.connect()

    // Check if table exists and have proper columns
    const { rows: tables } = await pgClient.query(`
    SELECT table_name
    FROM information_schema.tables
    WHERE table_schema = 'public'
  `)

    // eslint-disable-next-line camelcase
    const tableExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
    if (!tableExists) {
      throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
    }

  // TODO: Handle pgClient closing
  }

  if (process.env.PG_CONNECTION_STRING_NORMALIZED) {
    const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING_NORMALIZED)

    pgClientNormalized = new pg.Client(pgConfig)
    await pgClientNormalized.connect()

    // Check if table exists and have proper columns
    const { rows: tables } = await pgClientNormalized.query(`
    SELECT table_name
    FROM information_schema.tables
    WHERE table_schema = 'public'
  `)

    // eslint-disable-next-line camelcase
    const tableMainExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
    // eslint-disable-next-line camelcase
    const tablePricesExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_PRICE_TABLE)
    if (!tableMainExists) throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
    if (!tablePricesExists) throw new Error(`Table ${process.env.PG_DATA_PRICE_TABLE} does not exist in database ${pgConfig.database}`)

  // TODO: Handle pgClient closing
  }
}

// inspired by @drobnikj
// TODO: Similar, but less obfuscated for easier debugging
export const createUniqueKeyFromUrl = (url) => {
  const hash = createHash(`sha256`)
  const cleanUrl = url.split(`://`)[1] // Remove protocol
  hash.update(cleanUrl)
  return hash.digest(`hex`)
}

/**
 *
 * @param {Date} datetime
 * @return {Promise<void>}
 */
export const sleepUntil = async (datetime) => {
  const now = new Date()
  const difference = datetime - now
  if (difference > 0) {
    return new Promise((resolve) => {
      setTimeout(resolve, difference)
    })
  }
  return Promise.resolve()
}

// TODO: Uff, nicer! But at least it's tested
export function parsePrice (string) {
  let amount, currency
  const noText = string.replace(/[^\d,.]/g, ``)
  const decimals = noText.match(/([,.])(\d{2})$/)
  if (decimals) {
    const decimalSeparator = decimals[1] // ?
    // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
    const decimalAmount = decimals[2] // ?
    const mainAmount = noText.split(decimalSeparator)[0].replace(/\D/g, ``)
    amount = parseFloat(mainAmount + `.` + decimalAmount) // ?
  } else {
    const justNumbers = noText.replace(/[,.]/g, ``)
    amount = parseInt(justNumbers)
  }
  return { amount, currency }
}

export function toNumberOrNull (str) {
  // TODO: Handle better, but only after adding test
  if (str === undefined) return null
  if (str === null) return null
  if (str === ``) return null
  const num = Number(str)
  if (Number.isNaN(num)) return null
  return num
}

export async function save (objs) {
  if (!Array.isArray(objs)) objs = [objs]
  if (objs.length === 0) return console.log(`No data to save.`)

  const objsExtended = await Promise.all(objs.map(async (obj) => {
    const objExtended = {
      ...obj,
      actorName,
      ...globalLogsProps,
      // __NODE_VERSION: global.process.versions.node,
      // __NODE_UPTIME: global.process.uptime().toFixed(2), // seconds, 2 decimals
    }
    // if run on Apify
    if (process.env.APIFY_IS_AT_HOME) {
      objExtended.__APIFY_ACTOR_ID = process.env.APIFY_ACTOR_ID
      objExtended.__APIFY_ACTOR_RUN_ID = process.env.APIFY_ACTOR_RUN_ID
      objExtended.__APIFY_ACTOR_BUILD_ID = process.env.APIFY_ACTOR_BUILD_ID
      objExtended.__APIFY_ACTOR_BUILD_NUMBER = process.env.APIFY_ACTOR_BUILD_NUMBER
      objExtended.__APIFY_ACTOR_TASK_ID = process.env.APIFY_ACTOR_TASK_ID
      if (process.env.APIFY_DONT_STORE_IN_DATASET !== `true`) { // Note: dotenv is not casting vars, so they are strings
        await Dataset.pushData(obj)
      }
    }
    return objExtended
  }))

  // if runs on local machine (MacOS)
  if (os.platform() === `darwin`) {
    const cwd = process.cwd() // ~/Projects/apify-actors-monorepo/actors
    const storageDir = path.join(cwd, `${actorName}.storage`) // ~/Projects/apify-actors-monorepo/actors/foo.storage
    if (!fs.existsSync(storageDir)) fs.mkdirSync(storageDir)
    const dataDir = path.join(storageDir, `data`) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data
    if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir)
    for (const objExtended of objsExtended) {
      const id = String(objExtended.id ?? objExtended.pid) // ?? uuidv4()
      const fileName = `${filenamify(id)}.json`
      const dataFilePath = path.join(dataDir, fileName) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data/foo.json
      fs.writeFileSync(dataFilePath, JSON.stringify(objExtended, null, 2))
    }
  }

  if (pgClient) {
    const objsPg = objs.map((obj) => ({
      ...obj,
      // TODO: This is becoming not nice, and not clear
      shop: actorName,
      scrapedAt: new Date().toISOString().split(`T`)[0],
    }))

    const columns = getColumns(objsPg)
    const values = getValues(objsPg)
    const queryString = `
        INSERT INTO public."${process.env.PG_DATA_TABLE}" (${columns})
        VALUES (${values})
    `
    try {
      const { rowCount } = await pgClient.query(queryString)
      console.log(`[save] saved to database: ${JSON.stringify(rowCount)}`)
    } catch (err) {
      if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
      else throw err
    }
  }

  // Only make sense for HlidacShopu
  if (pgClientNormalized) {
    const objsPgData = objs.map((obj) => ({
      shop: actorName,
      pid: obj.pid,
      name: obj.name,
      url: obj.url,
      img: obj.img,
    }))

    const objsPgDataPrice = objs.map((obj) => ({
      shop: actorName,
      pid: obj.pid,
      scrapedAt: new Date().toISOString().split(`T`)[0],
      currentPrice: obj.currentPrice,
      originalPrice: obj.originalPrice,
      inStock: obj.inStock,
    }))

    const queryString = `
        INSERT INTO public."${process.env.PG_DATA_TABLE}" (${getColumns(objsPgData)})
        VALUES (${getValues(objsPgData)})
        ON CONFLICT DO NOTHING
    `
    try {
      const { rowCount } = await pgClientNormalized.query(queryString)
      console.log(`[save] saved to database (data): ${JSON.stringify(rowCount)}`)
    } catch (err) {
      if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
      else throw err
    }

    const queryStringPrice = `
        INSERT INTO public."${process.env.PG_DATA_PRICE_TABLE}" (${getColumns(objsPgDataPrice)})
        VALUES (${getValues(objsPgDataPrice)})
        ON CONFLICT DO NOTHING
    `
    try {
      const { rowCount } = await pgClientNormalized.query(queryStringPrice)
      console.log(`[save] saved to database (price): ${JSON.stringify(rowCount)}`)
    } catch (err) {
      if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
      else throw err
    }
  }

  if (elasticClient) {
    // .index creates or updates the document
    // .create creates a new document if it doesn't exist, 409 if it does
    // try {
    //   const res = await elasticClient.index({
    //     index: `actors-monorepo-shops`, // TODO: Consider using actorName
    //     id, // foo-bar
    //     document: objExtended, // {...}
    //   })
    // } catch (err) {
    //   // https://discuss.elastic.co/t/elasticsearch-503-ok-false-message-the-requested-deployment-is-currently-unavailable/200583
    //   if (err.message.includes(`requested resource is currently unavailable`)) console.log(`Elasticsearch is unavailable, skipping, but not aborting`)
    //   else throw err
    // }
  }
}

function getColumns (objs) {
  return Object.keys(objs[0]).map((key) => `"${key}"`).join(`, `)
}

function getValues (objs) {
  return objs.map(objPg => Object.values(objPg).map((value) => {
    // escape strings to prevent SQL injection
    if (typeof value === `string`) return `'${value.replace(/'/g, `''`)}'`
    // convert to DB specific null
    if (typeof value === `undefined` || value === null) return `NULL`
    return value
  }).join(`, `)).join(`), (`)
}

export function parseEnvFromInput (input) {
  const env = {}
  for (const key in input) {
    if (key === key.toUpperCase()) env[key] = input[key]
  }
  console.log(`[parseEnvFromInput] ${JSON.stringify(env)}`)
  Object.assign(process.env, env)
}

export const isInspect =
  process.execArgv.join().includes(`--inspect`) ||
  // @ts-ignore
  process?._preload_modules?.join(`|`)?.includes(`debug`)


Bike Components (bike-components.de) scraper

Output example

Where next?