![Bike Discount (bike-discount.de) scraper avatar](https://images.apifyusercontent.com/hN2bDakVMpjItEh793WxcfKRAkRH97Avw4-aa-xveic/rs:fill:250:250/aHR0cHM6Ly9pLmltZ3VyLmNvbS9DTGZNU1hOLnBuZw.webp)
Bike Discount (bike-discount.de) scraper
DeprecatedView all Actors![Bike Discount (bike-discount.de) scraper](https://images.apifyusercontent.com/hN2bDakVMpjItEh793WxcfKRAkRH97Avw4-aa-xveic/rs:fill:250:250/aHR0cHM6Ly9pLmltZ3VyLmNvbS9DTGZNU1hOLnBuZw.webp)
![](data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAYAAAD0eNT6AAAABHNCSVQICAgIfAhkiAAAGbtJREFUeJzt3V9onWl+2PHfZHJRKKwOhJYU4ozAC4W12WPG6TSjgDXIU9ZzMwpSGPfKqqV172I5lm8l45OLBGKplnIXWzJSKNRmZSLfrAevBsuwNhNssMQo9MIXMh7ITVokQymlC+mFfFyPZ+zzHun8k36fD4TAzpH0WnrP+37P+z7v87z3u++//88BAKTyW+3eAACg9QQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASOi3m/FN//E3v2nGt32rBysr2///3r24c/t2rK+uFv7ant7eWFxervm6yUolLlcq3/vfL0xMxNjExI6+th5XZmfj5NBQzde92NqK0eHhuLO0VPd2ttLD+/djoK+vId/r1ldfxcfHjjXke0VEPN/YiOfPnkVExPqTJ/FgZSUe3r8fW5ubDfn+RbZ3p7+frlIpfvb553G4XI5DR47Ej0qlOFwu73RTa2rk3zFie/s/PnYsDpXL0fPJJxGx/R7djcHjx18dI3bjQHd3fHzsWM3f7evHo/XV1VhfW4vnGxu7/vlVRY6vjTjmvK7o8ePf/PZ3TylFj6/nRkbixvz8jrfvTUWPCTvd3lZ7czsbpTnftcWqB4ie3t64cPFiPN/YiMuVStxcWGjzlu1eV6kUv1heLnQQf7G1FQN9fXUFEN93oLs7DnR3R8T2PnVmdDQiIr5ZXY07f/d3ce2v/7phMdAoX5w6FZ/198eJ/v52b0rdqtFycmho1yf7RjtULsfJU6fiRH//q32iltePR1XVfefm3/5tQ2Ngv7gyOxsR0dAIoLZ9eQvgQHd3TM/Nxd3Hjwu/aTvRoXI5vn76tNDJf31tLT46eNDJv4kOl8tx4eLF+Prp0xgbH++IfauntzfuPn4c03Nze+7k31Uqxdj4eHz99GlMz8111Mn/xOefx+Lycvzq8eM4Mzq66791dd/5+6dP48rsbBxq4lWZverK7Oye24f3un0ZAFWHy+W4++jRnnyznRwail89fhxdpVLN195cWIjBvr6O+1S6X3WVSnHh4sW4++hRnDl7tm3bMTY+HosFrw51mjNnz8bXT5/GhYsXC+3jrVK9BHz91q2mBUn1vX1ldraj/u2dQBy11r4OgIjtg/Xi8nJHfFor6srs7KtLYrVcm5mJ0eFhJ/826CqVojI1FdcXF1t+IL8yOxsXLl5s6c9shAPd3bG4vByVqamOO/lVg6pVVyJODg3F10+fdtSVj3arHq9FQGvs+wCI2N6pip5Q26mrVIq7jx8XGuwXsT1wZvz8+SZvFbWc6O+Pu48eteyEdmZ0tPA+0kkOvbwi12knvOr7rh1BVT3htfNKUqcRAa3TtkGAux2leqC7O3p6e+Pk0FCh0Z49vb1xor//e6PjO8WhcjkWl5cLnURebG3F6YGBhoxsflOjRkx3oiIjaQ90d8eBDz6IH5VK8Ucv95nf++CDQl/3i+Xl+JPjx5t6NeZAd3dUJicLv/7mwkLcmJ+P9dXVtl4lqmf/rvry9u349cpKrD95EhHRlP2ynkG2Vd8+exYPVlbiwcrKqwF9D1ZWXoVN9djU09tbaN+JiKhMTcWhcjnOjYzU/W/Yj6oR8B/+4A9aOmjywcrKjkbc7/QpiXbrrK2pw/ONjbixsRE35udjem4uvjh1qubXfNahAXByaKjwFYr1tbUYPX3aYL8meb6x8eqAc2dpKcbPn3/1dEmt0DxcLjc9Av5zHZ8UOyXm6o3bq9PTcXVmpunBUu/J/+H9+3Fjfv6tI9Vf/a5XVl69pui+E7F9HKj+DLb/PnOLi02P6sz2xS2A0eHh+PL27Zqv+9nnn7dga+pTz/3+9bW1GPSYX8s9WFmJgb6+ODcyEi+2tt752sPlclxo4nwLf1jw8vnE2FhHnPy7SqW4MjdX6OS/vrYWnx49GpcrlZYc8C9NThY++U+MjcVAX1/dJ+fqvjMxNlZz34kwCO5N1ajutPEi+8W+CICIiL+6dKnmazppJ6r3fv/NhYX49MMPlXAb3Zifj4G+vpoH8p+fPdu0g3jRE9bV6emm/Px6XZiYKLTN1f27VZd7T/T3F55Y69OjR3f9+7w6PV1o34mIuL64uKuftd+IgObZNwGwlz4V1/N8f8T2eInR4eEmbxVFrK+uFjqQV6amWrRFP6wTDpYHurvj5wVuWdxcWGjp/l3PoODR4eGGHVvWV1cL/TsPdHc39SrSXiQCmmPfBEB1trZ3KVLfzVbP8/0R2yP9GzmlJ7tX5EDe09vblKsA62trhV7XCROqFDmJra+ttTxuT/T3F3r/XZuZafiYoTtLS4VmKC0ypimbw+VyXKpjACy17YsAONHfH2Pj4zVf97DN90Trud9fvfRoQFBnurO0FA/v33/na/5jEx7Ve3jvXqHXXZqcbOu95K5SqdBJbPT06RZszXcVOVa82NpqWnhPnD9f88PIge7ujoi4Vin64ayeAdPUtmcD4EB396vpOotOxPLf2nQyrfd+f3Uw1F66rVGvf/3+b+34/zrF+J/92Tv/ezMGnf7NzEyh13WVSq9mm/v42LGWT4RV5OR1c2Gh5fv46+s8vMvN+fmmjbfZ2twsdGXhs0QBUM+CUiKgcdr2GOBYi1ene3j/ftseASxyH7Tq4f37cXpgoG2D/Rq9EtbbHkUr/4v34y/+1b+s+/v91xf/J/7L//zfjdi0XVtfXY1vnz1767PeB7q7o6tUaujf8vnGRkxWKoXfOyeHhnY8adA3q6vfWQ2xngF6RSb7+WUb3o9FJyH6dZOvFj5YWal5heQniZ4GWF9djXMjI4VP7NV92rwJu9M5H6eaqDpxTqe7ubAQA+b031NqPWrXjMvwlyuVmrcfGuFwuRwnh4Ziem4u/v7p07i+uFj4BFrkU3Y7gvxAwYl5mr1t37yc3Ohd9uIaD7txY36+rhP6yaGhQmO/eLt9HwDVJXI7/aQ6MTZmpP8e1K6lXU8PDLR8uesT/f2xuLxcaJrW3/v933/nf//22bNGblphPZ980paf+6b9fHtvN+qNgMrk5J6cFrtT7PsAuDo93fFvtgcrKx3z3DaNdejIkaZ8363NzRgdHo7JSqXlT7f09PbGr2qMaal1BeB5mwKgiG866HiRcVKgeiPgyuysCNihfR8AYxMThUb9tlN1CVLPuO4/6wUu9e7G5UolPj16NG4uLLQ8BN514K31uOJPfvrTZmxSTS8KXAnspEvvO/3w0uhjSauvnNyYn6/rCpcI2Jl9HwARERcuXuz4naOntzd+YQWsPedwjU/4rbj19HxjI0aHh+Ojgwfj3MhIfHn7dsti4Mrs7A+OC6h1ou0qldoSvEXuvUc0fyKl3ayIWOT2SdEpo4uqdUsnIho+LmV0eLjuCHD8rE/bngJ4eP9+PCj4TPPbVJ+V/VFXV83XXpqc/M7qXZ3o8MtFU4YHB9s2j/vNhYWG/o7edan3L/9HZ4zm36muUin+sMYiL628/bS1ufmdxWqqKxtGbN+K6CrwPql+XT0r2c0tLsa///GPvxM760+e1FwA50R/f8vnuSh6eb/Z21bk1tDbTqjP3/HkSdXhcjkOdHc35L3c09vb8sdIq6rjoopOjLS4vByDx483c5P2lbYFwIN79xoy0UZXqRSVqamaO0hXqRQnT51qy6x6L7a2CkVKxP9fBvPcyEhbJgG6MT/fkvi4+7/+b9N/RrPVmlGu6Kx9zfL6yob1/k27SqU4c/ZsoccNu0ql750wf72yUvPx15+fPdvyfbzoZGCfNTkAijzj/7bHJJ9vbBRaXbAyORmnBwfr3rY3FX3ktFm3u+qJgOrxs8itHvbBLYDqYKgil4p+1qaJNa5OT8fE2FhdX1PPrIG0XlepVHNsyV6exXFrczMuVyqFL8G+eYvtztJSoZUTW/0Y19bmZuGVQ3dzmf5denp7C53A3/YoYtH5E4ouePQuJ4eGOmLuhInz5wsHdVep1LYrFnvNng+AqiKf7Ns5uOfq9HQMHj9e173Zk0NDBgd2qEuTkzUPMjf3cABUFb1i9kOD+oo82TI2Pt7y+7Z/U/CJm0tTU0157xX5RP3l7dtvvXxfJK6qdjMldE9vb10fQpo5d8LW5ub2Uuhtvqq23+ybANgLHqysbE/xW8dObHBg5yky4vjazEzHzz1RRNF7yD90orw6M1PzRFW9ZNvK/bs6s2EtzVh85m2DJt80fv78O/970ZNtdUroM3XMRhoRcebs2bpmBS1yVWW3REDj7ZsAOLlHVs96vrERg319dY1urQ4ObNYlSYrpKpXi+uJizZP/t8+e7ZsVHIvucz80Mr16e66WagS0cvGb0eHhQp+iTw4NFV5r5F2K7jsR2/FYK7wu1zn/Q2Vqavt3XGN9ihOffx53Hz+ueznrWsHSKCKgsfZFAJzo7y90Wa1TdprqgbGecQHVg2S9JU9jfHHqVNx99KjQSWr8/Pl98em/q1QqfAn4bYMMiy5/Wz1BXpmdbcn92+cbGzFR8KR1or8/vn76dMdL9Naz7zy8f7/QyfT5xkbdk4f19PbG9Vu34r//0z/F4vJyVCYnY2x8PK7Mzsbi8nL8429+E9dv3ar7VmmjnxyqZWtzM04PDHTE8u57XdueAmiErlIpfv6nfxoXLl4s9PpOG5R1dXo61p88ieu3bhV+SqAyNRWHymWLYLTAx8eORU9vb5wcGip8Ujo3MtK2Raca6YtTp6JSxz3wd723RoeHo6tUKrQ6YnXxojtLS/HLl0suN+vkUt3mIpHTVSrF9NxcXJiYiDtLS/FgZSXuvOWyd1epFD/56U/js/7+ONHfX3jfWV9bq2vNksuVSpz44z+OQ3VOqtRVKkVPb29Drii+2NoqHFKN9HxjIwb6+uLWV18VPnbyfW0LgC9OnYqPd7kD1rMDv9ja6sgDc3VcwPVbtwq/kU8ODcWhI0fiT44fb/gnzUtTUy15hOYfVldbdtmwqug9zUPl8o4u+V6bmWlZZP751FTTVour99//5e3bNR8zHB0ejum5ucJLJJ94efKs+mZ1dcf75cT582+dj6GeCIjYniPhzOjod55eqG7bj0qlHQ80Xl9bi8EdrFky2NcXi199VXcENEK711lZX10VAbvUtgAoui53o1yuVDp2EqDquIB6DpCHy+W4++hRnB4cbOhkM616UuK9995ryc95XTPHUExWKi2973/oyJFCj5I12/raWqH7/Fubm/GfBgbiz6em6loeu2o3+2WtmLkxPx9bm5sxPTe3oxPJbt8zNxcWYmKHt42q98RbHQHVk3+711kRAbuzL8YA1HJzYaHjF9upHiAn6ziJHOjubvngKb5rfW0tPj16dN8M+qvHTj61jp8/H4PHj7dtNcC3ubO0FB8dPNiS0exVL7a24vTgYIwOD+/qU3Q1Aq7NzDRw697u4f37208zdciiSeurq1ZS3aF9HwDXZmb21M5xuVKJ04ODhQe4VAdPGRzYWi+2tmKyUolPP/ywYw6ErXRtZiY+/fDDHZ24HqysxL87eLAtKxm+SzXCB48fb/i89q+r7jsfHTzYsNuSW5ubTY+r6nYP9PV13NXUO0tLxkXtwL4NgPW1tRg8frzl95kb4c7S0vbltTqeWqhMTZk5sAW+ffYszo2MxL/9nd9J+an/5sJCfPTjHzfkfXX55Unw3MhIxzyhE7EdKAN9ffHp0aNxrcBcBkWtr63FuZGR+OjgwbhcqTTl3nk1rk4PDjbsakZ1n69ud6eqdxlh9vhTAG9aX1uLh/fuxS9fjtLdy9ZXV+seF9DMwYFZffvsWTxYWYlvVlfjwb176T7tV//9D1ZW4s7SUsP3q9cXMKouQvRZf38cKpcLL0bULOsvB6qOnz//atR8zyefxIEPPii0betra7H+5Mmr318rPzXfWVqKO0tLr36nPb29cejIkULjBPbyPl/voM7s3vvd99//50Z/03ZMWLPTE35XqVRoFrLnz5794Bv49RXX6v3aour9ff7Qzyuyna20tbnZsAPLTkftv8v66mrTIqrI9tb6/TTj3/ymTojoRuy3zfpbvu1vsNv3eyv80DGlGb+nosfXRu9rRd8fjfq5RffTTnhPva4pAQAAdLZ9OwYAAHg7AQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkJAAAICEBAAAJCQAACAhAQAACQkAAEhIAABAQgIAABISAACQkAAAgIQEAAAkJAAAICEBAAAJCQAASEgAAEBCAgAAEhIAAJCQAACAhAQAACQkAAAgIQEAAAkJAABISAAAQEICAAASEgAAkND/Ax1w95F7yRmhAAAAAElFTkSuQmCC)
This Actor is deprecated
This Actor is unavailable because the developer has decided to deprecate it. Would you like to try a similar Actor instead?
See alternative Actors![Bike Discount (bike-discount.de) scraper](https://images.apifyusercontent.com/hN2bDakVMpjItEh793WxcfKRAkRH97Avw4-aa-xveic/rs:fill:250:250/aHR0cHM6Ly9pLmltZ3VyLmNvbS9DTGZNU1hOLnBuZw.webp)
Bike Discount (bike-discount.de) scraper
strajk/bike-discount-bike-discount-de-scraper
Scrapes products titles, prices, images and availability. Does NOT scrape product details.
Dockerfile
1FROM apify/actor-node:16
2
3COPY package.json ./
4
5RUN npm --quiet set progress=false \
6 && npm install --only=prod --no-optional
7
8COPY . ./
INPUT_SCHEMA.json
1{
2 "title": "Bike Discount (bike-discount.de) scraper",
3 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
4 "type": "object",
5 "schemaVersion": 1,
6 "properties": {
7 "mode": {
8 "title": "Mode",
9 "description": "",
10 "type": "string",
11 "editor": "select",
12 "default": "TEST",
13 "prefill": "TEST",
14 "enum": [
15 "TEST",
16 "FULL"
17 ],
18 "enumTitles": [
19 "TEST",
20 "FULL"
21 ]
22 },
23 "debug": {
24 "title": "Debug",
25 "description": "Debug mode prints more logs, disables concurrency and other optimizations.",
26 "type": "boolean",
27 "editor": "checkbox",
28 "default": false
29 },
30 "APIFY_DONT_STORE_IN_DATASET": {
31 "sectionCaption": "Advanced",
32 "sectionDescription": "Advanced options, use only if you know what you're doing.",
33 "title": "Don't store in dataset",
34 "description": "If set to true, the actor will not store the results in the default dataset. Useful when using alternative storage, like own database",
35 "type": "boolean",
36 "default": false,
37 "editor": "checkbox"
38 },
39 "PG_CONNECTION_STRING_NORMALIZED": {
40 "title": "Postgres connection string for normalized data",
41 "description": "If set, actor will store normalized data in Postgres database in PG_DATA_TABLE and PG_DATA_PRICE_TABLE tables",
42 "type": "string",
43 "editor": "textfield"
44 },
45 "PG_DATA_TABLE": {
46 "title": "Postgres table name for product data",
47 "description": "Table name for storing product name, url, image, ...",
48 "type": "string",
49 "editor": "textfield"
50 },
51 "PG_DATA_PRICE_TABLE": {
52 "title": "Postgres table name for price data",
53 "description": "Table name for storing price, original price, stock status, ...",
54 "type": "string",
55 "editor": "textfield"
56 }
57 },
58 "required": [
59 "mode"
60 ]
61}
apify.json
1{
2 "name": "bike-discount-bike-discount-de-scraper",
3 "version": "0.1",
4 "buildTag": "latest",
5 "env": null,
6 "defaultRunOptions": {
7 "build": "latest",
8 "timeoutSecs": 3600,
9 "memoryMbytes": 1024
10 }
11}
main.js
1import { URL } from "node:url";
2import { Actor } from "apify3";
3import { CheerioCrawler, createCheerioRouter } from "crawlee";
4import { init, save, toNumberOrNull } from "./_utils/common.js";
5
6const LABELS = {
7 INDEX: `INDEX`,
8 PRODUCTS: `PRODUCTS`,
9};
10
11var MODE;
12
13(function (MODE) {
14 MODE["TEST"] = "TEST";
15 MODE["FULL"] = "FULL";
16})(MODE || (MODE = {}));
17
18async function enqueueInitial(mode, crawler) {
19 if (mode === MODE.FULL) {
20 await crawler.addRequests([
21 {
22 userData: { label: LABELS.INDEX },
23 url: `https://www.bike-discount.de/en/brands`,
24 },
25 ]);
26 } else if (mode === MODE.TEST) {
27 await crawler.addRequests([
28 {
29 userData: { label: LABELS.PRODUCTS },
30 url: `https://www.bike-discount.de/en/brand/poc/`,
31 },
32 ]);
33 }
34}
35
36const router = createCheerioRouter();
37
38router.addHandler(LABELS.INDEX, async ({ crawler, $ }) => {
39 const requests = [];
40 $(`.comer-product--info a.comer-supplier-detail`).each((i, el) => {
41 const url = $(el).attr(`href`); // urls are absolute
42 const name = $(el).text().trim(); // there's extra space at the beginning and the end
43 requests.push({
44 userData: { label: LABELS.PRODUCTS, category: name },
45 url,
46 });
47 });
48 void crawler.addRequests(requests);
49});
50
51router.addHandler(LABELS.PRODUCTS, async ({ crawler, $, request, log }) => {
52 log.info(`handleCategory ${request.url}`);
53 if (!request.url.includes(`?p=`)) {
54 // on first page
55 const totalPages = Number(
56 $(`.listing--bottom-paging .paging--link[title="Last page"]`).text()
57 ); // e.g. `6`
58 for (let i = 2; i <= totalPages; i++) {
59 // skip first page, that is already handled
60 const url = new URL(request.url);
61 url.searchParams.set(`p`, i.toString()); // toString() to make TS happy
62 void crawler.addRequests([
63 {
64 userData: { label: LABELS.PRODUCTS },
65 url: url.toString(),
66 },
67 ]);
68 }
69 }
70
71 const products = [];
72 $(`.listing--container .product--box`).each((i, el) => {
73 const id = $(el).attr(`data-ordernumber`); // e.g. 20080005-40200770
74 const url = $(el).find(`a.product--title`).attr(`href`); // absolute url
75 const title =
76 $(el).find(`a.product--title strong`).text() + // brand
77 ` ` +
78 $(el).find(`a.product--title`).attr(`title`); // title itself
79 const priceRaw = $(el)
80 .find(`.product--price .is--discount, .product--price .price--default`)
81 .text()
82 .trim(); // e.g. `€19.95`
83 const price = priceRaw.replace(/[^\d.]/g, ``); // keep dot as it's decimal separator
84 const priceOrigRaw = $(el)
85 .find(`.product--price .price--discount`)
86 .text()
87 .trim(); // `€39.95`
88 const priceOrig = priceOrigRaw.replace(/[^\d.]/g, ``);
89 const img = $(el).find(`.image--media img`).attr(`srcset`)?.split(`,`)?.[0];
90 const inStock = true;
91 const product = {
92 pid: id,
93 name: title,
94 url: url,
95 img: img,
96 inStock,
97 currentPrice: toNumberOrNull(price),
98 originalPrice: toNumberOrNull(priceOrig),
99 currency: `EUR`,
100 };
101 products.push(product);
102 });
103 await save(products);
104});
105
106void Actor.main(async () => {
107 const input = await Actor.getInput();
108 const { mode = MODE.FULL, ...rest } = input ?? {};
109 await init({ actorNameOverride: `bike-discount-de` }, rest);
110 const crawler = new CheerioCrawler({ requestHandler: router });
111 await enqueueInitial(mode, crawler);
112 await crawler.run();
113});
package.json
1{
2 "name": "bike-discount-bike-discount-de-scraper",
3 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
4 "type": "module",
5 "scripts": {
6 "start": "node ./main.js",
7 "push-to-apify-platform": "npx apify push"
8 },
9 "dependencies": {
10 "apify3": "npm:apify@^3.0.2",
11 "crawlee": "*",
12 "pg": "*",
13 "pg-connection-string": "*",
14 "dotenv": "*",
15 "find-config": "*",
16 "@elastic/elasticsearch": "*",
17 "filenamify": "*"
18 },
19 "apify": {
20 "title": "Bike Discount (bike-discount.de) scraper",
21 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
22 "isPublic": true,
23 "isDeprecated": false,
24 "isAnonymouslyRunnable": true,
25 "notice": "",
26 "pictureUrl": "",
27 "seoTitle": "",
28 "seoDescription": "",
29 "categories": [
30 "ECOMMERCE"
31 ]
32 }
33}
.actor/actor.json
1{
2 "actorSpecification": 1,
3 "name": "bike-discount-bike-discount-de-scraper",
4 "title": "Bike Discount (bike-discount.de) scraper",
5 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
6 "version": "0.1.0",
7 "storages": {
8 "dataset": {
9 "actorSpecification": 1,
10 "title": "Bike Discount (bike-discount.de) scraper",
11 "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.",
12 "views": {
13 "overview": {
14 "title": "Overview",
15 "description": "Overview of the most important fields",
16 "transformation": {
17 "fields": [
18 "pid",
19 "name",
20 "url",
21 "img",
22 "inStock",
23 "currentPrice",
24 "originalPrice",
25 "currency"
26 ]
27 },
28 "display": {
29 "component": "table",
30 "columns": [
31 {
32 "label": "Pid",
33 "field": "pid",
34 "format": "text"
35 },
36 {
37 "label": "Name",
38 "field": "name",
39 "format": "text"
40 },
41 {
42 "label": "Url",
43 "field": "url",
44 "format": "link"
45 },
46 {
47 "label": "Img",
48 "field": "img",
49 "format": "image"
50 },
51 {
52 "label": "In Stock",
53 "field": "inStock",
54 "format": "boolean"
55 },
56 {
57 "label": "Current Price",
58 "field": "currentPrice",
59 "format": "number"
60 },
61 {
62 "label": "Original Price",
63 "field": "originalPrice",
64 "format": "number"
65 },
66 {
67 "label": "Currency",
68 "field": "currency",
69 "format": "text"
70 }
71 ]
72 }
73 }
74 }
75 }
76 }
77}
.actor/logo.png
_utils/common.js
1import { createHash } from 'crypto'
2import os from "os"
3import path from "path"
4// eslint-disable-next-line @apify/apify-actor/no-forbidden-node-internals
5import fs from "fs"
6import pg from "pg"
7import pgConnectionString from 'pg-connection-string'
8import { config } from 'dotenv'
9import findConfig from "find-config"
10import { Client as ElasticClient } from "@elastic/elasticsearch"
11import filenamify from 'filenamify'
12import { Dataset } from 'crawlee'
13
14config({ path: findConfig(`.env`) })
15
16const elasticIndexName = `actors-monorepo-shops`
17
18const globalLogsProps = {
19 __NODE_STARTED: new Date().toISOString(),
20}
21
22let actorName
23let pgClient
24let pgClientNormalized
25let elasticClient
26export async function init ({ actorNameOverride }, restInput) {
27 parseEnvFromInput(restInput)
28
29 if (os.platform() === `darwin`) {
30 const filePath = process.argv[1] // ~/Projects/apify-actors-monorepo/actors/foo.ts
31 const basename = path.basename(filePath) // foo.ts
32 actorName = actorNameOverride ?? basename.split(`.`)[0] // foo
33 const gitBranch = fs.readFileSync(path.join(process.cwd(), `..`, `.git/HEAD`), `utf8`)
34 .split(` `)[1]
35 .trim()
36 .replace(`refs/heads/`, ``)
37 const gitCommit = fs.readFileSync(path.join(process.cwd(), `..`, `.git/refs/heads/${gitBranch}`), `utf8`)
38 const gitCommitShort = gitCommit.substring(0, 7)
39 globalLogsProps.__GIT_COMMIT = gitCommitShort
40 }
41
42 if (process.env.APIFY_IS_AT_HOME) {
43 actorName = actorNameOverride ?? process.env.APIFY_ACTOR_ID // Name would be better, but it's not in ENV
44 }
45
46 /* ELASTIC */
47 /* ======= */
48 if (process.env.ELASTIC_CLOUD_ID) {
49 elasticClient = new ElasticClient({
50 cloud: { id: process.env.ELASTIC_CLOUD_ID },
51 auth: { apiKey: process.env.ELASTIC_CLOUD_API_KEY },
52 })
53
54 // const mapping = await elasticClient.indices.getMapping({ index: actorName })
55
56 // eslint-disable-next-line no-inner-declarations
57 async function enforceIndexMapping () {
58 const doesIndexExist = await elasticClient.indices.exists({ index: elasticIndexName })
59 if (!doesIndexExist) await elasticClient.indices.create({ index: elasticIndexName })
60 await elasticClient.indices.putMapping({
61 index: elasticIndexName,
62 body: {
63 properties: {
64 _discount: { type: `float` },
65 originalPrice: { type: `float` },
66 currentPrice: { type: `float` },
67 },
68 },
69 })
70 }
71
72 try {
73 await enforceIndexMapping()
74 } catch (err) {
75 if (err.message.includes(`cannot be changed from type`)) {
76 console.log(`Elastic index ${elasticIndexName} already exists with incorrect mappings. As existing mapping cannot be changed, index will be deleted and recreated.`)
77 await elasticClient.indices.delete({ index: elasticIndexName })
78 await enforceIndexMapping()
79 }
80 }
81 }
82
83 /* POSTGRESQL */
84 /* ========== */
85 if (process.env.PG_CONNECTION_STRING) {
86 const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING)
87 // const pgPool = new pg.Pool(pgConfig)
88
89 pgClient = new pg.Client(pgConfig)
90 await pgClient.connect()
91
92 // Check if table exists and have proper columns
93 const { rows: tables } = await pgClient.query(`
94 SELECT table_name
95 FROM information_schema.tables
96 WHERE table_schema = 'public'
97 `)
98
99 // eslint-disable-next-line camelcase
100 const tableExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
101 if (!tableExists) {
102 throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
103 }
104
105 // TODO: Handle pgClient closing
106 }
107
108 if (process.env.PG_CONNECTION_STRING_NORMALIZED) {
109 const pgConfig = pgConnectionString(process.env.PG_CONNECTION_STRING_NORMALIZED)
110
111 pgClientNormalized = new pg.Client(pgConfig)
112 await pgClientNormalized.connect()
113
114 // Check if table exists and have proper columns
115 const { rows: tables } = await pgClientNormalized.query(`
116 SELECT table_name
117 FROM information_schema.tables
118 WHERE table_schema = 'public'
119 `)
120
121 // eslint-disable-next-line camelcase
122 const tableMainExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_TABLE)
123 // eslint-disable-next-line camelcase
124 const tablePricesExists = tables.some(({ table_name }) => table_name === process.env.PG_DATA_PRICE_TABLE)
125 if (!tableMainExists) throw new Error(`Table ${process.env.PG_DATA_TABLE} does not exist in database ${pgConfig.database}`)
126 if (!tablePricesExists) throw new Error(`Table ${process.env.PG_DATA_PRICE_TABLE} does not exist in database ${pgConfig.database}`)
127
128 // TODO: Handle pgClient closing
129 }
130}
131
132// inspired by @drobnikj
133// TODO: Similar, but less obfuscated for easier debugging
134export const createUniqueKeyFromUrl = (url) => {
135 const hash = createHash(`sha256`)
136 const cleanUrl = url.split(`://`)[1] // Remove protocol
137 hash.update(cleanUrl)
138 return hash.digest(`hex`)
139}
140
141/**
142 *
143 * @param {Date} datetime
144 * @return {Promise<void>}
145 */
146export const sleepUntil = async (datetime) => {
147 const now = new Date()
148 const difference = datetime - now
149 if (difference > 0) {
150 return new Promise((resolve) => {
151 setTimeout(resolve, difference)
152 })
153 }
154 return Promise.resolve()
155}
156
157export function parsePrice (string) {
158 let amount, currency
159 const noText = string.replace(/[^\d,.]/g, ``)
160 const decimals = noText.match(/([,.])(\d{2})$/)
161 if (decimals) {
162 const decimalSeparator = decimals[1]
163 // eslint-disable-next-line @typescript-eslint/no-unused-vars, no-unused-vars
164 const decimalAmount = decimals[2]
165 amount = parseInt(noText.split(decimalSeparator)[0])
166 } {
167 const justNumbers = noText.replace(/[,.]/g, ``)
168 amount = parseInt(justNumbers)
169 }
170 return { amount, currency }
171}
172
173export function toNumberOrNull (str) {
174 // TODO: Handle better, but only after adding test
175 if (str === undefined) return null
176 if (str === null) return null
177 if (str === ``) return null
178 const num = Number(str)
179 if (Number.isNaN(num)) return null
180 return num
181}
182
183export async function save (objs) {
184 if (!Array.isArray(objs)) objs = [objs]
185 if (objs.length === 0) return
186
187 const objsExtended = objs.map((obj) => {
188 const objExtended = {
189 ...obj,
190 actorName,
191 ...globalLogsProps,
192 // __NODE_VERSION: global.process.versions.node,
193 // __NODE_UPTIME: global.process.uptime().toFixed(2), // seconds, 2 decimals
194 }
195 // if run on Apify
196 if (process.env.APIFY_IS_AT_HOME) {
197 objExtended.__APIFY_ACTOR_ID = process.env.APIFY_ACTOR_ID
198 objExtended.__APIFY_ACTOR_RUN_ID = process.env.APIFY_ACTOR_RUN_ID
199 objExtended.__APIFY_ACTOR_BUILD_ID = process.env.APIFY_ACTOR_BUILD_ID
200 objExtended.__APIFY_ACTOR_BUILD_NUMBER = process.env.APIFY_ACTOR_BUILD_NUMBER
201 objExtended.__APIFY_ACTOR_TASK_ID = process.env.APIFY_ACTOR_TASK_ID
202 if (!process.env.APIFY_DONT_STORE_IN_DATASET) void Dataset.pushData(obj)
203 }
204 return objExtended
205 })
206 // if runs on local machine (MacOS)
207 if (os.platform() === `darwin`) {
208 const cwd = process.cwd() // ~/Projects/apify-actors-monorepo/actors
209 const storageDir = path.join(cwd, `${actorName}.storage`) // ~/Projects/apify-actors-monorepo/actors/foo.storage
210 if (!fs.existsSync(storageDir)) fs.mkdirSync(storageDir)
211 const dataDir = path.join(storageDir, `data`) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data
212 if (!fs.existsSync(dataDir)) fs.mkdirSync(dataDir)
213 for (const objExtended of objsExtended) {
214 const id = objExtended.id ?? objExtended.pid // ?? uuidv4()
215 const fileName = `${filenamify(id)}.json`
216 const dataFilePath = path.join(dataDir, fileName) // ~/Projects/apify-actors-monorepo/actors/foo.storage/data/foo.json
217 fs.writeFileSync(dataFilePath, JSON.stringify(objExtended, null, 2))
218 }
219 }
220
221 if (pgClient) {
222 const objsPg = objs.map((obj) => ({
223 ...obj,
224 // TODO: This is becoming not nice, and not clear
225 shop: actorName,
226 scrapedAt: new Date().toISOString().split(`T`)[0],
227 }))
228
229 const columns = getColumns(objsPg)
230 const values = getValues(objsPg)
231 const queryString = `
232 INSERT INTO public."${process.env.PG_DATA_TABLE}" (${columns})
233 VALUES (${values})
234 `
235 try {
236 const { rowCount } = await pgClient.query(queryString)
237 console.log(`[save] saved to database: ${JSON.stringify(rowCount)}`)
238 } catch (err) {
239 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
240 else throw err
241 }
242 }
243
244 // Only make sense for HlidacShopu
245 if (pgClientNormalized) {
246 const objsPgData = objs.map((obj) => ({
247 shop: actorName,
248 pid: obj.pid,
249 name: obj.name,
250 url: obj.url,
251 img: obj.img,
252 }))
253
254 const objsPgDataPrice = objs.map((obj) => ({
255 shop: actorName,
256 pid: obj.pid,
257 scrapedAt: new Date().toISOString().split(`T`)[0],
258 currentPrice: obj.currentPrice,
259 originalPrice: obj.originalPrice,
260 inStock: obj.inStock,
261 }))
262
263 const queryString = `
264 INSERT INTO public."${process.env.PG_DATA_TABLE}" (${getColumns(objsPgData)})
265 VALUES (${getValues(objsPgData)})
266 ON CONFLICT DO NOTHING
267 `
268 try {
269 const { rowCount } = await pgClientNormalized.query(queryString)
270 console.log(`[save] saved to database (data): ${JSON.stringify(rowCount)}`)
271 } catch (err) {
272 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
273 else throw err
274 }
275
276 const queryStringPrice = `
277 INSERT INTO public."${process.env.PG_DATA_PRICE_TABLE}" (${getColumns(objsPgDataPrice)})
278 VALUES (${getValues(objsPgDataPrice)})
279 ON CONFLICT DO NOTHING
280 `
281 try {
282 const { rowCount } = await pgClientNormalized.query(queryStringPrice)
283 console.log(`[save] saved to database (price): ${JSON.stringify(rowCount)}`)
284 } catch (err) {
285 if (err.message.includes(`violates unique constraint`)) console.warn(`PostgresSQL: violates unique constraint`)
286 else throw err
287 }
288 }
289
290 if (elasticClient) {
291 // .index creates or updates the document
292 // .create creates a new document if it doesn't exist, 409 if it does
293 // try {
294 // const res = await elasticClient.index({
295 // index: `actors-monorepo-shops`, // TODO: Consider using actorName
296 // id, // foo-bar
297 // document: objExtended, // {...}
298 // })
299 // } catch (err) {
300 // // https://discuss.elastic.co/t/elasticsearch-503-ok-false-message-the-requested-deployment-is-currently-unavailable/200583
301 // if (err.message.includes(`requested resource is currently unavailable`)) console.log(`Elasticsearch is unavailable, skipping, but not aborting`)
302 // else throw err
303 // }
304 }
305}
306
307function getColumns (objs) {
308 return Object.keys(objs[0]).map((key) => `"${key}"`).join(`, `)
309}
310
311function getValues (objs) {
312 return objs.map(objPg => Object.values(objPg).map((value) => {
313 // escape strings to prevent SQL injection
314 if (typeof value === `string`) return `'${value.replace(/'/g, `''`)}'`
315 // convert to DB specific null
316 if (typeof value === `undefined` || value === null) return `NULL`
317 return value
318 }).join(`, `)).join(`), (`)
319}
320
321export function parseEnvFromInput (input) {
322 const env = {}
323 for (const key in input) {
324 if (key === key.toUpperCase()) env[key] = input[key]
325 }
326 console.log(`[parseEnvFromInput] ${JSON.stringify(env)}`)
327 Object.assign(process.env, env)
328}
Developer
Maintained by Community
Categories