Pricing
Pay per usage
Scrapes products titles, prices, images and availability. Does NOT scrape product details.
0.0 (0)
Pricing
Pay per usage
1
3
1
Last modified
3 years ago
FROM apify/actor-node:16
COPY package.json ./
RUN npm --quiet set progress=false \ && npm install --only=prod --no-optional
COPY . ./{ "title": "Wiggle (wiggle.com) scraper", "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.", "type": "object", "schemaVersion": 1, "properties": { "mode": { "title": "Mode", "description": "", "type": "string", "editor": "select", "default": "TEST", "prefill": "TEST", "enumTitles": [ "TEST", "FULL" ], "enum": [ "TEST", "FULL" ] }, "debug": { "title": "Debug", "description": "Debug mode prints more logs, disables concurrency and other optimizations.", "type": "boolean", "editor": "checkbox", "default": false } }, "required": [ "mode" ]}{ "name": "wiggle-wiggle-com-scraper", "version": "0.1", "buildTag": "latest", "env": null, "defaultRunOptions": { "build": "latest", "timeoutSecs": 3600, "memoryMbytes": 1024 }}1import { URL } from "node:url";2import Apify from "apify";3
4const { log } = Apify.utils;5
6var LABEL;7
8(function (LABEL) {9 LABEL["INDEX"] = "INDEX";10 LABEL["PRODUCTS"] = "PRODUCTS";11})(LABEL || (LABEL = {}));12var MODE;13
14(function (MODE) {15 MODE["TEST"] = "TEST";16 MODE["FULL"] = "FULL";17})(MODE || (MODE = {}));18
19const BASE_URL = `https://www.wiggle.com`;20const PER_PAGE = 96;21
22async function enqueueInitialRequest(mode, requestQueue) {23 if (mode === MODE.FULL) {24 await requestQueue.addRequest({25 userData: { label: LABEL.INDEX },26 url: `${BASE_URL}/all-brands`,27 });28 } else if (mode === MODE.TEST) {29 await requestQueue.addRequest({30 userData: { label: LABEL.PRODUCTS },31 url: `${BASE_URL}/poc`,32 });33 }34}35
36const router = {37 [LABEL.INDEX]: async ({ $ }, { requestQueue }) => {38 $(`h2:contains("All brands") + .brandgroup .branditem a:first-child`).each(39 (i, el) => {40 const relUrl = $(el).attr(`href`); // urls are relative, starting with /41 const url = `${BASE_URL}${relUrl}`;42 void requestQueue.addRequest({43 userData: { label: LABEL.PRODUCTS },44 url,45 });46 }47 );48 },49 [LABEL.PRODUCTS]: async ({ $, request }, { requestQueue }) => {50 if (!request.url.includes(`?g=`)) {51 // on first page52 const totalItemsText = $(`#listing-page-header-title-row span`).text(); // e.g. `(856)`53 const totalItems = Number(totalItemsText.replace(/[^0-9]/g, ``));54 let offset = PER_PAGE; // initially 9655 while (offset < totalItems) {56 const url = new URL(request.url);57 url.searchParams.set(`g`, (offset + 1).toString()); // toString() to make TS happy58 void requestQueue.addRequest({59 userData: { label: LABEL.PRODUCTS },60 url: url.toString(),61 });62 offset += PER_PAGE;63 }64
65 $(`.js-result-list-item`).each((i, el) => {66 const id = $(el).attr(`data-id`);67 const url = $(el)68 .find(`a.bem-product-thumb__image-link--grid`)69 .attr(`href`);70 const title = $(el)71 .find(`a.bem-product-thumb__image-link--grid`)72 .attr(`title`);73 const priceRaw = $(el).find(`.bem-product-price__unit--grid`).text(); // e.g. €101.27 or "€28.47 - €69.90"74 const price = priceRaw.match(/\$([\d.]+)/)?.[1]; // TODO: Support both $ and €75 const discountRaw = $(el).find(`.bem-product_price__discount`).text(); // Save 30% - 45%76
77 let discount, priceOrig;78 if (discountRaw) {79 discount = discountRaw.match(/(\d+)%$/)[1]; // 4580 priceOrig = (price / (1 - discount / 100)).toFixed(2);81 }82 const img = $(el)83 .find(`.js-result-list-image`)84 .attr(`data-original`) // //www.wigglestatic.com/product-media/5360088903/1000-Mile-Women-s-Ultimate-Approach-Socks-Hiking-Socks-Navy-1998NLS-2.jpg?w=200&h=200&a=785 // leading "//" -> https://86 .replace(/^\/\//, `https://`)87 // remove part after ?88 .replace(/\.jpg?.+/, `.jpg`);89
90 const inStock = null;91 const product = {92 itemId: id,93 itemName: title,94 itemUrl: url,95 img: img,96 inStock,97 currentPrice: parseFloat(price),98 originalPrice: priceOrig ? parseFloat(priceOrig) : null,99 currency: `USD`,100
101 // Derived102 discounted: !!(priceOrig && priceOrig !== price),103 _discount: discount / 100 ?? null,104 };105 Apify.pushData(product);106 });107 }108 },109};110
111Apify.main(async () => {112 const input = await Apify.getInput();113 const { debug = false, mode = MODE.FULL } = input ?? {};114 if (debug) Apify.utils.log.setLevel(Apify.utils.log.LEVELS.DEBUG);115
116 const requestQueue = await Apify.openRequestQueue();117 await enqueueInitialRequest(mode, requestQueue);118
119 const globalContext = { mode, requestQueue };120 const crawler = new Apify.CheerioCrawler({121 requestQueue,122 maxConcurrency: debug ? 1 : 3,123 maxRequestRetries: debug ? 0 : 3,124 preNavigationHooks: [125 async ({ request }) => {126 if (request.userData.label === LABEL.PRODUCTS) {127 const url = new URL(request.url);128 url.searchParams.set(`ps`, PER_PAGE.toString()); // toString() to make TS happy129 request.url = url.toString();130 }131 },132 ],133 async handlePageFunction(context) {134 await router[context.request.userData.label](context, globalContext);135 },136 async handleFailedRequestFunction({ request }) {137 log.error(`Request ${request.url} failed multiple times`, request);138 },139 });140
141 await crawler.run();142 log.info(`crawler finished`);143});{ "name": "wiggle-wiggle-com-scraper", "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.", "type": "module", "scripts": { "start": "node ./main.js", "push-to-apify-platform": "npx apify push" }, "dependencies": { "apify": "*" }, "apify": { "title": "Wiggle (wiggle.com) scraper", "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.", "isPublic": true, "isDeprecated": false, "isAnonymouslyRunnable": true, "notice": "", "pictureUrl": "", "seoTitle": "", "seoDescription": "", "categories": [ "ECOMMERCE" ] }}{ "actorSpecification": 1, "name": "wiggle-wiggle-com-scraper", "title": "Wiggle (wiggle.com) scraper", "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.", "version": "0.1.0", "storages": { "dataset": { "actorSpecification": 1, "title": "Wiggle (wiggle.com) scraper", "description": "Scrapes products titles, prices, images and availability. Does NOT scrape product details.", "views": { "overview": { "title": "Overview", "description": "Overview of the most important fields", "transformation": { "fields": [ "itemId", "itemName", "itemUrl", "img", "inStock", "currentPrice", "originalPrice", "currency" ] }, "display": { "component": "table", "columns": [ { "label": "Item ID", "field": "itemUrl", "format": "link", "textField": "itemId" }, { "label": "Item Name", "field": "itemName", "format": "text" }, { "label": "Img", "field": "img", "format": "image" }, { "label": "In Stock", "field": "inStock", "format": "boolean" }, { "label": "Current Price", "field": "currentPrice", "format": "number" }, { "label": "Original Price", "field": "originalPrice", "format": "number" }, { "label": "Currency", "field": "currency", "format": "text" } ] } } } } }}