const Apify = require('apify'); /** * Gets attribute as text from a ElementHandle. * @param {ElementHandle} element - The element to get attribute from. * @param {string} attr - Name of the attribute to get. */ async function getAttribute(element, attr){ try{ const prop = await element.getProperty(attr); return (await prop.jsonValue()).trim(); } catch(e){return null;} } /** Main function */ Apify.main(async () => { // Open request queue and add statrUrl const requestQueue = await Apify.openRequestQueue(); await requestQueue.addRequest(new Apify.Request({ userData: {label: 'start'}, url: 'https://www.tsbohemia.cz/' })); // Disable all non-essential requests const gotoFunction = async ({ page, request }) => { await page.setRequestInterception(true); page.on('request', intercepted => { const type = intercepted.resourceType(); if(type === 'stylesheet'){intercepted.abort();} else{intercepted.continue();} }); await Apify.utils.puppeteer.hideWebDriver(page); return await page.goto(request.url, {timeout: 200000}); }; // Handle page context const handlePageFunction = async ({ page, request }) => { // This is the start page if(request.userData.label === 'start'){ await page.waitFor(2000); // Download all category links const urls = await page.evaluate(async () => { let result = []; const tcs = $('.i6lt_plus a'); for(let i = 0; i < tcs.length; i++){ const tc = tcs[i]; const id = $(tc).attr('data-strid'); const resp = await fetch("https://www.tsbohemia.cz/default_jx.asp?show=sptnavigator&strid=" + id, {"credentials":"include","headers":{"accept":"*/*","accept-language":"cs,en-US;q=0.9,en;q=0.8","x-requested-with":"XMLHttpRequest"},"referrer":"https://www.tsbohemia.cz/","referrerPolicy":"no-referrer-when-downgrade","body":null,"method":"GET","mode":"cors"}); const text = await resp.text(); const subLinks = $('.level6 > li > a', text); result = result.concat(subLinks.toArray().map(sl => $(sl).attr('href'))); } return result; }); // Enqueue category links for(const url of urls){ await requestQueue.addRequest(new Apify.Request({ userData: {label: 'page'}, url: 'https://www.tsbohemia.cz/' + url })); } } // This is the category page else if(request.userData.label === 'page'){ // Enqueue pagination pages await Apify.utils.puppeteer.enqueueLinks({ page: page, requestQueue: requestQueue, selector: '.numpage a', pseudoUrls: null, userData: {label: 'page'} }); // Scroll through whole page to force images to load await page.evaluate(async () => { const body = document.body; const html = document.documentElement; const height = Math.max( body.scrollHeight, body.offsetHeight, html.clientHeight, html.scrollHeight, html.offsetHeight); let yPos = 0; while(yPos < height){ yPos += 300; window.scrollTo(0, yPos); await new Promise(r => setTimeout(r, 100)); } }); // Iterate all products and extract data const items = await page.$$('.prodbox'); for(const item of items){ const toNumber = p => p.replace(/\s/g, '').match(/\d+/)[0]; try{ const linkElem = await item.$('h2 a'); const imgElem = await item.$('.img img'); const oPriceElem = await item.$('.price .mc'); const priceElem = await item.$('.price .wvat'); const img = await getAttribute(imgElem, 'src'); const link = await getAttribute(linkElem, 'href'); const name = await getAttribute(linkElem, 'textContent'); const price = await getAttribute(priceElem, 'textContent'); const dataItem = { img: img, itemId: link.match(/d(\d+)/)[1], itemUrl: link, itemName: name, discounted: oPriceElem ? true : false, currentPrice: price ? toNumber(price) : null }; if(oPriceElem){ const oPrice = await getAttribute(oPriceElem, 'textContent'); dataItem.originalPrice = toNumber(oPrice); } // Save data to dataset await Apify.pushData(dataItem); } catch(e){console.log(e);} } } }; // Create crawler const crawler = new Apify.PuppeteerCrawler({ requestQueue, // Use proxy launchPuppeteerOptions: { useApifyProxy: true }, handlePageFunction, // If request failed 4 times then this function is executed handleFailedRequestFunction: async ({ request }) => { console.log(`Request ${request.url} failed 4 times`); }, }); // Run crawler await crawler.run(); });

Build new tools

Get a custom solution