const Apify = require('apify'); const handleFailedRequestFunction = ({ request }) => { console.log(`Request ${request.url} failed 4 times`); }; const baseURL = 'https://amazon.com'; Apify.main(async () => { const input = await Apify.getInput(); const dataset = await Apify.openDataset('amazon-dataset'); const { keyword, email } = input; if (!keyword || !email) { throw new Error('Please, provide both the keyword and the email'); } const enqueueUrl = async (queue, url, userData = {}) => queue.addRequest({ url, userData }); const asinQueue = await Apify.openRequestQueue('asin'); const descQueue = await Apify.openRequestQueue('desc'); const offersQueue = await Apify.openRequestQueue('offers'); await enqueueUrl(asinQueue, `${baseURL}/s?k=${keyword}`); // initial url const asinCrawler = new Apify.PuppeteerCrawler({ requestQueue: asinQueue, handlePageFunction: async ({ page, request }) => { console.log(`Request ${request.url} succeeded!`); await Apify.utils.puppeteer.injectJQuery(page); const data = await page.evaluate(() => { const asins = []; $('[data-asin]').each((index, value) => { const asin = value.getAttribute('data-asin'); const itemUrl = $(value) .find('a') .attr('href'); asins.push({ itemUrl, asin, }); }); return asins; }); data.forEach((value) => { if (value.itemUrl.match('http')) enqueueUrl(descQueue, value.itemUrl, value); else { enqueueUrl(descQueue, `${baseURL}${value.itemUrl}`, { ...value, itemUrl: `${baseURL}${value.itemUrl}`, }); } }); }, // If request failed 4 times then this function is executed. handleFailedRequestFunction, }); const descCrawler = new Apify.PuppeteerCrawler({ requestQueue: descQueue, handlePageFunction: async ({ page, request }) => { console.log(`Request ${request.url} succeeded!`); await Apify.utils.puppeteer.injectJQuery(page); const { description, title } = await page.evaluate(() => { const description = $('#productDescription') .find('p') .text() .trim(); const titleMatchResult = document.title.match('(?<=:).*?(?=:)'); const title = titleMatchResult ? titleMatchResult[0].trim() : document.title.trim(); // removing amazon and category text return { description, title }; }); enqueueUrl( offersQueue, `https://www.amazon.com/gp/offer-listing/${request.userData.asin}?startIndex=0`, { ...request.userData, title, description, offers: [], // initialize offers to prevent crash when there are more than one page startIndex: 0, }, ); }, handleFailedRequestFunction, }); const offersCrawler = new Apify.PuppeteerCrawler({ requestQueue: offersQueue, handlePageFunction: async ({ page, request }) => { console.log(`Request ${request.url} succeeded!`); await Apify.utils.puppeteer.injectJQuery(page); const { offersList: offers, isOffersEnded } = await page.evaluate(() => { const offersList = []; $('.olpOffer').each((index, element) => { const sellerName = $(element) .find('.olpSellerName') .text() .trim(); const shippingBlock = $(element) .find('.olpShippingInfo') .text() .trim(); const shipping = shippingBlock.toLowerCase().match('free') ? 'free' : shippingBlock.replace(/\s+/g, ' '); const offer = $(element) .find('.olpOfferPrice') .text() .trim(); offersList.push({ sellerName, shipping, offer }); }); const isOffersEnded = $('.a-disabled.a-last').length || !$('.a-pagination').length; return { offersList, isOffersEnded }; }); if (isOffersEnded) { delete request.userData.asin; delete request.userData.startIndex; dataset.pushData({ ...request.userData, keyword, offers: [...request.userData.offers, ...offers] }); } else { enqueueUrl( offersQueue, `https://www.amazon.com/gp/offer-listing/${request.userData.asin}?startIndex=${request .userData.startIndex + 10}`, { ...request.userData, offers: [...request.userData.offers, ...offers], startIndex: request.userData.startIndex + 10, }, ); } }, handleFailedRequestFunction, }); // Run crawlers. await asinCrawler.run(); await descCrawler.run(); await offersCrawler.run(); const datasetInfo = await dataset.getInfo(); // send email await Apify.call('apify/send-mail', { to: email, subject: 'Your data from apify', text: `Here is your data from apify actor, https://api.apify.com/v2/datasets/${ datasetInfo.id }/items?format=json`, }); });

Where next?