My Actor
DeprecatedPricing
Pay per usage
Pricing
Pay per usage
Rating
0.0
(0)
Developer
Rohit Bansal
Maintained by CommunityActor stats
0
Bookmarked
1
Total users
1
Monthly active users
2 years ago
Last modified
Categories
Share
# Specify the base Docker image. You can read more about# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images# You can also use any other image from Docker Hub.FROM apify/actor-node:18
# Copy just package.json and package-lock.json# to speed up the build using Docker layer cache.COPY package*.json ./
# Install NPM packages, skip optional and development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version \ && rm -r ~/.npm
# Next, copy the remaining files and directories with the source code.# Since we do this after NPM install, quick build will be really fast# for most source file changes.COPY . ./
# Run the image.CMD npm start --silent{ "actorSpecification": 1, "name": "my-actor", "title": "Project Cheerio Crawler Javascript", "description": "Crawlee and Cheerio project in javascript.", "version": "0.0", "meta": { "templateId": "js-crawlee-cheerio" }, "input": "./input_schema.json", "dockerfile": "./Dockerfile"}{ "title": "Amazon", "description": "Daniil", "type": "object", "schemaVersion": 1, "properties": { "keyword": { "title": "Keyword", "type": "string", "description": "description required", "editor": "textfield" } }, "required": ["keyword"] }1import Apify from 'apify'2
3Apify.main(async () => {4 const keyword = 'phone'5
6 const StartUrl = 'https://www.amazon.com/s/ref=nb_sb_noss?url=search-alias%3Daps&field-keywords='7
8 const StartProductUrl = 'https://www.amazon.com/dp/'9
10 const StartOfferUrl = 'https://www.amazon.com/gp/offer-listing/'11
12 const requestQueue = await Apify.openRequestQueue();13
14 await requestQueue.addRequest({15 url: StartUrl + keyword,16 userData: {17 label: 'start',18 keyword,19 },20
21 });22
23 const crawler = new Apify.PuppeteerCrawler({24
25 requestQueue,26 handlePageFunction: async ({ page, request }) => {27 if (request.userData.label === 'start') {28 console.log('Start page is:', request.url)29
30 try {31 await page.waitForSelector('.s-result-list', { timeout: 10000 })32
33 const asins = await page.$$eval('.s-result-item', async (items) => {34 return items.map((item) => item.dataset.asin).filter((item) => item !== '' && item !== undefined)35 });36
37 const items = asins.map((asin) => {38 const productUrl = StartProductUrl + asin39 const sellerUrl = StartOfferUrl + asin40 return { asin, productUrl, sellerUrl, keyword: request.userData.keyword }41 });42
43 for (const item of items) {44 await requestQueue.addRequest({45 url: item.productUrl,46 userData: {47 label: 'product',48 asin: item.asin,49 keyword: item.keyword,50 productUrl: item.productUrl,51 sellerUrl: item.sellerUrl,52 },53 });54 }55 } catch (err) {56 console.log(err);57 await dataset.pushData({58 url: request.url,59 status: 'No results',60 });61 }62
63 console.log('Keep crawling.')64 } else if (request.userData.label === 'product') {65 console.log(`Go to product page: ${request.url}`)66
67 const PageInfo = await page.evaluate(() => {68 const titleEl = document.getElementById('productTitle')69
70 if (!titleEl) {71 return false72 }73
74 const title = titleEl.innerText75 const url = document.URL76 const description = document.getElementById('productDescription') ? document.getElementById('productDescription').innerText : 'No description.'77
78 return { title, url, description }79 });80
81 const { asin, keyword, productUrl } = request.userData82
83 await requestQueue.addRequest({84 url: request.userData.sellerUrl,85 userData: {86 label: 'seller',87 asin,88 keyword,89 productUrl,90 title: PageInfo.title,91 description: PageInfo.description,92 },93 });94
95 console.log(`End with ${request.url}`)96 } else if (request.userData.label === 'seller') {97 console.log(`Go to seller page: ${request.url}`)98
99 const offers = await page.evaluate(() => {100 const price = document.querySelector('span.a-price > span') ? document.querySelector('span.a-price > span').innerText : 'no Price'101 const seller = document.querySelector('#sellerProfileTriggerId') ? document.querySelector('#sellerProfileTriggerId').innerText : 'Amazon'102
103 return { price, seller }104 });105
106 const { asin, keyword, productUrl, title, description } = request.userData107
108 const item = {109 title,110 itemUrl: productUrl,111 description,112 keyword,113 asin,114 price: offers.price,115 seller: offers.seller,116 };117
118 await Apify.pushData(item)119 }120 },121 handleFailedRequestFunction: async ({ request }) => {122 console.log(`Request ${request.url} failed 4 times`)123
124 await Apify.pushData({125 url: request.url,126 errors: request.errorMessages,127 })128 },129 })130
131 await crawler.run()132});# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed filesnode_modules
# git folder.gitroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf{ "extends": "@apify", "root": true}# This file tells Git which files shouldn't be added to source control
.DS_Store.ideadistnode_modulesapify_storagestorage{ "name": "crawlee-cheerio-javascript", "version": "0.0.1", "type": "module", "description": "This is a boilerplate of an Apify actor.", "engines": { "node": ">=18.0.0" }, "dependencies": { "apify": "^3.1.10", "crawlee": "^3.5.4" }, "devDependencies": { "@apify/eslint-config": "^0.4.0", "eslint": "^8.50.0" }, "scripts": { "start": "node src/main.js", "lint": "eslint ./src --ext .js,.jsx", "lint:fix": "eslint ./src --ext .js,.jsx --fix", "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1" }, "author": "It's not you it's me", "license": "ISC"}