Pricing
Pay per usage
Using Cheerio & Crawlee, the actor checks if given pages satisfy required conditions and fails if not.
Pricing
Pay per usage
Rating
0.0
(0)
Developer
Josef Válek
Actor stats
3
Bookmarked
4
Total users
0
Monthly active users
2 years ago
Last modified
Categories
Share
# Specify the base Docker image. You can read more about# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images# You can also use any other image from Docker Hub.FROM apify/actor-node:16
# Copy just package.json and package-lock.json# to speed up the build using Docker layer cache.COPY package*.json ./
# Install NPM packages, skip optional and development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version \ && rm -r ~/.npm
# Next, copy the remaining files and directories with the source code.# Since we do this after NPM install, quick build will be really fast# for most source file changes.COPY . ./
# Run the image.CMD npm start --silent{ "actorSpecification": 1, "name": "project-cheerio-crawler-javascript", "title": "Project Cheerio Crawler Javascript", "description": "Crawlee and Cheerio project in javascript.", "version": "0.0", "meta": { "templateId": "js-crawlee-cheerio" }, "input": "./input_schema.json", "dockerfile": "./Dockerfile"}{ "title": "CheerioCrawler Template", "type": "object", "schemaVersion": 1, "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "URLs to start with.", "editor": "requestListSources", "prefill": [ { "url": "https://apify.com" } ] }, "checkFunction": { "title": "Check function", "type": "string", "description": "Function applied to each page", "editor": "javascript", "prefill": "async ({ $, request, log }) => { return $('title').text() === 'Hi' ; }" }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Select proxies to be used by your crawler.", "prefill": { "useApifyProxy": true }, "editor": "proxy" } }}1import { Actor } from 'apify';2import { CheerioCrawler } from 'crawlee';3
4import { Dataset, createCheerioRouter } from 'crawlee';5
6// Parses given string to js function7const parseCheckFunction = (fnStr) => {8 try {9 const checkFunction = eval(fnStr);10 if (typeof checkFunction !== 'function') {11 throw new Error('Check function is not a function');12 }13 return checkFunction;14 } catch(err) {15 console.error(err);16 throw new Error('Cannot parse check function');17 }18}19
20// Initialize the Apify SDK21await Actor.init();22
23// Get input24const {25 startUrls,26 checkFunction: checkFunctionString,27 proxyConfiguration: proxyConfigurationOptions,28} = await Actor.getInput();29
30// Parse check function31const checkFunction = parseCheckFunction(checkFunctionString);32
33// Generate proxy config34const proxyConfiguration = await Actor.createProxyConfiguration(proxyConfigurationOptions);35
36// Number of failed checks is kept in state, we need to base the exit code of actor on that37let { failedChecksCount } = await Actor.getValue('my-crawling-state') || { failedChecksCount: 0 };38Actor.on('migrating', () => {39 Actor.setValue('my-crawling-state', { failedChecksCount });40});41
42const router = createCheerioRouter();43
44router.addDefaultHandler(async (arg, ...rest) => {45 const isOk = await checkFunction(arg, ...rest);46 if (!isOk) failedChecksCount++;47 await Dataset.pushData({48 url: arg.request.loadedUrl,49 isOk,50 });51});52
53const crawler = new CheerioCrawler({54 proxyConfiguration,55 requestHandler: router,56});57
58await crawler.run(startUrls);59
60if (failedChecksCount > 0) {61 await Actor.fail('Check failed on some of the pages');62} else {63 await Actor.exit();64}# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed filesnode_modules
# git folder.gitroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf{ "extends": "@apify", "root": true}# This file tells Git which files shouldn't be added to source control
.DS_Store.ideadistnode_modulesapify_storagestorage{ "name": "crawlee-cheerio-javascript", "version": "0.0.1", "type": "module", "description": "This is a boilerplate of an Apify actor.", "engines": { "node": ">=16.0.0" }, "dependencies": { "apify": "^3.0.0", "crawlee": "^3.0.0" }, "devDependencies": { "@apify/eslint-config": "^0.3.1", "eslint": "^8.36.0" }, "scripts": { "start": "node src/main.js", "lint": "eslint ./src --ext .js,.jsx", "lint:fix": "eslint ./src --ext .js,.jsx --fix", "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1" }, "author": "It's not you it's me", "license": "ISC"}