Actor Build Starter

No credit card required

Actor Build Starter

Actor Build Starter

fjvs0283/actor-batch-builder

No credit card required

Run builds for multiple actors in your account simultaneously. This can be useful when many actors in a given project have been updated. For example, the documentation might have been updated in 50 actors. This tool will help you trigger actor builds for all 50 at once.

.editorconfig

1root = true 2 3[*] 4indent_style = space 5indent_size = 4 6charset = utf-8 7trim_trailing_whitespace = true 8insert_final_newline = true 9end_of_line = lf 10

.eslintrc

1{ 2 "extends": "@apify" 3} 4

.gitignore

1# This file tells Git which files shouldn't be added to source control 2 3.idea 4node_modules 5 6apify_storage

Dockerfile

1# First, specify the base Docker image. You can read more about 2# the available images at https://sdk.apify.com/docs/guides/docker-images 3# You can also use any other image from Docker Hub. 4FROM apify/actor-node:16 5 6# Second, copy just package.json and package-lock.json since it should be 7# the only file that affects "npm install" in the next step, to speed up the build 8COPY package*.json ./ 9 10# Install NPM packages, skip optional and development dependencies to 11# keep the image small. Avoid logging too much and print the dependency 12# tree for debugging 13RUN npm --quiet set progress=false \ 14 && npm install --only=prod --no-optional \ 15 && echo "Installed NPM packages:" \ 16 && (npm list --all || true) \ 17 && echo "Node.js version:" \ 18 && node --version \ 19 && echo "NPM version:" \ 20 && npm --version 21 22# Next, copy the remaining files and directories with the source code. 23# Since we do this after NPM install, quick build will be really fast 24# for most source file changes. 25COPY . ./ 26 27# Optionally, specify how to launch the source code of your actor. 28# By default, Apify's base Docker images define the CMD instruction 29# that runs the Node.js source code using the command specified 30# in the "scripts.start" section of the package.json file. 31# In short, the instruction looks something like this: 32# 33# CMD npm start 34

INPUT_SCHEMA.json

1{ 2 "title": "PuppeteerCrawler Template", 3 "type": "object", 4 "schemaVersion": 1, 5 "properties": { 6 "batchType": { 7 "title": "Batch Mode", 8 "description": "Choose how to identify the target actors to batch process. You can target actors that contain specific text as part of the actor name, target actors by their id, or target all actors in your account.", 9 "type": "string", 10 "editor": "select", 11 "enum": [ 12 "actorNameContains", 13 "actorIds", 14 "all" 15 ], 16 "enumTitles": [ 17 "Actor name contains", 18 "Actor IDs", 19 "All actors" 20 ], 21 "default": "actorNameContains" 22 }, 23 "actorNameContains": { 24 "title": "Actor Name Contains", 25 "type": "string", 26 "description": "Text included in names of target actors.", 27 "editor": "textfield", 28 "prefill": "my-awesome" 29 }, 30 "actorIds": { 31 "title": "Actor IDs", 32 "type": "array", 33 "description": "Enter the ids of the target actors.", 34 "prefill": [ 35 "abc123", 36 "abc123" 37 ], 38 "editor": "json" 39 } 40 } 41}

apify.json

1{ 2 "name": "actor-batch-builder", 3 "version": "0.0", 4 "buildTag": "latest", 5 "env": null, 6 "template": "project_empty" 7} 8

main.js

1import Apify from 'apify'; 2 3const { utils: { log, sleep } } = Apify; 4 5Apify.main(async () => { 6 7 const client = Apify.newClient(); 8 const { items: actors } = await client.actors().list(); 9 const { batchType, actorNameContains, actorIds } = await Apify.getInput(); 10 11 let batchTypeDescription = ""; 12 let targetActors = []; 13 let actorCount = 0; 14 let totalBuildsFinished = 0; 15 let requestsCount = 0; 16 17 let results = { 18 "builds": [], 19 "stats": { 20 "total": 0, 21 "failed": 0, 22 "succeeded": 0, 23 "requests": 0 24 } 25 }; 26 27 28 async function prepTargetActors() { 29 for (const item in actors) { 30 if (batchType == "actorNameContains" && actors[item].name.includes(actorNameContains)) { 31 targetActors.push(actors[item]); 32 batchTypeDescription = "matching part of the actor name."; 33 } 34 else if (batchType == "actorIds" && actorIds.includes(actors[item].id)) { 35 targetActors.push(actors[item]); 36 batchTypeDescription = "matching selected actor ids."; 37 } 38 else if (batchType == "all") { 39 targetActors.push(actors[item]); 40 batchTypeDescription = "for all actors in the account."; 41 } 42 } 43 log.info(`Starting builds ${batchTypeDescription}`); 44 log.info('–––––––––––––––––––––––––––––––––––––––––'); 45 } 46 47 48 async function buildActors(targetActors) { 49 let actorsInProgress = []; 50 for (const item in targetActors) { 51 actorCount++; 52 await sleep(1000); 53 const { id: actorId, name: actorName } = targetActors[item]; 54 const actor = client.actor(actorId); 55 56 try { 57 const startBuild = await actor.build('0.0'); 58 requestsCount++; 59 60 log.info(`${actorCount}.`); 61 log.info(`actor name: ${actorName}`); 62 log.info(`build id: ${startBuild.id}`); 63 log.info(`started at: ${startBuild.startedAt}`); 64 log.info(`status: ${startBuild.status}`); 65 log.info('–––––––––––––––––––––––––––––––––––––––––'); 66 67 actorsInProgress.push(targetActors[item]); 68 69 } catch (error) { 70 actorCount--; 71 72 log.warning(`skipping actor ${actorName} due to ${error.type}.`); 73 log.warning(`actor id: ${actorId}`); 74 log.info('–––––––––––––––––––––––––––––––––––––––––'); 75 } 76 } 77 log.info(`builds started for ${actorCount} actors.`); 78 log.info('waiting for builds to finish...'); 79 log.info('–––––––––––––––––––––––––––––––––––––––––'); 80 81 return actorsInProgress; 82 }; 83 84 85 async function checkBuilds(targetActor) { 86 const actor = client.actor(targetActor.id); 87 const { id: actorId, name: actorName, buildtag: actorBuildTag } = await actor.get(); 88 requestsCount++; 89 90 let buildFinshed = false; 91 while (!buildFinshed) { 92 await sleep(delay); 93 requestsCount++; 94 const { items: actorBuilds } = await actor.builds().list({ 'desc': true }); 95 const buildInProgress = actorBuilds[0]; 96 97 let result = { ...buildInProgress, actorId, actorName, actorBuildTag }; 98 99 if (buildInProgress.status == 'RUNNING') { 100 buildFinshed = false; 101 102 } else if (buildInProgress.status == 'SUCCEEDED') { 103 results.stats.total++; 104 results.stats.succeeded++; 105 106 buildFinshed = true; 107 totalBuildsFinished++; 108 results.builds.push(result); 109 110 log.info(`${totalBuildsFinished} of ${actorCount}`); 111 log.info(`actor name: ${actorName}`); 112 log.info(`finished at: ${buildInProgress.finishedAt}`); 113 log.info(`status: ${buildInProgress.status}`); 114 log.info('–––––––––––––––––––––––––––––––––––––––––'); 115 116 } else if (buildInProgress.status == 'FAILED') { 117 results.stats.total++; 118 results.stats.failed++; 119 120 buildFinshed = true; 121 totalBuildsFinished++; 122 results.builds.push(result); 123 124 log.info(`${totalBuildsFinished} of ${actorCount}`); 125 log.error(`actor name: ${actorName}`); 126 log.error(`finished at: ${buildInProgress.finishedAt}`); 127 log.error(`status: ${buildInProgress.status}`); 128 log.info('–––––––––––––––––––––––––––––––––––––––––'); 129 } 130 } 131 } 132 133 async function startChecks(actorsInProgress) { 134 for (const item in actorsInProgress) { 135 checkBuilds(actorsInProgress[item]); 136 } 137 while (totalBuildsFinished != actorsInProgress.length) { 138 await sleep(delay); 139 } 140 results.stats.requests = requestsCount; 141 } 142 143 144 await prepTargetActors(); 145 let actorsInProgress = await buildActors(targetActors.slice(0)); 146 let delay = (actorCount > 0) ? 10000 : 0; 147 await sleep(delay); 148 await startChecks(actorsInProgress); 149 150 log.info('Done.'); 151 log.info(`total builds requested: ${results.stats.total}`); 152 log.info(`total builds successful: ${results.stats.succeeded}`); 153 log.info(`total builds failed: ${results.stats.failed}`); 154 log.info(`total API calls: ${results.stats.requests}`); 155 156 const store = await Apify.openKeyValueStore(); 157 await store.setValue('STATS', results.stats); 158 await Apify.pushData(results.builds); 159});

package.json

1{ 2 "name": "actor-batch-builder", 3 "version": "0.0.1", 4 "type": "module", 5 "description": "This is a boilerplate of an Apify actor.", 6 "dependencies": { 7 "apify": "^2.0.7" 8 }, 9 "devDependencies": { 10 "@apify/eslint-config": "^0.1.3", 11 "eslint": "^7.0.0" 12 }, 13 "scripts": { 14 "start": "node main.js", 15 "lint": "./node_modules/.bin/eslint ./src --ext .js,.jsx", 16 "lint:fix": "./node_modules/.bin/eslint ./src --ext .js,.jsx --fix", 17 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1" 18 }, 19 "author": "It's not you it's me", 20 "license": "ISC" 21} 22
Developer
Maintained by Community
Actor stats
  • 3 users
  • 748 runs
  • Modified 7 months ago

You might also like these Actors