No credit card required

Actor Build Starter

fjvs0283/actor-batch-builder

No credit card required

Run builds for multiple actors in your account simultaneously. This can be useful when many actors in a given project have been updated. For example, the documentation might have been updated in 50 actors. This tool will help you trigger actor builds for all 50 at once.

.editorconfig

1root = true
2
3[*]
4indent_style = space
5indent_size = 4
6charset = utf-8
7trim_trailing_whitespace = true
8insert_final_newline = true
9end_of_line = lf

.eslintrc

1{
2    "extends": "@apify"
3}

.gitignore

1# This file tells Git which files shouldn't be added to source control
2
3.idea
4node_modules
5
6apify_storage

Dockerfile

1# First, specify the base Docker image. You can read more about
2# the available images at https://sdk.apify.com/docs/guides/docker-images
3# You can also use any other image from Docker Hub.
4FROM apify/actor-node:16
5
6# Second, copy just package.json and package-lock.json since it should be
7# the only file that affects "npm install" in the next step, to speed up the build
8COPY package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14 && npm install --only=prod --no-optional \
15 && echo "Installed NPM packages:" \
16 && (npm list --all || true) \
17 && echo "Node.js version:" \
18 && node --version \
19 && echo "NPM version:" \
20 && npm --version
21
22# Next, copy the remaining files and directories with the source code.
23# Since we do this after NPM install, quick build will be really fast
24# for most source file changes.
25COPY . ./
26
27# Optionally, specify how to launch the source code of your actor.
28# By default, Apify's base Docker images define the CMD instruction
29# that runs the Node.js source code using the command specified
30# in the "scripts.start" section of the package.json file.
31# In short, the instruction looks something like this:
32#
33# CMD npm start

INPUT_SCHEMA.json

1{
2    "title": "PuppeteerCrawler Template",
3    "type": "object",
4    "schemaVersion": 1,
5    "properties": {
6        "batchType": {
7            "title": "Batch Mode",
8            "description": "Choose how to identify the target actors to batch process. You can target actors that contain specific text as part of the actor name, target actors by their id, or target all actors in your account.",
9            "type": "string",
10            "editor": "select",
11            "enum": [
12                "actorNameContains",
13                "actorIds",
14                "all"
15            ],
16            "enumTitles": [
17                "Actor name contains",
18                "Actor IDs",
19                "All actors"
20            ],
21            "default": "actorNameContains"
22        },
23        "actorNameContains": {
24            "title": "Actor Name Contains",
25            "type": "string",
26            "description": "Text included in names of target actors.",
27            "editor": "textfield",
28            "prefill": "my-awesome"
29        },
30        "actorIds": {
31            "title": "Actor IDs",
32            "type": "array",
33            "description": "Enter the ids of the target actors.",
34            "prefill": [
35                "abc123",
36                "abc123"
37            ],
38            "editor": "json"
39        }
40    }
41}

apify.json

1{
2	"name": "actor-batch-builder",
3	"version": "0.0",
4	"buildTag": "latest",
5	"env": null,
6	"template": "project_empty"
7}

main.js

1import Apify from 'apify';
2
3const { utils: { log, sleep } } = Apify;
4
5Apify.main(async () => {
6
7    const client = Apify.newClient();
8    const { items: actors } = await client.actors().list();
9    const { batchType, actorNameContains, actorIds } = await Apify.getInput();
10
11    let batchTypeDescription = "";
12    let targetActors = [];
13    let actorCount = 0;
14    let totalBuildsFinished = 0;
15    let requestsCount = 0;
16
17    let results = {
18        "builds": [],
19        "stats": {
20            "total": 0,
21            "failed": 0,
22            "succeeded": 0,
23            "requests": 0
24        }
25    };
26
27
28    async function prepTargetActors() {
29        for (const item in actors) {
30            if (batchType == "actorNameContains" && actors[item].name.includes(actorNameContains)) {
31                targetActors.push(actors[item]);
32                batchTypeDescription = "matching part of the actor name.";
33            }
34            else if (batchType == "actorIds" && actorIds.includes(actors[item].id)) {
35                targetActors.push(actors[item]);
36                batchTypeDescription = "matching selected actor ids.";
37            }
38            else if (batchType == "all") {
39                targetActors.push(actors[item]);
40                batchTypeDescription = "for all actors in the account.";
41            }
42        }
43        log.info(`Starting builds ${batchTypeDescription}`);
44        log.info('–––––––––––––––––––––––––––––––––––––––––');
45    }
46
47
48    async function buildActors(targetActors) {
49        let actorsInProgress = [];
50        for (const item in targetActors) {
51            actorCount++;
52            await sleep(1000);
53            const { id: actorId, name: actorName } = targetActors[item];
54            const actor = client.actor(actorId);
55
56            try {
57                const startBuild = await actor.build('0.0');
58                requestsCount++;
59
60                log.info(`${actorCount}.`);
61                log.info(`actor name: ${actorName}`);
62                log.info(`build id:   ${startBuild.id}`);
63                log.info(`started at: ${startBuild.startedAt}`);
64                log.info(`status:     ${startBuild.status}`);
65                log.info('–––––––––––––––––––––––––––––––––––––––––');
66
67                actorsInProgress.push(targetActors[item]);
68
69            } catch (error) {
70                actorCount--;
71
72                log.warning(`skipping actor ${actorName} due to ${error.type}.`);
73                log.warning(`actor id: ${actorId}`);
74                log.info('–––––––––––––––––––––––––––––––––––––––––');
75            }
76        }
77        log.info(`builds started for ${actorCount} actors.`);
78        log.info('waiting for builds to finish...');
79        log.info('–––––––––––––––––––––––––––––––––––––––––');
80
81        return actorsInProgress;
82    };
83
84
85    async function checkBuilds(targetActor) {
86        const actor = client.actor(targetActor.id);
87        const { id: actorId, name: actorName, buildtag: actorBuildTag } = await actor.get();
88        requestsCount++;
89
90        let buildFinshed = false;
91        while (!buildFinshed) {
92            await sleep(delay);
93            requestsCount++;
94            const { items: actorBuilds } = await actor.builds().list({ 'desc': true });
95            const buildInProgress = actorBuilds[0];
96
97            let result = { ...buildInProgress, actorId, actorName, actorBuildTag };
98
99            if (buildInProgress.status == 'RUNNING') {
100                buildFinshed = false;
101
102            } else if (buildInProgress.status == 'SUCCEEDED') {
103                results.stats.total++;
104                results.stats.succeeded++;
105
106                buildFinshed = true;
107                totalBuildsFinished++;
108                results.builds.push(result);
109
110                log.info(`${totalBuildsFinished} of ${actorCount}`);
111                log.info(`actor name:  ${actorName}`);
112                log.info(`finished at: ${buildInProgress.finishedAt}`);
113                log.info(`status:      ${buildInProgress.status}`);
114                log.info('–––––––––––––––––––––––––––––––––––––––––');
115
116            } else if (buildInProgress.status == 'FAILED') {
117                results.stats.total++;
118                results.stats.failed++;
119
120                buildFinshed = true;
121                totalBuildsFinished++;
122                results.builds.push(result);
123
124                log.info(`${totalBuildsFinished} of ${actorCount}`);
125                log.error(`actor name:  ${actorName}`);
126                log.error(`finished at: ${buildInProgress.finishedAt}`);
127                log.error(`status:      ${buildInProgress.status}`);
128                log.info('–––––––––––––––––––––––––––––––––––––––––');
129            }
130        }
131    }
132
133    async function startChecks(actorsInProgress) {
134        for (const item in actorsInProgress) {
135            checkBuilds(actorsInProgress[item]);
136        }
137        while (totalBuildsFinished != actorsInProgress.length) {
138            await sleep(delay);
139        }
140        results.stats.requests = requestsCount;
141    }
142
143
144    await prepTargetActors();
145    let actorsInProgress = await buildActors(targetActors.slice(0));
146    let delay = (actorCount > 0) ? 10000 : 0;
147    await sleep(delay);
148    await startChecks(actorsInProgress);
149
150    log.info('Done.');
151    log.info(`total builds requested:  ${results.stats.total}`);
152    log.info(`total builds successful: ${results.stats.succeeded}`);
153    log.info(`total builds failed:     ${results.stats.failed}`);
154    log.info(`total API calls:         ${results.stats.requests}`);
155
156    const store = await Apify.openKeyValueStore();
157    await store.setValue('STATS', results.stats);
158    await Apify.pushData(results.builds);
159});

package.json

1{
2	"name": "actor-batch-builder",
3	"version": "0.0.1",
4	"type": "module",
5	"description": "This is a boilerplate of an Apify actor.",
6	"dependencies": {
7		"apify": "^2.0.7"
8	},
9	"devDependencies": {
10		"@apify/eslint-config": "^0.1.3",
11		"eslint": "^7.0.0"
12	},
13	"scripts": {
14		"start": "node main.js",
15		"lint": "./node_modules/.bin/eslint ./src --ext .js,.jsx",
16		"lint:fix": "./node_modules/.bin/eslint ./src --ext .js,.jsx --fix",
17		"test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
18	},
19	"author": "It's not you it's me",
20	"license": "ISC"
21}

Developer

Francisco Villarreal

Actor Metrics

1 monthly user
2 stars
>99% runs succeeded
Created in Oct 2021
Modified 2 years ago

Categories

Developer tools

Automation

🧲 Social Media Leads Analyzer

apify/social-media-leads-analyzer

Add a website and extract emails, phone numbers, and social media details. Scrape basic social media profile data from 8 platforms. Export results in JSON, CSV, HTML, use APIs, schedule runs, build integrations, and more.

Apify

Indeed job scraper

curious_coder/indeed-scraper

This is an actively maintained scraper which can extract job postings and hiring company details at scale from any indeed search results page for a fixed monthly rental price. Well documented with examples and demos

Curious Coder

729

TikTok Data Extractor

clockworks/free-tiktok-scraper

Extract data about videos, users, and channels based on hashtags or scrape full user profiles including posts, total likes, name, nickname, numbers of comments, shares, followers, following, and more.

Clockworks

17k

198

Google Maps Scraper

compass/crawler-google-places

Extract data from hundreds of Google Maps locations and businesses. Get Google Maps data including reviews, images, contact info, opening hours, location, popular times, prices & more. Export scraped data, run the scraper via API, schedule and monitor runs, or integrate with other tools.

Compass

78.9k

561

Tiktok Hashtag Scraper

clockworks/tiktok-hashtag-scraper

Scrape TikTok hashtag data. Just add one or more hashtags and extract TikTok videos with that hashtag: URLs, likes, country of creation, video and music metadata, TikTok creator data. Export scraped data, run the scraper via API, schedule and monitor runs or integrate with other tools.

Clockworks

2.4k

Tiktok Video Scraper

clockworks/tiktok-video-scraper

Extract data from chosen tiktoks. Just add a TikTok URL and get TikTok video and profile data: URLs, numbers of shares, followers, hashtags, hearts, video, and music metadata. Export scraped data, run the scraper via API, schedule and monitor runs or integrate with other tools.

Clockworks

1.6k

Tiktok Profile Scraper

clockworks/tiktok-profile-scraper

Extract data from TikTok profiles and videos they post. Get TikTok profile data, URLs, numbers of shares, followers, comments, hearts, video and music metadata. Export scraped data, run the scraper via API, schedule and monitor runs or integrate with other tools.

Clockworks

4.7k

302

Indeed Scraper

misceres/indeed-scraper

Scrape jobs posted on Indeed. Get detailed information from this job portal about saved and sponsored jobs. Specify the search based on location with the output attributes position, location, and description.

Misceres

4.8k

122

TikTok Comments Scraper

clockworks/tiktok-comments-scraper

Extract TikTok comments. Just add a TikTok URL and get TikTok video and profile data: comments, URLs, numbers of shares, followers, hashtags, hearts, video, and music metadata. Export scraped data, run the scraper via API, schedule and monitor runs or integrate with other tools.

Clockworks

4.2k

Google Maps Reviews Scraper

compass/Google-Maps-Reviews-Scraper

Extract all reviews of Google Maps places using place URLs. Get review text, published date, response from owner, review URL, and reviewer's details. Download scraped data, run the scraper via API, schedule and monitor runs or integrate with other tools.

Compass

4.3k