Abort And Resurrect
No credit card required
No credit card required
Aborts actor runs in bulk and resurrects them with new input and options.
Actor Metrics
2 monthly users
No reviews yet
2 bookmarks
Created in Feb 2023
Modified 2 years ago
Ondra Urban
1# configurations
2.idea
3
4# crawlee and apify storage folders
5apify_storage
6crawlee_storage
7storage
8
9# installed files
10node_modules
11
12# git folder
13.git1storage
2node_modules1{
2 "name": "actor-resurrect-with-new-input",
3 "version": "0.0.1",
4 "type": "module",
5 "description": "This is an example of an Apify actor.",
6 "dependencies": {
7 "apify": "^3.0.0"
8 },
9 "devDependencies": {},
10 "scripts": {
11 "start": "node src/main.js",
12 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
13 },
14 "author": "It's not you it's me",
15 "license": "ISC"
16}1# Specify the base Docker image. You can read more about
2# the available images at https://sdk.apify.com/docs/guides/docker-images
3# You can also use any other image from Docker Hub.
4FROM apify/actor-node:16
5
6# Copy just package.json and package-lock.json
7# to speed up the build using Docker layer cache.
8COPY package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14 && npm install --omit=dev --omit=optional \
15 && echo "Installed NPM packages:" \
16 && (npm list --omit=dev --all || true) \
17 && echo "Node.js version:" \
18 && node --version \
19 && echo "NPM version:" \
20 && npm --version \
21 && rm -r ~/.npm
22
23# Next, copy the remaining files and directories with the source code.
24# Since we do this after NPM install, quick build will be really fast
25# for most source file changes.
26COPY . ./
27
28
29# Run the image.
30CMD npm start --silent1{
2 "actorSpecification": 1,
3 "name": "abort-and-resurrect",
4 "title": "Abort & Resurrect With New Input",
5 "description": "Aborts selected actor or task runs ",
6 "version": "0.0",
7 "buildTag": "version-0",
8 "input": "./input-schema.json"
9}1{
2 "title": "Actor input schema",
3 "description": "This is actor input schema",
4 "type": "object",
5 "schemaVersion": 1,
6 "properties": {
7 "runIds": {
8 "title": "Run IDs",
9 "type": "array",
10 "description": "IDs of individual runs you want to abort and resurrect.",
11 "editor": "stringList",
12 "default": []
13 },
14 "actorIds": {
15 "title": "Actor IDs",
16 "type": "array",
17 "description": "When you provide an actor ID, all running runs of the actor will be aborted and resurrected.",
18 "editor": "stringList",
19 "default": []
20 },
21 "taskIds": {
22 "title": "Task IDs",
23 "type": "array",
24 "description": "When you provide a task ID, all running runs of the task will be aborted and resurrected.",
25 "editor": "stringList",
26 "default": []
27 },
28 "transformInputSource": {
29 "title": "Transform Input Function",
30 "type": "string",
31 "description": "This function will be used to transform the original input of the actor run into the new input. The `input` and `option` arguments are mutable and you should make your changes directly in the provided objects.",
32 "editor": "javascript",
33 "prefill": "function transformInput({ input, options }) {\n input.foo = 'bar';\n options.memory = 2048;\n }"
34 }
35 },
36 "required": [
37 "transformInputSource"
38 ]
39}1# Default ignored files
2/shelf/
3/workspace.xml
4# Editor-based HTTP Client requests
5/httpRequests/1<?xml version="1.0" encoding="UTF-8"?>
2<module type="WEB_MODULE" version="4">
3 <component name="NewModuleRootManager">
4 <content url="file://$MODULE_DIR$">
5 <excludeFolder url="file://$MODULE_DIR$/temp" />
6 <excludeFolder url="file://$MODULE_DIR$/.tmp" />
7 <excludeFolder url="file://$MODULE_DIR$/tmp" />
8 </content>
9 <orderEntry type="inheritedJdk" />
10 <orderEntry type="sourceFolder" forTests="false" />
11 </component>
12</module>1<?xml version="1.0" encoding="UTF-8"?>
2<project version="4">
3 <component name="JavaScriptLibraryMappings">
4 <includedPredefinedLibrary name="Node.js Core" />
5 </component>
6</project>1<?xml version="1.0" encoding="UTF-8"?>
2<project version="4">
3 <component name="ProjectModuleManager">
4 <modules>
5 <module fileurl="file://$PROJECT_DIR$/.idea/actor-resurrect-with-new-input.iml" filepath="$PROJECT_DIR$/.idea/actor-resurrect-with-new-input.iml" />
6 </modules>
7 </component>
8</project>1<?xml version="1.0" encoding="UTF-8"?>
2<project version="4">
3 <component name="AutoImportSettings">
4 <option name="autoReloadType" value="SELECTIVE" />
5 </component>
6 <component name="ChangeListManager">
7 <list default="true" id="3df15837-a08d-4cd4-934f-7978ad5b076a" name="Changes" comment="" />
8 <option name="SHOW_DIALOG" value="false" />
9 <option name="HIGHLIGHT_CONFLICTS" value="true" />
10 <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
11 <option name="LAST_RESOLUTION" value="IGNORE" />
12 </component>
13 <component name="MarkdownSettingsMigration">
14 <option name="stateVersion" value="1" />
15 </component>
16 <component name="ProjectId" id="2LM98PiybFyoOanD9AfPLRS0ExL" />
17 <component name="ProjectViewState">
18 <option name="hideEmptyMiddlePackages" value="true" />
19 <option name="showLibraryContents" value="true" />
20 </component>
21 <component name="PropertiesComponent"><![CDATA[{
22 "keyToString": {
23 "RunOnceActivity.OpenProjectViewOnStart": "true",
24 "RunOnceActivity.ShowReadmeOnStart": "true",
25 "WebServerToolWindowFactoryState": "false",
26 "javascript.nodejs.core.library.configured.version": "16.16.0",
27 "javascript.nodejs.core.library.typings.version": "16.18.12",
28 "node.js.detected.package.eslint": "true",
29 "node.js.detected.package.tslint": "true",
30 "node.js.selected.package.eslint": "(autodetect)",
31 "node.js.selected.package.tslint": "(autodetect)",
32 "nodejs_package_manager_path": "npm",
33 "ts.external.directory.path": "/Users/mnmkng/Library/Application Support/JetBrains/Toolbox/apps/WebStorm/ch-0/223.8617.44/WebStorm.app/Contents/plugins/javascript-impl/jsLanguageServicesImpl/external",
34 "vue.rearranger.settings.migration": "true"
35 }
36}]]></component>
37 <component name="RecentsManager">
38 <key name="MoveFile.RECENT_KEYS">
39 <recent name="$PROJECT_DIR$" />
40 </key>
41 </component>
42 <component name="RunManager">
43 <configuration name="main.js" type="NodeJSConfigurationType" temporary="true" nameIsGenerated="true" path-to-js-file="src/main.js" working-dir="$PROJECT_DIR$">
44 <envs>
45 <env name="APIFY_TOKEN" value="apify_api_iPPqEVeHrcqhYcFSFb4NXTnXqCixIm0SBRnJ" />
46 </envs>
47 <method v="2" />
48 </configuration>
49 <recent_temporary>
50 <list>
51 <item itemvalue="Node.js.main.js" />
52 </list>
53 </recent_temporary>
54 </component>
55 <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
56 <component name="TaskManager">
57 <task active="true" id="Default" summary="Default task">
58 <changelist id="3df15837-a08d-4cd4-934f-7978ad5b076a" name="Changes" comment="" />
59 <created>1675669924902</created>
60 <option name="number" value="Default" />
61 <option name="presentableId" value="Default" />
62 <updated>1675669924902</updated>
63 <workItem from="1675669926072" duration="13837000" />
64 </task>
65 <servers />
66 </component>
67 <component name="TypeScriptGeneratedFilesManager">
68 <option name="version" value="3" />
69 </component>
70 <component name="XDebuggerManager">
71 <breakpoint-manager>
72 <breakpoints>
73 <line-breakpoint enabled="true" type="javascript">
74 <url>file://$PROJECT_DIR$/src/utils.js</url>
75 <line>19</line>
76 <properties lambdaOrdinal="-1" />
77 <option name="timeStamp" value="3" />
78 </line-breakpoint>
79 <line-breakpoint enabled="true" type="javascript">
80 <url>file://$PROJECT_DIR$/src/utils.js</url>
81 <line>69</line>
82 <properties lambdaOrdinal="-1" />
83 <option name="timeStamp" value="7" />
84 </line-breakpoint>
85 <line-breakpoint enabled="true" type="javascript">
86 <url>file://$PROJECT_DIR$/src/utils.js</url>
87 <line>74</line>
88 <properties lambdaOrdinal="-1" />
89 <option name="timeStamp" value="8" />
90 </line-breakpoint>
91 </breakpoints>
92 </breakpoint-manager>
93 </component>
94</project>1import { Actor } from 'apify';
2import { evalFunctionOrThrow, diff, fetchRuns, fetchTaskRuns, fetchActorRuns } from "./utils.js";
3
4await Actor.init();
5const client = Actor.newClient();
6
7const { runIds, actorIds, taskIds, transformInputSource } = await Actor.getInput();
8
9// PREP & VALIDATION
10/**
11 * @param {object} context
12 * @param {object} context.input
13 * @param {object} context.options
14 */
15const transformInput = evalFunctionOrThrow(transformInputSource);
16
17if (!(runIds.length || actorIds.length || taskIds.length)) {
18 throw new Error('Did not receive any run IDs, actor IDs or task IDs. Aborting.');
19}
20
21// FETCH RUNNING RUNS
22const collectedRuns = [];
23
24if (runIds.length) {
25 const runs = await fetchRuns(client, runIds);
26 console.log(`Fetched ${runs.length} runs based on their IDs.`);
27 collectedRuns.push(...runs);
28}
29
30if (actorIds.length) {
31 const runs = await fetchActorRuns(client, actorIds)
32 collectedRuns.push(...runs);
33}
34
35if (taskIds.length) {
36 const runs = await fetchTaskRuns(client, taskIds)
37 collectedRuns.push(...runs);
38}
39
40// TRANSFORM, ABORT, RESURRECT
41const processedRunPromises = collectedRuns.map(async (run) => {
42 const runClient = client.run(run.id);
43 const kvsClient = runClient.keyValueStore();
44
45 console.log(`[${run.id}] Fetching input.`);
46 const { value: input } = await kvsClient.getRecord('INPUT');
47 const newInput = JSON.parse(JSON.stringify(input));
48 // The run object contains the options in a different
49 // format than the API expects them in resurrect.
50 const apiOptions = {
51 build: run.options.build,
52 memory: run.options.memoryMbytes,
53 timeout: run.options.timeoutSecs,
54 }
55 const newOptions = { ...apiOptions };
56
57 console.log(`[${run.id}] Transforming input with transformInput function.`);
58 transformInput({
59 input: newInput,
60 options: newOptions,
61 })
62 const inputDiff = diff(input, newInput);
63 if (inputDiff) {
64 console.log(`[${run.id}] Changed input:`, inputDiff);
65 }
66 const optionsDiff = diff(apiOptions, newOptions);
67 if (optionsDiff) {
68 console.log(`[${run.id}] Changed options:`, optionsDiff);
69 }
70
71 console.log(`[${run.id}] Aborting run.`)
72 await runClient.abort({ gracefully: true });
73 // Graceful abort can take some time.
74 await runClient.waitForFinish();
75
76 console.log(`[${run.id}] Saving new input to key-value store.`)
77 await kvsClient.setRecord({
78 key: 'INPUT',
79 value: newInput,
80 })
81
82 console.log(`[${run.id}] Resurrecting run.`)
83 const resurrectedRun = await runClient.resurrect(newOptions);
84
85 console.log(`[${run.id}] Saving input and run information to dataset.`)
86 await Actor.pushData({
87 input,
88 newInput,
89 abortedRun: run,
90 resurrectedRun,
91 })
92})
93
94const processedRuns = await Promise.all(processedRunPromises);
95console.log(`Aborted and resurrected ${processedRuns.length} runs. The end.`);
96await Actor.exit();1import vm from 'node:vm'
2
3export const fetchRuns = async (apifyClient, runIds) => {
4 const runPromises = runIds.map(async (runId) => {
5 return apifyClient.run(runId).get();
6 })
7 const runs = await Promise.all(runPromises);
8 return runs.filter(run => run.status === 'RUNNING');
9}
10
11export const fetchActorRuns = async (apifyClient, actorIds) => {
12 const runPromises = actorIds.map(async (actorId) => {
13 const { items } = await apifyClient.actor(actorId).runs().list({ status: 'RUNNING'});
14 const runIds = items.map(run => run.id);
15 const runs = await fetchRuns(apifyClient, runIds);
16 console.log(`Fetched ${runs.length} running runs of actor: ${actorId}`);
17 return runs;
18 })
19 const runsInArrays = await Promise.all(runPromises);
20 return runsInArrays.flat();
21}
22
23export const fetchTaskRuns = async (apifyClient, taskIds) => {
24 const runPromises = taskIds.map(async (taskId) => {
25 const { items } = await apifyClient.task(taskId).runs().list({ status: 'RUNNING'});
26 const runIds = items.map(run => run.id);
27 const runs = await fetchRuns(apifyClient, runIds);
28 console.log(`Fetched ${runs.length} running runs of actor: ${taskId}`);
29 return runs;
30 })
31 const runsInArrays = await Promise.all(runPromises);
32 return runsInArrays.flat();
33}
34
35/**
36 * Transforms a page function string into a Function object.
37 * @param {string} funcString
38 * @return {Function}
39 */
40export const evalFunctionOrThrow = (funcString) => {
41 let func;
42
43 try {
44 func = vm.runInThisContext(`(${funcString})`);
45 } catch (err) {
46 throw new Error(`Compilation of transformInput source code failed.\n${err.message}\n${err.stack.substr(err.stack.indexOf('\n'))}`);
47 }
48
49 if (typeof func !== 'function') throw new Error('`transformInput` function is not a function.');
50
51 return func;
52};
53
54/**
55 * Creates a diff between the first and second object.
56 * @param {object} obj1
57 * @param {object} obj2
58 * @returns {object|undefined}
59 */
60export const diff = (obj1, obj2) => {
61 const result = {};
62 if (Object.is(obj1, obj2)) {
63 return undefined;
64 }
65 if (!obj2 || typeof obj2 !== 'object') {
66 return obj2;
67 }
68 Object.keys(obj1 || {}).concat(Object.keys(obj2 || {})).forEach(key => {
69 if (obj2[key] !== obj1[key] && !Object.is(obj1[key], obj2[key])) {
70 result[key] = obj2[key];
71 }
72 if (typeof obj2[key] === 'object' && typeof obj1[key] === 'object') {
73 const value = diff(obj1[key], obj2[key]);
74 if (value !== undefined) {
75 result[key] = value;
76 }
77 }
78 });
79 // clean empty objects
80 for (const [k, v] of Object.entries(result)) {
81 if (!Object.keys(v).length) {
82 delete result[k];
83 }
84 }
85 return result;
86}