Abort And Resurrect

No credit card required

Abort And Resurrect

Abort And Resurrect

mnmkng/abort-and-resurrect

No credit card required

Aborts actor runs in bulk and resurrects them with new input and options.

.dockerignore

1# configurations 2.idea 3 4# crawlee and apify storage folders 5apify_storage 6crawlee_storage 7storage 8 9# installed files 10node_modules 11 12# git folder 13.git 14

.gitignore

1storage 2node_modules

package.json

1{ 2 "name": "actor-resurrect-with-new-input", 3 "version": "0.0.1", 4 "type": "module", 5 "description": "This is an example of an Apify actor.", 6 "dependencies": { 7 "apify": "^3.0.0" 8 }, 9 "devDependencies": {}, 10 "scripts": { 11 "start": "node src/main.js", 12 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1" 13 }, 14 "author": "It's not you it's me", 15 "license": "ISC" 16} 17

.actor/Dockerfile

1# Specify the base Docker image. You can read more about 2# the available images at https://sdk.apify.com/docs/guides/docker-images 3# You can also use any other image from Docker Hub. 4FROM apify/actor-node:16 5 6# Copy just package.json and package-lock.json 7# to speed up the build using Docker layer cache. 8COPY package*.json ./ 9 10# Install NPM packages, skip optional and development dependencies to 11# keep the image small. Avoid logging too much and print the dependency 12# tree for debugging 13RUN npm --quiet set progress=false \ 14 && npm install --omit=dev --omit=optional \ 15 && echo "Installed NPM packages:" \ 16 && (npm list --omit=dev --all || true) \ 17 && echo "Node.js version:" \ 18 && node --version \ 19 && echo "NPM version:" \ 20 && npm --version \ 21 && rm -r ~/.npm 22 23# Next, copy the remaining files and directories with the source code. 24# Since we do this after NPM install, quick build will be really fast 25# for most source file changes. 26COPY . ./ 27 28 29# Run the image. 30CMD npm start --silent 31

.actor/actor.json

1{ 2 "actorSpecification": 1, 3 "name": "abort-and-resurrect", 4 "title": "Abort & Resurrect With New Input", 5 "description": "Aborts selected actor or task runs ", 6 "version": "0.0", 7 "buildTag": "version-0", 8 "input": "./input-schema.json" 9} 10

.actor/input-schema.json

1{ 2 "title": "Actor input schema", 3 "description": "This is actor input schema", 4 "type": "object", 5 "schemaVersion": 1, 6 "properties": { 7 "runIds": { 8 "title": "Run IDs", 9 "type": "array", 10 "description": "IDs of individual runs you want to abort and resurrect.", 11 "editor": "stringList", 12 "default": [] 13 }, 14 "actorIds": { 15 "title": "Actor IDs", 16 "type": "array", 17 "description": "When you provide an actor ID, all running runs of the actor will be aborted and resurrected.", 18 "editor": "stringList", 19 "default": [] 20 }, 21 "taskIds": { 22 "title": "Task IDs", 23 "type": "array", 24 "description": "When you provide a task ID, all running runs of the task will be aborted and resurrected.", 25 "editor": "stringList", 26 "default": [] 27 }, 28 "transformInputSource": { 29 "title": "Transform Input Function", 30 "type": "string", 31 "description": "This function will be used to transform the original input of the actor run into the new input. The `input` and `option` arguments are mutable and you should make your changes directly in the provided objects.", 32 "editor": "javascript", 33 "prefill": "function transformInput({ input, options }) {\n input.foo = 'bar';\n options.memory = 2048;\n }" 34 } 35 }, 36 "required": [ 37 "transformInputSource" 38 ] 39} 40

.idea/.gitignore

1# Default ignored files 2/shelf/ 3/workspace.xml 4# Editor-based HTTP Client requests 5/httpRequests/ 6

.idea/actor-resurrect-with-new-input.iml

1<?xml version="1.0" encoding="UTF-8"?> 2<module type="WEB_MODULE" version="4"> 3 <component name="NewModuleRootManager"> 4 <content url="file://$MODULE_DIR$"> 5 <excludeFolder url="file://$MODULE_DIR$/temp" /> 6 <excludeFolder url="file://$MODULE_DIR$/.tmp" /> 7 <excludeFolder url="file://$MODULE_DIR$/tmp" /> 8 </content> 9 <orderEntry type="inheritedJdk" /> 10 <orderEntry type="sourceFolder" forTests="false" /> 11 </component> 12</module>

.idea/jsLibraryMappings.xml

1<?xml version="1.0" encoding="UTF-8"?> 2<project version="4"> 3 <component name="JavaScriptLibraryMappings"> 4 <includedPredefinedLibrary name="Node.js Core" /> 5 </component> 6</project>

.idea/modules.xml

1<?xml version="1.0" encoding="UTF-8"?> 2<project version="4"> 3 <component name="ProjectModuleManager"> 4 <modules> 5 <module fileurl="file://$PROJECT_DIR$/.idea/actor-resurrect-with-new-input.iml" filepath="$PROJECT_DIR$/.idea/actor-resurrect-with-new-input.iml" /> 6 </modules> 7 </component> 8</project>

.idea/workspace.xml

1<?xml version="1.0" encoding="UTF-8"?> 2<project version="4"> 3 <component name="AutoImportSettings"> 4 <option name="autoReloadType" value="SELECTIVE" /> 5 </component> 6 <component name="ChangeListManager"> 7 <list default="true" id="3df15837-a08d-4cd4-934f-7978ad5b076a" name="Changes" comment="" /> 8 <option name="SHOW_DIALOG" value="false" /> 9 <option name="HIGHLIGHT_CONFLICTS" value="true" /> 10 <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" /> 11 <option name="LAST_RESOLUTION" value="IGNORE" /> 12 </component> 13 <component name="MarkdownSettingsMigration"> 14 <option name="stateVersion" value="1" /> 15 </component> 16 <component name="ProjectId" id="2LM98PiybFyoOanD9AfPLRS0ExL" /> 17 <component name="ProjectViewState"> 18 <option name="hideEmptyMiddlePackages" value="true" /> 19 <option name="showLibraryContents" value="true" /> 20 </component> 21 <component name="PropertiesComponent"><![CDATA[{ 22 "keyToString": { 23 "RunOnceActivity.OpenProjectViewOnStart": "true", 24 "RunOnceActivity.ShowReadmeOnStart": "true", 25 "WebServerToolWindowFactoryState": "false", 26 "javascript.nodejs.core.library.configured.version": "16.16.0", 27 "javascript.nodejs.core.library.typings.version": "16.18.12", 28 "node.js.detected.package.eslint": "true", 29 "node.js.detected.package.tslint": "true", 30 "node.js.selected.package.eslint": "(autodetect)", 31 "node.js.selected.package.tslint": "(autodetect)", 32 "nodejs_package_manager_path": "npm", 33 "ts.external.directory.path": "/Users/mnmkng/Library/Application Support/JetBrains/Toolbox/apps/WebStorm/ch-0/223.8617.44/WebStorm.app/Contents/plugins/javascript-impl/jsLanguageServicesImpl/external", 34 "vue.rearranger.settings.migration": "true" 35 } 36}]]></component> 37 <component name="RecentsManager"> 38 <key name="MoveFile.RECENT_KEYS"> 39 <recent name="$PROJECT_DIR$" /> 40 </key> 41 </component> 42 <component name="RunManager"> 43 <configuration name="main.js" type="NodeJSConfigurationType" temporary="true" nameIsGenerated="true" path-to-js-file="src/main.js" working-dir="$PROJECT_DIR$"> 44 <envs> 45 <env name="APIFY_TOKEN" value="apify_api_iPPqEVeHrcqhYcFSFb4NXTnXqCixIm0SBRnJ" /> 46 </envs> 47 <method v="2" /> 48 </configuration> 49 <recent_temporary> 50 <list> 51 <item itemvalue="Node.js.main.js" /> 52 </list> 53 </recent_temporary> 54 </component> 55 <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" /> 56 <component name="TaskManager"> 57 <task active="true" id="Default" summary="Default task"> 58 <changelist id="3df15837-a08d-4cd4-934f-7978ad5b076a" name="Changes" comment="" /> 59 <created>1675669924902</created> 60 <option name="number" value="Default" /> 61 <option name="presentableId" value="Default" /> 62 <updated>1675669924902</updated> 63 <workItem from="1675669926072" duration="13837000" /> 64 </task> 65 <servers /> 66 </component> 67 <component name="TypeScriptGeneratedFilesManager"> 68 <option name="version" value="3" /> 69 </component> 70 <component name="XDebuggerManager"> 71 <breakpoint-manager> 72 <breakpoints> 73 <line-breakpoint enabled="true" type="javascript"> 74 <url>file://$PROJECT_DIR$/src/utils.js</url> 75 <line>19</line> 76 <properties lambdaOrdinal="-1" /> 77 <option name="timeStamp" value="3" /> 78 </line-breakpoint> 79 <line-breakpoint enabled="true" type="javascript"> 80 <url>file://$PROJECT_DIR$/src/utils.js</url> 81 <line>69</line> 82 <properties lambdaOrdinal="-1" /> 83 <option name="timeStamp" value="7" /> 84 </line-breakpoint> 85 <line-breakpoint enabled="true" type="javascript"> 86 <url>file://$PROJECT_DIR$/src/utils.js</url> 87 <line>74</line> 88 <properties lambdaOrdinal="-1" /> 89 <option name="timeStamp" value="8" /> 90 </line-breakpoint> 91 </breakpoints> 92 </breakpoint-manager> 93 </component> 94</project>

src/main.js

1import { Actor } from 'apify'; 2import { evalFunctionOrThrow, diff, fetchRuns, fetchTaskRuns, fetchActorRuns } from "./utils.js"; 3 4await Actor.init(); 5const client = Actor.newClient(); 6 7const { runIds, actorIds, taskIds, transformInputSource } = await Actor.getInput(); 8 9// PREP & VALIDATION 10/** 11 * @param {object} context 12 * @param {object} context.input 13 * @param {object} context.options 14 */ 15const transformInput = evalFunctionOrThrow(transformInputSource); 16 17if (!(runIds.length || actorIds.length || taskIds.length)) { 18 throw new Error('Did not receive any run IDs, actor IDs or task IDs. Aborting.'); 19} 20 21// FETCH RUNNING RUNS 22const collectedRuns = []; 23 24if (runIds.length) { 25 const runs = await fetchRuns(client, runIds); 26 console.log(`Fetched ${runs.length} runs based on their IDs.`); 27 collectedRuns.push(...runs); 28} 29 30if (actorIds.length) { 31 const runs = await fetchActorRuns(client, actorIds) 32 collectedRuns.push(...runs); 33} 34 35if (taskIds.length) { 36 const runs = await fetchTaskRuns(client, taskIds) 37 collectedRuns.push(...runs); 38} 39 40// TRANSFORM, ABORT, RESURRECT 41const processedRunPromises = collectedRuns.map(async (run) => { 42 const runClient = client.run(run.id); 43 const kvsClient = runClient.keyValueStore(); 44 45 console.log(`[${run.id}] Fetching input.`); 46 const { value: input } = await kvsClient.getRecord('INPUT'); 47 const newInput = JSON.parse(JSON.stringify(input)); 48 // The run object contains the options in a different 49 // format than the API expects them in resurrect. 50 const apiOptions = { 51 build: run.options.build, 52 memory: run.options.memoryMbytes, 53 timeout: run.options.timeoutSecs, 54 } 55 const newOptions = { ...apiOptions }; 56 57 console.log(`[${run.id}] Transforming input with transformInput function.`); 58 transformInput({ 59 input: newInput, 60 options: newOptions, 61 }) 62 const inputDiff = diff(input, newInput); 63 if (inputDiff) { 64 console.log(`[${run.id}] Changed input:`, inputDiff); 65 } 66 const optionsDiff = diff(apiOptions, newOptions); 67 if (optionsDiff) { 68 console.log(`[${run.id}] Changed options:`, optionsDiff); 69 } 70 71 console.log(`[${run.id}] Aborting run.`) 72 await runClient.abort({ gracefully: true }); 73 // Graceful abort can take some time. 74 await runClient.waitForFinish(); 75 76 console.log(`[${run.id}] Saving new input to key-value store.`) 77 await kvsClient.setRecord({ 78 key: 'INPUT', 79 value: newInput, 80 }) 81 82 console.log(`[${run.id}] Resurrecting run.`) 83 const resurrectedRun = await runClient.resurrect(newOptions); 84 85 console.log(`[${run.id}] Saving input and run information to dataset.`) 86 await Actor.pushData({ 87 input, 88 newInput, 89 abortedRun: run, 90 resurrectedRun, 91 }) 92}) 93 94const processedRuns = await Promise.all(processedRunPromises); 95console.log(`Aborted and resurrected ${processedRuns.length} runs. The end.`); 96await Actor.exit(); 97

src/utils.js

1import vm from 'node:vm' 2 3export const fetchRuns = async (apifyClient, runIds) => { 4 const runPromises = runIds.map(async (runId) => { 5 return apifyClient.run(runId).get(); 6 }) 7 const runs = await Promise.all(runPromises); 8 return runs.filter(run => run.status === 'RUNNING'); 9} 10 11export const fetchActorRuns = async (apifyClient, actorIds) => { 12 const runPromises = actorIds.map(async (actorId) => { 13 const { items } = await apifyClient.actor(actorId).runs().list({ status: 'RUNNING'}); 14 const runIds = items.map(run => run.id); 15 const runs = await fetchRuns(apifyClient, runIds); 16 console.log(`Fetched ${runs.length} running runs of actor: ${actorId}`); 17 return runs; 18 }) 19 const runsInArrays = await Promise.all(runPromises); 20 return runsInArrays.flat(); 21} 22 23export const fetchTaskRuns = async (apifyClient, taskIds) => { 24 const runPromises = taskIds.map(async (taskId) => { 25 const { items } = await apifyClient.task(taskId).runs().list({ status: 'RUNNING'}); 26 const runIds = items.map(run => run.id); 27 const runs = await fetchRuns(apifyClient, runIds); 28 console.log(`Fetched ${runs.length} running runs of actor: ${taskId}`); 29 return runs; 30 }) 31 const runsInArrays = await Promise.all(runPromises); 32 return runsInArrays.flat(); 33} 34 35/** 36 * Transforms a page function string into a Function object. 37 * @param {string} funcString 38 * @return {Function} 39 */ 40export const evalFunctionOrThrow = (funcString) => { 41 let func; 42 43 try { 44 func = vm.runInThisContext(`(${funcString})`); 45 } catch (err) { 46 throw new Error(`Compilation of transformInput source code failed.\n${err.message}\n${err.stack.substr(err.stack.indexOf('\n'))}`); 47 } 48 49 if (typeof func !== 'function') throw new Error('`transformInput` function is not a function.'); 50 51 return func; 52}; 53 54/** 55 * Creates a diff between the first and second object. 56 * @param {object} obj1 57 * @param {object} obj2 58 * @returns {object|undefined} 59 */ 60export const diff = (obj1, obj2) => { 61 const result = {}; 62 if (Object.is(obj1, obj2)) { 63 return undefined; 64 } 65 if (!obj2 || typeof obj2 !== 'object') { 66 return obj2; 67 } 68 Object.keys(obj1 || {}).concat(Object.keys(obj2 || {})).forEach(key => { 69 if (obj2[key] !== obj1[key] && !Object.is(obj1[key], obj2[key])) { 70 result[key] = obj2[key]; 71 } 72 if (typeof obj2[key] === 'object' && typeof obj1[key] === 'object') { 73 const value = diff(obj1[key], obj2[key]); 74 if (value !== undefined) { 75 result[key] = value; 76 } 77 } 78 }); 79 // clean empty objects 80 for (const [k, v] of Object.entries(result)) { 81 if (!Object.keys(v).length) { 82 delete result[k]; 83 } 84 } 85 return result; 86} 87
Developer
Maintained by Community
Actor stats
  • 4 users
  • 21 runs
  • Modified 9 months ago
Categories

You might also like these Actors