Icon Burglar
Deprecated
Pricing
Pay per usage
Go to Store
Icon Burglar
Deprecated
This act takes the URLs prepared by a crawler and downloads the images to finally zip them
0.0 (0)
Pricing
Pay per usage
1
Total users
6
Monthly users
1
Runs succeeded
0%
Last modified
3 years ago
Dockerfile
# This is a template for a Dockerfile used to run acts in Actor system.# The base image name below is set during the act build, based on user settings.# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/userFROM apify/actor-node-basic:v0.21.10
# Second, copy just package.json and package-lock.json since it should be# the only file that affects "npm install" in the next step, to speed up the buildCOPY package*.json ./
# Install NPM packages, skip optional and development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \ && npm install --only=prod --no-optional \ && echo "Installed NPM packages:" \ && (npm list --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version
# Copy source code to container# Do this in the last step, to have fast build if only the source code changedCOPY . ./
# NOTE: The CMD is already defined by the base image.# Uncomment this for local node inspector debugging:# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]
package.json
{ "name": "apify-project", "version": "0.0.1", "description": "", "author": "It's not you it's me", "license": "ISC", "dependencies": { "apify": "0.21.10", "underscore": "latest", "request-promise": "latest", "bluebird": "latest" }, "scripts": { "start": "node main.js" }}
main.js
1// Get all the libraries this crawler uses2const Apify = require('apify');3const _ = require('underscore');4const rp = require('request-promise');5const Promise = require('bluebird');6
7Apify.main(async () => {8 // Get act input and validate it9 const input = await Apify.getValue('INPUT');10 console.log('Input:')11 console.dir(input);12 if (!input || !input._id) {13 throw new Error('Input is missing the "_id" attribute. Did you start it from crawler finish webhook?');14 }15 const executionId = input._id;16 17 // Print info about crawler run18 const crawlerRunDetails = await Apify.client.crawlers.getExecutionDetails({ executionId });19 if (!crawlerRunDetails) {20 throw new Error(`There is no crawler run with ID: "${executionId}"`);21 }22 console.log(`Details of the crawler run (ID: ${executionId}):`);23 console.dir(crawlerRunDetails);24 25 // Iterate through all crawler results and count them26 // Here is the place where you can add something more adventurous :)27 console.log(`Counting results from crawler run...`);28 29 const limit = 100;30 let offset = 0;31 let totalItems = 0;32 let results;33
34 results = await Apify.client.crawlers.getExecutionResults({ 35 executionId,36 limit,37 offset38 });39 40 // Prepare each result for downloading41 await Promise.each(results.items[0].pageFunctionResult, async function(value, i){42 43 console.log('url', value.url)44 45 // Move each result in the key-value store46 const file = await rp ({47 url : value.url, encoding : null48 });49 50 // Attach the right filename and encoding51 await Apify.setValue(value.file+'.png', file, { contentType : 'image/png'})52 });53 54 // Take the key-value store and zip data (this uses a different Actor (https://www.apify.com/jaroslavhejlek/zip-key-value-store))55 const run = await Apify.call('jaroslavhejlek/zip-key-value-store', { "keyValueStoreId": process.env.APIFY_DEFAULT_KEY_VALUE_STORE_ID, "filesPerZipFile": 2000 });56 console.dir(run);57 58});