Zip Download of KV Store avatar
Zip Download of KV Store
Try for free

No credit card required

View all Actors
Zip Download of KV Store

Zip Download of KV Store

vaclavrut/downloadkvstorezip
Try for free

No credit card required

Creates a zip file from all items in the key-value store, zips them, and downloads them in a unique file. On input, you can specify the number of the files or keep it null to download them all. The key-value store must be under your Apify account.

Dockerfile

1# This is a template for a Dockerfile used to run acts in Actor system.
2# The base image name below is set during the act build, based on user settings.
3# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/user
4FROM apify/actor-node-basic:v0.21.10
5
6# Second, copy just package.json and package-lock.json since it should be
7# the only file that affects "npm install" in the next step, to speed up the build
8COPY package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14 && npm install --only=prod --no-optional \
15 && echo "Installed NPM packages:" \
16 && (npm list --all || true) \
17 && echo "Node.js version:" \
18 && node --version \
19 && echo "NPM version:" \
20 && npm --version
21
22# Copy source code to container
23# Do this in the last step, to have fast build if only the source code changed
24COPY  . ./
25
26# NOTE: The CMD is already defined by the base image.
27# Uncomment this for local node inspector debugging:
28# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]

main.js

1const Apify = require('apify');
2const ApifyClient = require('apify-client');
3const AdmZip = require('adm-zip');
4
5async function asyncForEach(array, max, callback) {
6    let itemCount = max !== null ? max : array.length;
7    for (let index = 0; index < itemCount; index++) {
8        console.log("Solving " + index + " from " + array.length);
9        await callback(array[index], index, array)
10    }
11}
12
13Apify.main(async () => {
14    const input = await Apify.getValue('INPUT');
15    const storeId = input.storeId;
16    const maximumItems = input.numberOfItemsInZipFromKv ? input.numberOfItemsInZipFromKv : null;
17
18    console.log("Loading data from KV with ID: " + storeId);
19
20    if(maximumItems !== null){
21        console.log("Going to download only:" + maximumItems + " items from the KV store.")
22    }else{
23        console.log("Going to download whole KV store");
24    }
25    const zip = await new AdmZip();
26
27    const apifyClient = new ApifyClient();
28    const keyValueStores = apifyClient.keyValueStores;
29    apifyClient.setOptions({storeId: storeId});
30    const keys = await keyValueStores.listKeys();
31
32    console.log("Total number of items in store: " + keys.items.length);
33
34    await asyncForEach(keys.items, maximumItems, async (item) => {
35        console.dir(item)
36        if (item.key && item.key !== "INPUT") {
37            console.log(`Processing file: ${item.key}`)
38            const itemData = await keyValueStores.getRecord({key: item.key});
39            try{
40                await zip.addFile(item.key, new Buffer(itemData.body));
41            }catch{
42                console.log("Failed item.")
43                console.dir(itemData)
44            }
45        
46        }
47    });
48
49    const willSendthis = await zip.toBuffer();
50    console.log("Saving the zip file.")
51
52    await Apify.setValue("zip_file.zip", willSendthis, {contentType: 'application/zip'});
53    const enviroment = await Apify.getEnv();
54
55    console.log("Download the file here.");
56    console.log("https://api.apify.com/v2/key-value-stores/" + enviroment.defaultKeyValueStoreId + "/records/zip_file.zip?disableRedirect=true")
57});

package.json

1{
2    "name": "apify-project",
3    "version": "0.0.1",
4    "description": "",
5    "author": "It's not you it's me",
6    "license": "ISC",
7    "dependencies": {
8        "apify": "0.21.10",
9        "apify-client": "0.6.0",
10        "adm-zip": "latest"
11    },
12    "scripts": {
13        "start": "node main.js"
14    }
15}
Developer
Maintained by Community
Actor metrics
  • 4 monthly users
  • 0 stars
  • Created in Jul 2018
  • Modified 25 days ago
Categories