Actor picture

Zip Download of KV Store

vaclavrut/downloadkvstorezip

Creates a zip file from all items in the key-value store, zips them, and downloads them in a unique file. On input, you can specify the number of the files or keep it null to download them all. The key-value store must be under your Apify account.

No credit card required

Author's avatarVaclav Rut
  • Modified
  • Users9
  • Runs27
Actor picture
Zip Download of KV Store

Dockerfile

# This is a template for a Dockerfile used to run acts in Actor system.
# The base image name below is set during the act build, based on user settings.
# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/user
FROM apify/actor-node-basic:v0.21.10

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Copy source code to container
# Do this in the last step, to have fast build if only the source code changed
COPY  . ./

# NOTE: The CMD is already defined by the base image.
# Uncomment this for local node inspector debugging:
# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]

package.json

{
    "name": "apify-project",
    "version": "0.0.1",
    "description": "",
    "author": "It's not you it's me",
    "license": "ISC",
    "dependencies": {
        "apify": "0.21.10",
        "apify-client": "0.6.0",
        "adm-zip": "latest"
    },
    "scripts": {
        "start": "node main.js"
    }
}

main.js

const Apify = require('apify');
const ApifyClient = require('apify-client');
const AdmZip = require('adm-zip');

async function asyncForEach(array, max, callback) {
    let itemCount = max !== null ? max : array.length;
    for (let index = 0; index < itemCount; index++) {
        console.log("Solving " + index + " from " + array.length);
        await callback(array[index], index, array)
    }
}

Apify.main(async () => {
    const input = await Apify.getValue('INPUT');
    const storeId = input.storeId;
    const maximumItems = input.numberOfItemsInZipFromKv ? input.numberOfItemsInZipFromKv : null;

    console.log("Loading data from KV with ID: " + storeId);

    if(maximumItems !== null){
        console.log("Going to download only:" + maximumItems + " items from the KV store.")
    }else{
        console.log("Going to download whole KV store");
    }
    const zip = await new AdmZip();

    const apifyClient = new ApifyClient();
    const keyValueStores = apifyClient.keyValueStores;
    apifyClient.setOptions({storeId: storeId});
    const keys = await keyValueStores.listKeys();

    console.log("Total number of items in store: " + keys.items.length);

    await asyncForEach(keys.items, maximumItems, async (item) => {
        if (item.key !== "INPUT") {
            const itemData = await keyValueStores.getRecord({key: item.key});
            await zip.addFile(item.key, new Buffer(itemData.body));
        }
    });

    const willSendthis = await zip.toBuffer();
    console.log("Saving the zip file.")

    await Apify.setValue("zip_file.zip", willSendthis, {contentType: 'application/zip'});
    const enviroment = await Apify.getEnv();

    console.log("Download the file here.");
    console.log("https://api.apify.com/v2/key-value-stores/" + enviroment.defaultKeyValueStoreId + "/records/zip_file.zip?disableRedirect=true")
});