Actor picture

Zip Key-value Store

jaroslavhejlek/zip-key-value-store

Takes the ID of the key-value store, archives all their keys into a zip file, and saves them into the key-value store of the actor. For more than 1000 keys, multiple zip files are created. If their total size is bigger than the actor's available memory, it creates multiple smaller zip files.

No credit card required

Author's avatarJaroslav Hejlek
  • Modified
  • Users19
  • Runs886
Actor picture
Zip Key-value Store

Dockerfile

# This is a template for a Dockerfile used to run acts in Actor system.
# The base image name below is set during the act build, based on user settings.
# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/user
FROM apify/actor-node-basic:v0.21.10

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Copy source code to container
# Do this in the last step, to have fast build if only the source code changed
COPY  . ./

# NOTE: The CMD is already defined by the base image.
# Uncomment this for local node inspector debugging:
# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]

package.json

{
    "name": "apify-project",
    "version": "0.0.1",
    "description": "",
    "author": "It's not you it's me",
    "license": "ISC",
    "dependencies": {
        "apify": "0.21.10",
        "bluebird": "latest",
        "archiver": "latest"
    },
    "scripts": {
        "start": "node main.js"
    }
}

main.js

const Apify = require('apify');
const Promise = require('bluebird');
const archiver = require('archiver');

let FILES_PER_ARCHIVE = 1000;

const archiveKey = async (keyValueStores, zip, storeId, item) => {
    console.log('Archiving key:', item);
    const response = await keyValueStores.getRecord({ storeId, key: item.key, disableBodyParser: true });
    zip.append(
        response.body, 
        {
            name: item.key,
        }
    );
}

const zipKeys = async (keyValueStores, storeId, keys) => {
    const buffers = [];
    await new Promise(async (resolve, reject) => {
        const zip = new archiver('zip', {
          zlib: { level: 9 } // Sets the compression level.
        });
        
        await Promise.map(
            keys, 
            key => archiveKey(keyValueStores, zip, storeId, key),
            { concurrency: 1 }
        );
        
        zip.on('data', chunk => {
            buffers.push(chunk)
        });
        zip.on('end', () => {
            console.log('End called');
            resolve();
        });
        zip.on('close', function() {
          console.log(archiver.pointer() + ' total bytes');
        });
        zip.on('error', reject);
        
        zip.finalize();
    })
    return Buffer.concat(buffers);
}

Apify.main(async () => {
    const input = await Apify.getValue('INPUT');
    
    const storeId = input.keyValueStoreId;
    FILES_PER_ARCHIVE = Math.min(input.filesPerZipFile, FILES_PER_ARCHIVE);
    
    // Get input
    const { client } = Apify;
    const keyValueStores = client.keyValueStores;
    
    // get key value store to check that it exists
    const store = await keyValueStores.getStore({ storeId });
    
    if (!store) {
        console.error('Store with key', input.keyValueStoreId, 'does not exist');
        process.exit(1);
    }
    
    let { items, nextExclusiveStartKey } = await keyValueStores.listKeys({ 
        storeId, 
        limit: FILES_PER_ARCHIVE,
    });
    let zipCounter = 1;
    while (items && items.length) {
        console.log('Found', items.length, 'keys');
        if (nextExclusiveStartKey !== null) console.log('Store contains more keys');
        
        
        const zipBuffer = await zipKeys(keyValueStores, storeId, items);
        console.log('Zip is ready');
        const zipName = `zip-${zipCounter}.zip`;
        console.log('Saving zip to key-value store');
        await Apify.setValue(zipName, zipBuffer, { contentType: 'application/zip' });
        console.log('Outputed', items.length, 'as', zipName);
        zipCounter++;
        
        if (nextExclusiveStartKey) {
            const listResponse = await keyValueStores.listKeys({ 
                storeId, 
                exclusiveStartKey: nextExclusiveStartKey, 
                limit: FILES_PER_ARCHIVE
            });
            items = listResponse.items;
            nextExclusiveStartKey = listResponse.nextExclusiveStartKey;
        } else {
            items = null;
        }
    }
});