Actor picture

Zip Key-value Store

jaroslavhejlek/zip-key-value-store

Takes the ID of the key-value store, archives all their keys into a zip file, and saves them into the key-value store of the actor. For more than 1000 keys, multiple zip files are created. If their total size is bigger than the actor's available memory, it creates multiple smaller zip files.

No credit card required

Author's avatarJaroslav Hejlek
  • Modified
  • Users12
  • Runs219
Actor picture

Zip Key-value Store

Based on the apify/actor-node-basic:v0.21.10 Docker image (see docs).

const Apify = require('apify');
const Promise = require('bluebird');
const archiver = require('archiver');

let FILES_PER_ARCHIVE = 1000;

const archiveKey = async (keyValueStores, zip, storeId, item) => {
    console.log('Archiving key:', item);
    const response = await keyValueStores.getRecord({ storeId, key: item.key, disableBodyParser: true });
    zip.append(
        response.body, 
        {
            name: item.key,
        }
    );
}

const zipKeys = async (keyValueStores, storeId, keys) => {
    const buffers = [];
    await new Promise(async (resolve, reject) => {
        const zip = new archiver('zip', {
          zlib: { level: 9 } // Sets the compression level.
        });
        
        await Promise.map(
            keys, 
            key => archiveKey(keyValueStores, zip, storeId, key),
            { concurrency: 1 }
        );
        
        zip.on('data', chunk => {
            buffers.push(chunk)
        });
        zip.on('end', () => {
            console.log('End called');
            resolve();
        });
        zip.on('close', function() {
          console.log(archiver.pointer() + ' total bytes');
        });
        zip.on('error', reject);
        
        zip.finalize();
    })
    return Buffer.concat(buffers);
}

Apify.main(async () => {
    const input = await Apify.getValue('INPUT');
    
    const storeId = input.keyValueStoreId;
    FILES_PER_ARCHIVE = Math.min(input.filesPerZipFile, FILES_PER_ARCHIVE);
    
    // Get input
    const { client } = Apify;
    const keyValueStores = client.keyValueStores;
    
    // get key value store to check that it exists
    const store = await keyValueStores.getStore({ storeId });
    
    if (!store) {
        console.error('Store with key', input.keyValueStoreId, 'does not exist');
        process.exit(1);
    }
    
    let { items, nextExclusiveStartKey } = await keyValueStores.listKeys({ 
        storeId, 
        limit: FILES_PER_ARCHIVE,
    });
    let zipCounter = 1;
    while (items && items.length) {
        console.log('Found', items.length, 'keys');
        if (nextExclusiveStartKey !== null) console.log('Store contains more keys');
        
        
        const zipBuffer = await zipKeys(keyValueStores, storeId, items);
        console.log('Zip is ready');
        const zipName = `zip-${zipCounter}.zip`;
        console.log('Saving zip to key-value store');
        await Apify.setValue(zipName, zipBuffer, { contentType: 'application/zip' });
        console.log('Outputed', items.length, 'as', zipName);
        zipCounter++;
        
        if (nextExclusiveStartKey) {
            const listResponse = await keyValueStores.listKeys({ 
                storeId, 
                exclusiveStartKey: nextExclusiveStartKey, 
                limit: FILES_PER_ARCHIVE
            });
            items = listResponse.items;
            nextExclusiveStartKey = listResponse.nextExclusiveStartKey;
        } else {
            items = null;
        }
    }
});