Zip Key-value Store
Try for free
No credit card required
Go to Store
Zip Key-value Store
jaroslavhejlek/zip-key-value-store
Try for free
No credit card required
Takes the ID of the key-value store, archives all their keys into a zip file, and saves them into the key-value store of the actor. For more than 1000 keys, multiple zip files are created. If their total size is bigger than the actor's available memory, it creates multiple smaller zip files.
Dockerfile
1# This is a template for a Dockerfile used to run acts in Actor system.
2# The base image name below is set during the act build, based on user settings.
3# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/user
4FROM apify/actor-node-basic:v0.21.10
5
6# Second, copy just package.json and package-lock.json since it should be
7# the only file that affects "npm install" in the next step, to speed up the build
8COPY package*.json ./
9
10# Install NPM packages, skip optional and development dependencies to
11# keep the image small. Avoid logging too much and print the dependency
12# tree for debugging
13RUN npm --quiet set progress=false \
14 && npm install --only=prod --no-optional \
15 && echo "Installed NPM packages:" \
16 && (npm list --all || true) \
17 && echo "Node.js version:" \
18 && node --version \
19 && echo "NPM version:" \
20 && npm --version
21
22# Copy source code to container
23# Do this in the last step, to have fast build if only the source code changed
24COPY . ./
25
26# NOTE: The CMD is already defined by the base image.
27# Uncomment this for local node inspector debugging:
28# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]package.json
1{
2 "name": "apify-project",
3 "version": "0.0.1",
4 "description": "",
5 "author": "It's not you it's me",
6 "license": "ISC",
7 "dependencies": {
8 "apify": "0.21.10",
9 "bluebird": "latest",
10 "archiver": "latest"
11 },
12 "scripts": {
13 "start": "node main.js"
14 }
15}main.js
1const Apify = require('apify');
2const Promise = require('bluebird');
3const archiver = require('archiver');
4
5let FILES_PER_ARCHIVE = 1000;
6
7const archiveKey = async (keyValueStores, zip, storeId, item) => {
8 console.log('Archiving key:', item);
9 const response = await keyValueStores.getRecord({ storeId, key: item.key, disableBodyParser: true });
10 zip.append(
11 response.body,
12 {
13 name: item.key,
14 }
15 );
16}
17
18const zipKeys = async (keyValueStores, storeId, keys) => {
19 const buffers = [];
20 await new Promise(async (resolve, reject) => {
21 const zip = new archiver('zip', {
22 zlib: { level: 9 } // Sets the compression level.
23 });
24
25 await Promise.map(
26 keys,
27 key => archiveKey(keyValueStores, zip, storeId, key),
28 { concurrency: 1 }
29 );
30
31 zip.on('data', chunk => {
32 buffers.push(chunk)
33 });
34 zip.on('end', () => {
35 console.log('End called');
36 resolve();
37 });
38 zip.on('close', function() {
39 console.log(archiver.pointer() + ' total bytes');
40 });
41 zip.on('error', reject);
42
43 zip.finalize();
44 })
45 return Buffer.concat(buffers);
46}
47
48Apify.main(async () => {
49 const input = await Apify.getValue('INPUT');
50
51 const storeId = input.keyValueStoreId;
52 FILES_PER_ARCHIVE = Math.min(input.filesPerZipFile, FILES_PER_ARCHIVE);
53
54 // Get input
55 const { client } = Apify;
56 const keyValueStores = client.keyValueStores;
57
58 // get key value store to check that it exists
59 const store = await keyValueStores.getStore({ storeId });
60
61 if (!store) {
62 console.error('Store with key', input.keyValueStoreId, 'does not exist');
63 process.exit(1);
64 }
65
66 let { items, nextExclusiveStartKey } = await keyValueStores.listKeys({
67 storeId,
68 limit: FILES_PER_ARCHIVE,
69 });
70 let zipCounter = 1;
71 while (items && items.length) {
72 console.log('Found', items.length, 'keys');
73 if (nextExclusiveStartKey !== null) console.log('Store contains more keys');
74
75
76 const zipBuffer = await zipKeys(keyValueStores, storeId, items);
77 console.log('Zip is ready');
78 const zipName = `zip-${zipCounter}.zip`;
79 console.log('Saving zip to key-value store');
80 await Apify.setValue(zipName, zipBuffer, { contentType: 'application/zip' });
81 console.log('Outputed', items.length, 'as', zipName);
82 zipCounter++;
83
84 if (nextExclusiveStartKey) {
85 const listResponse = await keyValueStores.listKeys({
86 storeId,
87 exclusiveStartKey: nextExclusiveStartKey,
88 limit: FILES_PER_ARCHIVE
89 });
90 items = listResponse.items;
91 nextExclusiveStartKey = listResponse.nextExclusiveStartKey;
92 } else {
93 items = null;
94 }
95 }
96});Developer
Jaroslav HejlekMaintained by Community
Actor Metrics
6 monthly users
-
5 stars
95% runs succeeded
Created in May 2018
Modified a year ago
Categories
