1const Apify = require('apify');
2const Promise = require('bluebird');
3const archiver = require('archiver');
4
5let FILES_PER_ARCHIVE = 1000;
6
7const archiveKey = async (keyValueStores, zip, storeId, item) => {
8 console.log('Archiving key:', item);
9 const response = await keyValueStores.getRecord({ storeId, key: item.key, disableBodyParser: true });
10 zip.append(
11 response.body,
12 {
13 name: item.key,
14 }
15 );
16}
17
18const zipKeys = async (keyValueStores, storeId, keys) => {
19 const buffers = [];
20 await new Promise(async (resolve, reject) => {
21 const zip = new archiver('zip', {
22 zlib: { level: 9 } // Sets the compression level.
23 });
24
25 await Promise.map(
26 keys,
27 key => archiveKey(keyValueStores, zip, storeId, key),
28 { concurrency: 1 }
29 );
30
31 zip.on('data', chunk => {
32 buffers.push(chunk)
33 });
34 zip.on('end', () => {
35 console.log('End called');
36 resolve();
37 });
38 zip.on('close', function() {
39 console.log(archiver.pointer() + ' total bytes');
40 });
41 zip.on('error', reject);
42
43 zip.finalize();
44 })
45 return Buffer.concat(buffers);
46}
47
48Apify.main(async () => {
49 const input = await Apify.getValue('INPUT');
50
51 const storeId = input.keyValueStoreId;
52 FILES_PER_ARCHIVE = Math.min(input.filesPerZipFile, FILES_PER_ARCHIVE);
53
54 // Get input
55 const { client } = Apify;
56 const keyValueStores = client.keyValueStores;
57
58 // get key value store to check that it exists
59 const store = await keyValueStores.getStore({ storeId });
60
61 if (!store) {
62 console.error('Store with key', input.keyValueStoreId, 'does not exist');
63 process.exit(1);
64 }
65
66 let { items, nextExclusiveStartKey } = await keyValueStores.listKeys({
67 storeId,
68 limit: FILES_PER_ARCHIVE,
69 });
70 let zipCounter = 1;
71 while (items && items.length) {
72 console.log('Found', items.length, 'keys');
73 if (nextExclusiveStartKey !== null) console.log('Store contains more keys');
74
75
76 const zipBuffer = await zipKeys(keyValueStores, storeId, items);
77 console.log('Zip is ready');
78 const zipName = `zip-${zipCounter}.zip`;
79 console.log('Saving zip to key-value store');
80 await Apify.setValue(zipName, zipBuffer, { contentType: 'application/zip' });
81 console.log('Outputed', items.length, 'as', zipName);
82 zipCounter++;
83
84 if (nextExclusiveStartKey) {
85 const listResponse = await keyValueStores.listKeys({
86 storeId,
87 exclusiveStartKey: nextExclusiveStartKey,
88 limit: FILES_PER_ARCHIVE
89 });
90 items = listResponse.items;
91 nextExclusiveStartKey = listResponse.nextExclusiveStartKey;
92 } else {
93 items = null;
94 }
95 }
96});