Delete Named Storages

1const crypto = require('crypto');
2const Apify = require('apify');
3const moment = require('moment');
4
5const { utils: { log } } = Apify;
6
7const KEY_VALUE_STORE_NAME = 'DELETE-NAMED-STORAGES-SECURITY-STORE';
8
9exports.filterStoragesByPattern = function (storages, storageNamePattern) {
10    if (!storageNamePattern) return storages;
11    const pattern = new RegExp(storageNamePattern);
12    const storagesMatchingPattern = storages.filter((storage) => {
13        return pattern.test(storage.name);
14    });
15    log.debug(`Removed ${storages.length - storagesMatchingPattern.length} storages because their name did not match the provided pattern.`);
16    return storagesMatchingPattern;
17};
18
19exports.filterStoragesByDate = function (storages, date) {
20    if (!date) return storages;
21    const day = moment(date);
22    const storagesCreatedAtDate = storages.filter((storage) => {
23        return day.isSame(storage.createdAt, 'day');
24    });
25    log.debug(`Removed ${storages.length - storagesCreatedAtDate.length} storages because they were created at a different date.`);
26    return storagesCreatedAtDate;
27};
28
29exports.checkIfSafeToDelete = async function (storages) {
30    const kvs = await Apify.openKeyValueStore(KEY_VALUE_STORE_NAME);
31    const hash = hashStorages(storages);
32    const storagesToBeDeleted = await kvs.getValue(hash);
33    if (storagesToBeDeleted) {
34        log.info('Destruction credentials accepted. The selected storages will now be destroyed.');
35        return true;
36    }
37    await kvs.setValue(hash, storages);
38    log.warning('Are you sure you want to do this? Please confirm destruction by running this actor again with the same input.');
39    const link = kvs.getPublicUrl(hash);
40    log.info(`You can verify the storages to be deleted at: ${link}`);
41    return false;
42};
43
44function hashStorages(storages) {
45    const data = JSON.stringify(storages);
46    return crypto.createHash('md5')
47        .update(data)
48        .digest('hex');
49}

1const { client } = require('apify');
2
3const STORAGE_OPERATIONS = {
4    DATASET: {
5        listStorages: client.datasets.listDatasets,
6        deleteStorage: datasetId => client.datasets.deleteDataset({ datasetId }),
7    },
8    KEY_VALUE_STORE: {
9        listStorages: client.keyValueStores.listStores,
10        deleteStorage: storeId => client.keyValueStores.deleteStore({ storeId }),
11    },
12    REQUEST_QUEUE: {
13        listStorages: client.requestQueues.listQueues,
14        deleteStorage: queueId => client.requestQueues.deleteQueue({ queueId }),
15    },
16};
17
18const PROTECTED_STORAGES = [
19    'apify-cli-deployments',
20];
21
22module.exports = class Storages {
23    constructor(type) {
24        this.type = type;
25    }
26
27    async list(desc) {
28        const { listStorages } = STORAGE_OPERATIONS[this.type];
29        let hasNextPage = true;
30        let storages = [];
31        while (hasNextPage) {
32            const { items, offset, limit, total } = await listStorages({ desc });
33            const safeStorages = items.filter(storage => !PROTECTED_STORAGES.includes(storage.name));
34            storages = storages.concat(safeStorages);
35            hasNextPage = offset + limit < total;
36        }
37        return storages;
38    }
39
40    async deleteOne(storageId) {
41        const { deleteStorage } = STORAGE_OPERATIONS[this.type];
42        return deleteStorage(storageId);
43    }
44};

1const Apify = require('apify');
2const StoragesClient = require('./src/storages-client');
3const { filterStoragesByPattern, filterStoragesByDate, checkIfSafeToDelete } = require('./src/functions');
4
5const { utils: { log } } = Apify;
6
7Apify.main(async () => {
8    log.info('Loading input.');
9    const {
10        storageType,
11        matchingExpression,
12        createdAt,
13        maxDeletedStorages,
14        newestFirst,
15        forceDelete,
16    } = await Apify.getInput();
17
18    const storagesClient = new StoragesClient(storageType);
19    log.info(`Fetching ${storageType}S in ${newestFirst ? 'descending' : 'ascending'} order.`);
20    let storages = await storagesClient.list(newestFirst);
21
22    log.info(`Fetched ${storages.length} storage(s). Filtering.`);
23    storages = filterStoragesByPattern(storages, matchingExpression);
24    storages = filterStoragesByDate(storages, createdAt);
25    log.info(`${storages.length} storages were kept after filtering.`);
26
27    if (maxDeletedStorages) {
28        log.info(`Applying limit: Max ${maxDeletedStorages} storages will be deleted.`);
29        storages = storages.slice(0, maxDeletedStorages);
30    }
31
32    if (!storages.length) {
33        log.info('No storages selected for destruction. Aborting the process.');
34        return;
35    }
36
37    if (forceDelete) {
38        log.info('You have chosen immediate destruction. We certainly hope you did not make a mistake.');
39    } else {
40        log.info('Preparing for destruction of storages.');
41        const shouldDelete = await checkIfSafeToDelete(storages);
42        if (!shouldDelete) {
43            log.info('Aborting the destruction process.');
44            return;
45        }
46    }
47
48    log.info('Destruction initiated.');
49    const deletePromises = storages.map(async (storage) => {
50        await storagesClient.deleteOne(storage.id);
51        log.info(`${storage.name} successfully deleted.`);
52    });
53    await Promise.all(deletePromises);
54    log.info('Destruction successful.');
55});