No credit card required

Delete Named Storages

mnmkng/delete-named-storages

No credit card required

Deletes your named storages by matching their names with a RegExp, selecting a date, or more. Enables deleting multiple named storages fast and safe using a UI rather than API.

src/functions.js

1const crypto = require('crypto');
2const Apify = require('apify');
3const moment = require('moment');
4
5const { utils: { log } } = Apify;
6
7const KEY_VALUE_STORE_NAME = 'DELETE-NAMED-STORAGES-SECURITY-STORE';
8
9exports.filterStoragesByPattern = function (storages, storageNamePattern) {
10    if (!storageNamePattern) return storages;
11    const pattern = new RegExp(storageNamePattern);
12    const storagesMatchingPattern = storages.filter((storage) => {
13        return pattern.test(storage.name);
14    });
15    log.debug(`Removed ${storages.length - storagesMatchingPattern.length} storages because their name did not match the provided pattern.`);
16    return storagesMatchingPattern;
17};
18
19exports.filterStoragesByDate = function (storages, date) {
20    if (!date) return storages;
21    const day = moment(date);
22    const storagesCreatedAtDate = storages.filter((storage) => {
23        return day.isSame(storage.createdAt, 'day');
24    });
25    log.debug(`Removed ${storages.length - storagesCreatedAtDate.length} storages because they were created at a different date.`);
26    return storagesCreatedAtDate;
27};
28
29exports.checkIfSafeToDelete = async function (storages) {
30    const kvs = await Apify.openKeyValueStore(KEY_VALUE_STORE_NAME);
31    const hash = hashStorages(storages);
32    const storagesToBeDeleted = await kvs.getValue(hash);
33    if (storagesToBeDeleted) {
34        log.info('Destruction credentials accepted. The selected storages will now be destroyed.');
35        return true;
36    }
37    await kvs.setValue(hash, storages);
38    log.warning('Are you sure you want to do this? Please confirm destruction by running this actor again with the same input.');
39    const link = kvs.getPublicUrl(hash);
40    log.info(`You can verify the storages to be deleted at: ${link}`);
41    return false;
42};
43
44function hashStorages(storages) {
45    const data = JSON.stringify(storages);
46    return crypto.createHash('md5')
47        .update(data)
48        .digest('hex');
49}

src/storages-client.js

1const { client } = require('apify');
2
3const STORAGE_OPERATIONS = {
4    DATASET: {
5        listStorages: client.datasets.listDatasets,
6        deleteStorage: datasetId => client.datasets.deleteDataset({ datasetId }),
7    },
8    KEY_VALUE_STORE: {
9        listStorages: client.keyValueStores.listStores,
10        deleteStorage: storeId => client.keyValueStores.deleteStore({ storeId }),
11    },
12    REQUEST_QUEUE: {
13        listStorages: client.requestQueues.listQueues,
14        deleteStorage: queueId => client.requestQueues.deleteQueue({ queueId }),
15    },
16};
17
18const PROTECTED_STORAGES = [
19    'apify-cli-deployments',
20];
21
22module.exports = class Storages {
23    constructor(type) {
24        this.type = type;
25    }
26
27    async list(desc) {
28        const { listStorages } = STORAGE_OPERATIONS[this.type];
29        let hasNextPage = true;
30        let storages = [];
31        while (hasNextPage) {
32            const { items, offset, limit, total } = await listStorages({ desc });
33            const safeStorages = items.filter(storage => !PROTECTED_STORAGES.includes(storage.name));
34            storages = storages.concat(safeStorages);
35            hasNextPage = offset + limit < total;
36        }
37        return storages;
38    }
39
40    async deleteOne(storageId) {
41        const { deleteStorage } = STORAGE_OPERATIONS[this.type];
42        return deleteStorage(storageId);
43    }
44};

.eslintrc

1{
2    "extends": "@apify"
3}

.gitignore

1apify_storage
2node_modules
3package-lock.json

Dockerfile

1FROM apify/actor-node-basic
2
3COPY package*.json ./
4
5RUN npm --quiet set progress=false && npm install --only=prod --no-optional
6RUN echo "Installed NPM packages:" && npm list || true
7RUN echo "Node.js version:" && node --version && echo "NPM version:" && npm --version
8
9COPY . ./

INPUT_SCHEMA.json

1{
2  "title": "Storage Destroyer",
3  "description": "",
4  "type": "object",
5  "schemaVersion": 1,
6  "properties": {
7    "storageType": {
8      "title": "Storage type",
9      "type": "string",
10      "description": "Type of the storages you want to delete.",
11      "editor": "select",
12      "enum": ["DATASET", "KEY_VALUE_STORE", "REQUEST_QUEUE"],
13      "enumTitles": ["Datasets", "Key-Value Stores", "Request Queues"]
14    },
15    "matchingExpression": {
16      "title": "Matching expression",
17      "type": "string",
18      "description": "A regular expression in a string format used to match the names of storages to be deleted. It will be used to create an instance of <code>RegExp</code> using <code>new RegExp(expression)</code> All storages matching the pattern will be deleted unless other filters are provided.",
19      "editor": "textfield",
20      "minLength": 1
21    },
22    "createdAt": {
23      "title": "Storage creation date",
24      "type": "string",
25      "description": "Use YYYY-MM-DD format. The option limits the deleted storages to the ones created on the provided date. If you want to delete all storages from a given date, use '.*' as your Matching expression.",
26      "editor": "textfield",
27      "sectionCaption": "Advanced settings"
28    },
29    "maxDeletedStorages": {
30      "title": "Max deleted storages",
31      "type": "integer",
32      "description": "Limit the number of deleted storages to the given amount.",
33      "editor": "number",
34      "unit": "storages"
35    },
36    "newestFirst": {
37      "title": "Newest first",
38      "type": "boolean",
39      "description": "Sort storages from newest to oldest. In combination with the Max storages to delete option, this enables to \"delete 5 newest storages\" and similar commands.",
40      "editor": "checkbox"
41    },
42    "forceDelete": {
43      "title": "Force delete",
44      "type": "boolean",
45      "description": "If you are REALLY AND ABSOLUTELY sure that your filters will delete exactly the storages you need then you can use this flag to delete them immediately without the need for confirmation by a second run of the actor. Tread carefully!",
46      "default": false,
47      "editor": "checkbox",
48      "groupCaption": "DANGER ZONE"
49    }
50  },
51  "required": [
52    "storageType",
53    "matchingExpression"
54  ]
55}

apify.json

1{
2	"name": "delete-named-storages",
3	"version": "0.1",
4	"buildTag": "latest",
5	"env": null
6}

main.js

1const Apify = require('apify');
2const StoragesClient = require('./src/storages-client');
3const { filterStoragesByPattern, filterStoragesByDate, checkIfSafeToDelete } = require('./src/functions');
4
5const { utils: { log } } = Apify;
6
7Apify.main(async () => {
8    log.info('Loading input.');
9    const {
10        storageType,
11        matchingExpression,
12        createdAt,
13        maxDeletedStorages,
14        newestFirst,
15        forceDelete,
16    } = await Apify.getInput();
17
18    const storagesClient = new StoragesClient(storageType);
19    log.info(`Fetching ${storageType}S in ${newestFirst ? 'descending' : 'ascending'} order.`);
20    let storages = await storagesClient.list(newestFirst);
21
22    log.info(`Fetched ${storages.length} storage(s). Filtering.`);
23    storages = filterStoragesByPattern(storages, matchingExpression);
24    storages = filterStoragesByDate(storages, createdAt);
25    log.info(`${storages.length} storages were kept after filtering.`);
26
27    if (maxDeletedStorages) {
28        log.info(`Applying limit: Max ${maxDeletedStorages} storages will be deleted.`);
29        storages = storages.slice(0, maxDeletedStorages);
30    }
31
32    if (!storages.length) {
33        log.info('No storages selected for destruction. Aborting the process.');
34        return;
35    }
36
37    if (forceDelete) {
38        log.info('You have chosen immediate destruction. We certainly hope you did not make a mistake.');
39    } else {
40        log.info('Preparing for destruction of storages.');
41        const shouldDelete = await checkIfSafeToDelete(storages);
42        if (!shouldDelete) {
43            log.info('Aborting the destruction process.');
44            return;
45        }
46    }
47
48    log.info('Destruction initiated.');
49    const deletePromises = storages.map(async (storage) => {
50        await storagesClient.deleteOne(storage.id);
51        log.info(`${storage.name} successfully deleted.`);
52    });
53    await Promise.all(deletePromises);
54    log.info('Destruction successful.');
55});

package.json

1{
2	"name": "delete-named-storages",
3	"version": "1.0.0",
4	"description": "Deletes named storages from Apify Platform using simple filters.",
5	"main": "main.js",
6	"dependencies": {
7		"apify": "^0.19.0",
8		"moment": "^2.24.0"
9	},
10	"scripts": {
11		"start": "node main.js"
12	},
13	"devDependencies": {
14		"@apify/eslint-config": "^0.0.3",
15		"eslint": "^6.8.0"
16	}
17}

Developer

Ondra Urban

Actor Metrics

1 monthly user
1 star
Created in Jan 2020
Modified 4 years ago

Categories

Developer tools

All Social Media Video Downloader

wilcode/all-social-media-video-downloader

All Social Media Video Downloader API allows easy extraction of video links from platforms like Facebook, Instagram, Twitter, TikTok, YouTube, etc.

Wilcode

Instagram Scraper - All in one

curious_coder/instagram-scraper

Scrape instragram profiles, followers, posts, comments and get valuable information like email addresses, websites, post engagement insights, commented users, etc

Curious Coder

2.5k

Website Content Crawler

apify/website-content-crawler

Crawl websites and extract text content to feed AI models, LLM applications, vector databases, or RAG pipelines. The Actor supports rich formatting using Markdown, cleans the HTML, downloads files, and integrates well with 🦜🔗 LangChain, LlamaIndex, and the wider LLM ecosystem.

Apify

26.7k

658

Cheerio Scraper

apify/cheerio-scraper

Crawls websites using raw HTTP requests, parses the HTML with the Cheerio library, and extracts data from the pages using a Node.js code. Supports both recursive crawling and lists of URLs. This actor is a high-performance alternative to apify/web-scraper for websites that do not require JavaScript.

Apify

5.6k

Web Scraper

apify/web-scraper

Crawls arbitrary websites using the Chrome browser and extracts data from pages using JavaScript code. The Actor supports both recursive crawling and lists of URLs and automatically manages concurrency for maximum performance. This is Apify's basic tool for web crawling and scraping.

Apify

71.5k

224

Puppeteer Scraper

apify/puppeteer-scraper

Crawls websites with the headless Chrome and Puppeteer library using a provided server-side Node.js code. This crawler is an alternative to apify/web-scraper that gives you finer control over the process. Supports both recursive crawling and list of URLs. Supports login to website.

Apify

4.6k

Legacy PhantomJS Crawler

apify/legacy-phantomjs-crawler

Replacement for the legacy Apify Crawler product with a backward-compatible interface. The actor uses PhantomJS headless browser to recursively crawl websites and extract data from them using a piece of front-end JavaScript code.

Apify

1.6k

Extended GPT Scraper

drobnikj/extended-gpt-scraper

Extract data from any website and feed it into GPT via the OpenAI API. Use ChatGPT to proofread content, analyze sentiment, summarize reviews, extract contact details, and much more.

Jakub Drobník

1.2k

Playwright Scraper

apify/playwright-scraper

Crawls websites with the headless Chromium, Chrome, or Firefox browser and Playwright library using a provided server-side Node.js code. Supports both recursive crawling and a list of URLs. Supports login to a website.

Apify

849

Monitoring Reporter Dashboard

apify/monitoring-reporter-dashboard

The monitoring reporter dashboard is a part of the Apify Monitoring Suite (apify/monitoring). See its readme for more information and how to use this.

Apify