Website Image Downloader
Pricing
from $2.00 / 1,000 results
Uses the Website Image Crawler to discover images on a site, then downloads every image and provides them as downloadable files alongside a metadata list.
Pricing
from $2.00 / 1,000 results
Rating
0.0
(0)
Developer
F. Gutz
Maintained by CommunityActor stats
0
Bookmarked
2
Total users
1
Monthly active users
2 days ago
Last modified
Categories
Share
node_modules.gitstorageapify_storageroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lfquote_type = singlenode_modulesstorage.venv.DS_Store.idea.vscode.zed{ "printWidth": 120, "tabWidth": 4, "singleQuote": true}FROM apify/actor-node:22
RUN npm ls crawlee apify puppeteer playwright
COPY package*.json ./
RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version \ && rm -r ~/.npm
COPY . ./
RUN adduser -h /home/apify -D apify && \ chown -R apify:apify ./USER apify
CMD npm start --silent1import prettier from 'eslint-config-prettier';2
3import apify from '@apify/eslint-config/js.js';4
5// eslint-disable-next-line import/no-default-export6export default [{ ignores: ['**/dist'] }, ...apify, prettier];{ "name": "website-image-downloader", "version": "0.0.1", "type": "module", "description": "Uses the Website Image Crawler to find images, then downloads them all as downloadable output.", "engines": { "node": ">=18.0.0" }, "dependencies": { "apify": "^3.7.2", "archiver": "^8.0.0", "got": "^14.6.6" }, "devDependencies": { "@apify/eslint-config": "^1.2.0", "eslint": "^9.39.4", "eslint-config-prettier": "^10.1.8", "prettier": "^3.8.3" }, "scripts": { "start": "node ./src/main.js", "format": "prettier --write .", "format:check": "prettier --check .", "lint": "eslint", "lint:fix": "eslint --fix" }, "author": "", "license": "ISC"}{ "actorSpecification": 1, "name": "website-image-downloader", "title": "Website Image Downloader", "description": "Uses the Website Image Crawler to discover images on a site, then downloads every image and provides them as downloadable files alongside a metadata list.", "version": "1.0", "buildTag": "latest", "input": "./input_schema.json", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json", "keyValueStore": "./key_value_store_schema.json" }, "meta": { "templateId": "js-empty" }, "dockerfile": "../Dockerfile", "defaultRunOptions": { "build": "latest", "timeoutSecs": 7200, "memoryMbytes": 512 }}{ "actorSpecification": 1, "views": { "overview": { "title": "Overview", "transformation": { "fields": ["imageUrl", "sourcePage", "contentType", "sizeBytes", "key"] }, "display": { "component": "table", "properties": { "imageUrl": { "label": "Image URL", "format": "link" }, "sourcePage": { "label": "Source Page", "format": "link" }, "contentType": { "label": "Content Type" }, "sizeBytes": { "label": "Size" }, "key": { "label": "File Key" } } } } }}{ "title": "Image Downloader Input", "description": "Crawl a website for image URLs, then download every image. Uses the Website Image Crawler under the hood.", "type": "object", "schemaVersion": 1, "properties": { "startUrl": { "title": "Start URL", "type": "string", "description": "The URL of the website to start crawling from", "prefill": "https://example.com", "editor": "textfield" }, "maxCrawlDepth": { "title": "Maximum Crawl Depth", "type": "integer", "description": "How deep to follow links (0 = start URL only)", "default": 1, "minimum": 0, "maximum": 5, "editor": "number" }, "maxConcurrency": { "title": "Maximum Concurrency", "type": "integer", "description": "Number of pages/images processed in parallel", "default": 20, "minimum": 1, "maximum": 50, "editor": "number" }, "imageExtensions": { "title": "Image Extensions", "type": "array", "description": "File extensions to search for and download", "default": ["jpg", "jpeg", "png", "gif", "webp", "bmp", "svg"], "editor": "stringList" }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Whether to respect robots.txt rules", "default": true, "sectionCaption": "Advanced", "sectionDescription": "Crawler and download settings" }, "userAgent": { "title": "User Agent", "type": "string", "description": "Custom User-Agent for HTTP requests", "default": "Mozilla/5.0 (compatible; ApifyBot/1.0; +https://apify.com/bot)", "editor": "textfield" }, "useScope": { "title": "Restrict to Specific Domains", "type": "boolean", "description": "Limit crawling to listed domains only", "default": false }, "scope": { "title": "Allowed Domains", "type": "array", "description": "Domains to restrict crawling to", "editor": "stringList", "default": [] }, "includeSubdomains": { "title": "Include Subdomains", "type": "boolean", "description": "Also crawl subdomains of allowed domains", "default": false }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Apify Proxy settings", "prefill": { "useApifyProxy": true }, "editor": "proxy" } }, "required": ["startUrl"]}{ "actorKeyValueStoreSchemaVersion": 1, "title": "Key-Value Store Schema", "collections": { "images": { "title": "Downloaded Images", "description": "All image files downloaded by the crawler. Keys are 'image-00001.jpg', 'image-00002.png', etc.", "keyPrefix": "image-", "contentTypes": ["image/jpeg", "image/png", "image/gif", "image/webp", "image/bmp", "image/svg+xml"] }, "archive": { "title": "ZIP Archive", "description": "All images bundled into a single ZIP file for easy download.", "key": "images.zip", "contentTypes": ["application/zip"] } }}{ "actorOutputSchemaVersion": 1, "title": "Output schema of the Image Downloader", "description": "Downloaded image files are stored in the key-value store. A ZIP bundle of all images is also available. Metadata (URLs, source pages, file keys) is in the default dataset.", "type": "object", "properties": { "results": { "type": "string", "title": "Image Metadata", "description": "Dataset listing every downloaded image with its URL, source page, file key, content type, and size.", "template": "{{links.apiDefaultDatasetUrl}}/items" }, "zipFile": { "type": "string", "title": "ZIP Archive", "description": "All downloaded images bundled into a single images.zip file. Download and extract to get every image at once.", "template": "{{links.apiDefaultKeyValueStoreUrl}}/records/images.zip" }, "downloads": { "type": "string", "title": "Individual Files", "description": "All downloaded image files, accessible individually by key.", "template": "{{links.apiDefaultKeyValueStoreUrl}}/keys?collection=images" } }}1import path from 'node:path';2
3import { Actor, log } from 'apify';4import * as archiver from 'archiver';5import got from 'got';6
7await Actor.init();8
9const input = await Actor.getInput();10const proxyConfiguration = await Actor.createProxyConfiguration(input.proxyConfiguration);11
12const {13 startUrl,14 maxCrawlDepth = 1,15 maxConcurrency = 20,16 imageExtensions = ['jpg', 'jpeg', 'png', 'gif', 'webp', 'bmp', 'svg'],17 respectRobotsTxt = true,18 userAgent = 'Mozilla/5.0 (compatible; ApifyBot/1.0; +https://apify.com/bot)',19 useScope = false,20 scope = [],21 includeSubdomains = false,22} = input;23
24if (!startUrl) throw new Error('startUrl is required!');25
26// ── Step 1: Call the Website Image Crawler ──────────────────27
28const crawlerInput = {29 startUrl,30 maxCrawlDepth,31 maxConcurrency: Math.min(maxConcurrency, 100),32 imageExtensions,33 respectRobotsTxt,34 userAgent,35 useScope,36 scope,37 includeSubdomains,38 proxyConfiguration: input.proxyConfiguration,39};40
41log.info('Calling Website Image Crawler to discover image URLs...');42log.info(JSON.stringify(crawlerInput));43
44const crawlerRun = await Actor.call('GomorrhaDev/website-image-scraper', crawlerInput, {45 build: 'latest',46 waitSecs: 1200,47});48
49log.info(`Crawler finished. Status: ${crawlerRun.status}`);50
51if (crawlerRun.status !== 'SUCCEEDED') {52 throw new Error(`Crawler run failed with status: ${crawlerRun.status}`);53}54
55// ── Step 2: Fetch discovered image URLs ─────────────────────56
57const crawlerDataset = await Actor.apifyClient.dataset(crawlerRun.defaultDatasetId);58const { items } = await crawlerDataset.listItems();59
60log.info(`Crawler found ${items.length} image URLs. Downloading...`);61
62if (items.length === 0) {63 log.info('No images found. Nothing to download.');64 await Actor.exit();65}66
67// ── Step 3: Download images concurrently ────────────────────68
69const httpAgent = proxyConfiguration?.newProxyAgent?.();70const httpsAgent = proxyConfiguration?.newProxyAgent?.();71const agent = { http: httpAgent, https: httpsAgent };72
73const contentTypeMap = {74 jpg: 'image/jpeg',75 jpeg: 'image/jpeg',76 png: 'image/png',77 gif: 'image/gif',78 webp: 'image/webp',79 bmp: 'image/bmp',80 svg: 'image/svg+xml',81};82
83let downloadIndex = 0;84let downloaded = 0;85let failed = 0;86const downloadedFiles = [];87
88async function downloadOne(item) {89 const imageUrl = item.url;90 try {91 const response = await got(imageUrl, {92 timeout: { request: 30000 },93 headers: { 'User-Agent': userAgent },94 agent,95 responseType: 'buffer',96 maxRedirects: 5,97 });98
99 const ext = path.extname(imageUrl).split('?')[0].substring(1).toLowerCase() || 'jpg';100 const contentType = response.headers['content-type'] || contentTypeMap[ext] || 'application/octet-stream';101 const buffer = Buffer.from(response.rawBody);102
103 const idx = String(++downloadIndex).padStart(5, '0');104 const key = `image-${idx}.${ext}`;105
106 await Actor.setValue(key, buffer, { contentType });107
108 await Actor.pushData({109 imageUrl,110 sourcePage: item.sourcePage || '',111 foundAt: item.foundAt || '',112 key,113 contentType,114 sizeBytes: buffer.length,115 });116
117 downloaded++;118 downloadedFiles.push({ key, buffer });119 log.info(`[${downloaded}/${items.length}] ${key} (${(buffer.length / 1024).toFixed(1)} KB) <- ${imageUrl}`);120 } catch (err) {121 failed++;122 log.warning(`Failed to download: ${imageUrl} — ${err.message}`);123 }124}125
126// Concurrency-limited download loop127const concurrency = Math.min(maxConcurrency, 50);128const queue = [...items];129const workers = Array.from({ length: concurrency }, async () => {130 while (queue.length > 0) {131 const item = queue.shift();132 await downloadOne(item);133 }134});135
136await Promise.all(workers);137log.info(`Downloads complete. ${downloaded} downloaded, ${failed} failed, ${items.length} total.`);138
139// ── Step 4: Bundle into ZIP ─────────────────────────────────140
141if (downloadedFiles.length > 0) {142 log.info(`Creating ZIP archive from ${downloadedFiles.length} files...`);143
144 const zipBuffers = [];145 await new Promise((resolve, reject) => {146 const archive = new archiver.ZipArchive({ zlib: { level: 9 } });147 archive.on('data', (chunk) => zipBuffers.push(chunk));148 archive.on('end', resolve);149 archive.on('error', reject);150
151 for (const { key, buffer } of downloadedFiles) {152 archive.append(buffer, { name: key });153 }154 archive.finalize();155 });156
157 const zipBuffer = Buffer.concat(zipBuffers);158 await Actor.setValue('images.zip', zipBuffer, { contentType: 'application/zip' });159
160 log.info(`ZIP archive saved: images.zip (${(zipBuffer.length / 1024 / 1024).toFixed(1)} MB)`);161}162
163await Actor.exit();