1import { Actor, log } from 'apify';
2import { gotScraping } from 'got-scraping';
3import { createHash } from 'node:crypto';
4import { gzipSync, gunzipSync } from 'node:zlib';
5import {
6 COMMON_SITEMAP_PATHS, parseSitemap, parseRobotsSitemaps, maybeGunzip, looksLikeSitemapUrl,
7 parseDateInput, compilePatterns, absolutize,
8} from './sitemap.js';
9
10const URL_EVENT = 'url';
11const STATUS_EVENT = 'url-status-check';
12const MAX_BODY_BYTES = 60 * 1024 * 1024;
13const BATCH = 500;
14const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
15
16await Actor.init();
17
18class InputError extends Error {}
19
20try {
21 const input = (await Actor.getInput()) ?? {};
22 const {
23 startUrls = [],
24 discoverFromRobotsTxt = true,
25 tryCommonSitemapPaths = true,
26 maxSitemapDepth = 5,
27 maxSitemaps = 1000,
28 maxUrls: maxUrlsIn = 1000,
29 maxUrlsPerSite = 0,
30 includeUrlPatterns = [],
31 excludeUrlPatterns = [],
32 lastmodFrom = null,
33 lastmodTo = null,
34 onlyNewUrls = false,
35 stateKey = '',
36 includeImages = false,
37 includeAlternates = false,
38 checkUrlStatus = false,
39 statusCheckConcurrency = 10,
40 requestTimeoutSecs = 30,
41 maxRetries = 3,
42 sitemapConcurrency = 5,
43 restrictToStartPath = true,
44 proxyConfiguration = { useApifyProxy: false },
45 emitStatusRows = true,
46 } = input;
47
48 const capList = [maxUrlsIn, input.maxItems, input.limit, input.maxResults, process.env.ACTOR_MAX_PAID_DATASET_ITEMS]
49 .map(Number).filter((n) => Number.isFinite(n) && n > 0).map(Math.floor);
50 const maxUrls = capList.length ? Math.min(...capList) : 0;
51 if (maxUrls > 0) log.info(`Output is capped at ${maxUrls} URLs (maxUrls). Set maxUrls to 0 for no limit.`);
52
53
54 const rawStarts = (Array.isArray(startUrls) ? startUrls : [startUrls])
55 .map((s) => (typeof s === 'string' ? s : s?.url)).filter(Boolean).map((s) => s.trim());
56 if (!rawStarts.length) throw new InputError('Input "startUrls" is empty. Add at least one website (https://example.com) or sitemap URL.');
57 const starts = [];
58 for (const s of rawStarts) {
59 const u = absolutize(/^https?:\/\//i.test(s) ? s : `https://${s}`);
60 if (!u) throw new InputError(`Invalid start URL: "${s}"`);
61 starts.push(u);
62 }
63 let includeRe, excludeRe;
64 try { includeRe = compilePatterns(includeUrlPatterns, 'includeUrlPatterns'); excludeRe = compilePatterns(excludeUrlPatterns, 'excludeUrlPatterns'); } catch (e) { throw new InputError(e.message); }
65 let fromDate; let toDate;
66 try { fromDate = parseDateInput(lastmodFrom); toDate = parseDateInput(lastmodTo); } catch (e) { throw new InputError(e.message); }
67
68 const proxy = proxyConfiguration?.useApifyProxy || proxyConfiguration?.proxyUrls?.length
69 ? await Actor.createProxyConfiguration(proxyConfiguration) : null;
70
71 const charging = Actor.getChargingManager();
72 let chargeLimitHit = false;
73
74
75 const stats = { requests: 0, retries: 0, failedRequests: 0 };
76 async function fetchWithRetry(url, { method = 'GET', optional = false, session = 'main' } = {}) {
77 let lastErr;
78 for (let attempt = 0; attempt <= maxRetries; attempt++) {
79 stats.requests++;
80 try {
81 const res = await gotScraping({
82 url, method,
83 proxyUrl: proxy ? await proxy.newUrl(`s${session}${attempt}`.replace(/[^\w]/g, '').slice(0, 50)) : undefined,
84 timeout: { request: requestTimeoutSecs * 1000 },
85 responseType: 'buffer',
86 throwHttpErrors: false,
87 followRedirect: true,
88 maxRedirects: 10,
89 retry: { limit: 0 },
90 decompress: true,
91 headerGeneratorOptions: { browsers: ['chrome'], operatingSystems: ['windows', 'macos'], devices: ['desktop'] },
92 });
93 const code = res.statusCode;
94 if (code === 429 || code >= 500 || code === 403) {
95 const ra = Number(res.headers['retry-after']);
96 lastErr = new Error(`HTTP ${code}`);
97 lastErr.statusCode = code;
98 if (attempt < maxRetries) {
99 stats.retries++;
100 await sleep(Number.isFinite(ra) && ra > 0 ? Math.min(ra, 30) * 1000 : Math.min(1000 * 2 ** attempt, 15000) + Math.random() * 500);
101 continue;
102 }
103 break;
104 }
105 if (res.rawBody?.length > MAX_BODY_BYTES) throw new Error(`Response too large (${res.rawBody.length} bytes)`);
106 return res;
107 } catch (e) {
108 lastErr = e;
109 if (attempt < maxRetries) { stats.retries++; await sleep(Math.min(1000 * 2 ** attempt, 15000) + Math.random() * 500); }
110 }
111 }
112 if (!optional) stats.failedRequests++;
113 throw lastErr;
114 }
115
116
117 const stateStore = onlyNewUrls ? await Actor.openKeyValueStore('sitemap-url-extractor-state') : null;
118 const key = (stateKey && String(stateKey).replace(/[^\w.-]/g, '-').slice(0, 200))
119 || `state-${createHash('sha1').update(starts.slice().sort().join('|')).digest('hex').slice(0, 16)}`;
120 let previouslySeen = null;
121 if (stateStore) {
122 try {
123 const meta = await stateStore.getValue(`${key}-meta`);
124 if (meta?.chunks) {
125 const parts = [];
126 for (let i = 0; i < meta.chunks; i++) parts.push(...JSON.parse(gunzipSync(Buffer.from(await stateStore.getValue(`${key}-${i}`))).toString('utf8')));
127 previouslySeen = new Set(parts);
128 }
129 } catch (e) { log.warning(`Could not read previous state "${key}", treating all URLs as new: ${e.message}`); }
130 log.info(previouslySeen ? `Loaded ${previouslySeen.size} previously seen URLs (state key "${key}").` : `No previous state for key "${key}" — first run: every URL counts as new and becomes the baseline.`);
131 }
132
133
134 const seen = new Set();
135 const eligible = new Set();
136 let pushed = 0;
137 let accepted = 0;
138 let buffer = [];
139 async function flush(batch) {
140 let items = batch || buffer;
141 if (!batch) buffer = [];
142 if (!items.length || chargeLimitHit) return;
143 if (charging) {
144 const allowed = charging.calculateMaxEventChargeCountWithinLimit(URL_EVENT);
145 if (Number.isFinite(allowed) && allowed < items.length) {
146 items = items.slice(0, Math.max(0, allowed));
147 chargeLimitHit = true;
148 log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.');
149 }
150 }
151 if (!items.length) return;
152 const r = await Actor.pushData(items, URL_EVENT);
153 pushed += items.length;
154 if (r?.eventChargeLimitReached) chargeLimitHit = true;
155 }
156
157 const limitReached = () => chargeLimitHit || (maxUrls > 0 && accepted >= maxUrls);
158
159
160
161 async function statusRequest(url, attempt) {
162 const proxyUrl = proxy ? await proxy.newUrl(`st${attempt}${Math.floor(Math.random() * 1e6)}`) : undefined;
163 return new Promise((resolve, reject) => {
164 const st = gotScraping.stream({
165 url, proxyUrl, throwHttpErrors: false, followRedirect: true, maxRedirects: 10, retry: { limit: 0 },
166 timeout: { request: requestTimeoutSecs * 1000 },
167 headerGeneratorOptions: { browsers: ['chrome'], operatingSystems: ['windows', 'macos'], devices: ['desktop'] },
168 });
169 st.once('response', (res) => { resolve({ statusCode: res.statusCode, url: res.url }); st.destroy(); });
170 st.once('error', reject);
171 });
172 }
173 async function checkStatuses(items) {
174 let idx = 0;
175 const worker = async () => {
176 while (idx < items.length) {
177 const it = items[idx++];
178 const tries = Math.min(maxRetries, 2);
179 for (let attempt = 0; attempt <= tries; attempt++) {
180 stats.statusRequests = (stats.statusRequests || 0) + 1;
181 try {
182 const res = await statusRequest(it.url, attempt);
183 if ((res.statusCode === 429 || res.statusCode >= 500) && attempt < tries) { await sleep(1000 * 2 ** attempt); continue; }
184 it.httpStatus = res.statusCode;
185 it.finalUrl = res.url && res.url !== it.url ? res.url : null;
186 it.statusError = null;
187 break;
188 } catch (e) {
189 if (attempt < tries) { await sleep(1000 * 2 ** attempt); continue; }
190 it.httpStatus = null; it.finalUrl = null; it.statusError = String(e.code || e.message).slice(0, 200);
191 }
192 }
193 }
194 };
195 await Promise.all(Array.from({ length: Math.max(1, Math.min(50, statusCheckConcurrency)) }, worker));
196 if (charging) {
197 try { await Actor.charge({ eventName: STATUS_EVENT, count: items.length }); } catch (e) { log.debug(`charge: ${e.message}`); }
198 }
199 }
200
201 async function emit(item) {
202 if (limitReached()) return;
203 accepted++;
204 buffer.push(item);
205 if (buffer.length >= BATCH) {
206 const batch = buffer; buffer = [];
207 if (checkUrlStatus) await checkStatuses(batch);
208 await flush(batch);
209 }
210 }
211
212
213 const summary = { startedAt: new Date().toISOString(), sites: [], totals: {} };
214 let sitemapsProcessed = 0;
215 const processedSitemaps = new Set();
216
217 for (const start of starts) {
218 if (limitReached()) break;
219 const origin = new URL(start).origin;
220 const site = { input: start, origin, discoveredVia: null, sitemapsFound: [], sitemapsFailed: [], urlsOutput: 0, urlsFiltered: 0, duplicates: 0 };
221 summary.sites.push(site);
222 const before = pushed + buffer.length;
223
224 const startPath = new URL(start).pathname;
225 const scopePrefix = restrictToStartPath && startPath !== '/' && !looksLikeSitemapUrl(start) ? start.replace(/[?#].*$/, '') : null;
226 site.scope = scopePrefix;
227 let queue = [];
228 if (looksLikeSitemapUrl(start) && startPath !== '/') {
229 queue.push({ url: start, depth: 0, parent: null });
230 site.discoveredVia = 'input';
231 } else {
232
233 if (scopePrefix && tryCommonSitemapPaths) {
234 const base = scopePrefix.endsWith('/') ? scopePrefix : `${scopePrefix}/`;
235 for (const p of ['sitemap.xml', 'sitemap_index.xml', 'wp-sitemap.xml']) {
236 try {
237 const r = await fetchWithRetry(base + p, { optional: true });
238 if (r.statusCode < 400 && ['sitemapindex', 'urlset'].includes(parseSitemap(maybeGunzip(r.rawBody), r.url).type)) {
239 queue.push({ url: base + p, depth: 0, parent: 'sub-path' }); site.discoveredVia = 'sub-path'; break;
240 }
241 } catch { }
242 }
243 }
244 if (!queue.length && discoverFromRobotsTxt) {
245 try {
246 const r = await fetchWithRetry(`${origin}/robots.txt`, { optional: true });
247 if (r.statusCode < 400) {
248 const sm = parseRobotsSitemaps(r.body.toString('utf8'), origin);
249 if (sm.length) { queue.push(...sm.map((u) => ({ url: u, depth: 0, parent: 'robots.txt' }))); site.discoveredVia = 'robots.txt'; }
250 }
251 } catch (e) { log.info(`${origin}: robots.txt not reachable (${e.message}).`); }
252 }
253 if (!queue.length && tryCommonSitemapPaths) {
254 queue.push(...COMMON_SITEMAP_PATHS.map((p) => ({ url: origin + p, depth: 0, parent: 'common-path', probe: true })));
255 site.discoveredVia = 'common-paths';
256 }
257 }
258
259 let siteCount = 0;
260 let active = 0;
261 const siteDone = () => limitReached() || sitemapsProcessed >= maxSitemaps || (maxUrlsPerSite > 0 && siteCount >= maxUrlsPerSite);
262 const worker = async () => {
263 while (!siteDone()) {
264 if (!queue.length) { if (active === 0) break; await sleep(50); continue; }
265 const job = queue.shift();
266 active++;
267 try {
268 if (processedSitemaps.has(job.url)) continue;
269 processedSitemaps.add(job.url);
270 sitemapsProcessed++;
271 let res;
272 try {
273 res = await fetchWithRetry(job.url, { optional: !!job.probe });
274 } catch (e) {
275 if (!job.probe) { site.sitemapsFailed.push({ url: job.url, error: String(e.message).slice(0, 200) }); log.warning(`Failed to fetch sitemap ${job.url}: ${e.message}`); }
276 continue;
277 }
278 if (res.statusCode >= 400) {
279 if (!job.probe) site.sitemapsFailed.push({ url: job.url, error: `HTTP ${res.statusCode}` });
280 continue;
281 }
282 let parsed;
283 try {
284 parsed = parseSitemap(maybeGunzip(res.rawBody), res.url || job.url, { includeImages, includeAlternates });
285 } catch (e) {
286 site.sitemapsFailed.push({ url: job.url, error: `Parse error: ${e.message}` });
287 continue;
288 }
289 if (parsed.type === 'html' || parsed.type === 'unknown') {
290 if (!job.probe) site.sitemapsFailed.push({ url: job.url, error: `Not a sitemap (looks like ${parsed.type})` });
291 continue;
292 }
293 site.sitemapsFound.push({ url: job.url, type: parsed.type, entries: parsed.type === 'sitemapindex' ? parsed.sitemaps.length : parsed.urls.length });
294 log.info(`${job.url}: ${parsed.type}, ${parsed.type === 'sitemapindex' ? `${parsed.sitemaps.length} child sitemaps` : `${parsed.urls.length} URLs`}`);
295
296 if (parsed.type === 'sitemapindex') {
297 if (job.depth >= maxSitemapDepth) { log.warning(`Max sitemap depth ${maxSitemapDepth} reached at ${job.url}`); continue; }
298
299 for (const c of parsed.sitemaps) {
300 if (fromDate && c.lastmod && new Date(c.lastmod) < fromDate) { site.sitemapsSkippedByLastmod = (site.sitemapsSkippedByLastmod || 0) + 1; continue; }
301 queue.push({ url: c.url, depth: job.depth + 1, parent: job.url });
302 }
303 continue;
304 }
305 for (const u of parsed.urls) {
306 if (limitReached() || (maxUrlsPerSite > 0 && siteCount >= maxUrlsPerSite)) break;
307 if (seen.has(u.url)) { site.duplicates++; continue; }
308 seen.add(u.url);
309 if (scopePrefix && !u.url.startsWith(scopePrefix)) { site.urlsFiltered++; continue; }
310 if (includeRe.length && !includeRe.some((r) => r.test(u.url))) { site.urlsFiltered++; continue; }
311 if (excludeRe.some((r) => r.test(u.url))) { site.urlsFiltered++; continue; }
312 if ((fromDate || toDate)) {
313 const lm = u.lastmod ? new Date(u.lastmod) : null;
314 if (!lm || (fromDate && lm < fromDate) || (toDate && lm > toDate)) { site.urlsFiltered++; continue; }
315 }
316 eligible.add(u.url);
317 const isNew = previouslySeen ? !previouslySeen.has(u.url) : (stateStore ? true : null);
318 if (onlyNewUrls && previouslySeen && !isNew) { site.urlsFiltered++; continue; }
319 siteCount++;
320 await emit({
321 url: u.url,
322 lastmod: u.lastmod,
323 changefreq: u.changefreq,
324 priority: u.priority,
325 imageCount: u.imageCount,
326 ...(includeImages ? { images: u.images || [] } : {}),
327 ...(includeAlternates ? { alternates: u.alternates || [] } : {}),
328 ...(u.newsPublicationDate ? { newsPublicationDate: u.newsPublicationDate } : {}),
329 isNew,
330 ...(checkUrlStatus ? { httpStatus: null, finalUrl: null, statusError: null } : {}),
331 sitemapUrl: job.url,
332 site: origin,
333 extractedAt: new Date().toISOString(),
334 });
335 }
336 } finally { active--; }
337 }
338 };
339 await Promise.all(Array.from({ length: Math.max(1, Math.min(20, sitemapConcurrency)) }, worker));
340 { const batch = buffer; buffer = []; if (checkUrlStatus && batch.length) await checkStatuses(batch); await flush(batch); }
341 site.urlsOutput = pushed - before;
342 if (!site.sitemapsFound.length) log.warning(`${origin}: no sitemap found${site.sitemapsFailed.length ? ` (${site.sitemapsFailed.length} failed)` : ''}. Tip: pass the sitemap URL directly.`);
343 if (!site.urlsOutput && !chargeLimitHit && !(maxUrls > 0 && accepted >= maxUrls)) {
344 if (!site.sitemapsFound.length) site.reason = `No sitemap found for ${origin}${site.discoveredVia === 'common-paths' || !site.discoveredVia ? ' in robots.txt or at /sitemap.xml, /sitemap_index.xml, /wp-sitemap.xml and similar paths' : ''}${site.sitemapsFailed.length ? `; failed: ${site.sitemapsFailed.slice(0, 3).map((f) => `${f.url} (${f.error})`).join(', ')}` : ''}. Pass the sitemap URL directly if you know it.`;
345 else if (onlyNewUrls && previouslySeen) site.reason = null;
346 else site.reason = `${site.sitemapsFound.length} sitemap(s) read, but all ${site.urlsFiltered + site.duplicates} URLs were removed by your filters (scope, include/exclude patterns, lastmod dates) or were duplicates.`;
347 if (site.reason && emitStatusRows !== false) {
348 await Actor.pushData({ status: 'no_results', url: null, site: origin, sitemapUrl: null, input: start, reason: site.reason, extractedAt: new Date().toISOString() });
349 }
350 }
351 await Actor.setStatusMessage(`${summary.sites.length}/${starts.length} sites, ${pushed} URLs so far`);
352 }
353 { const batch = buffer; buffer = []; if (checkUrlStatus && batch.length) await checkStatuses(batch); await flush(batch); }
354
355
356 if (stateStore && !chargeLimitHit) {
357 const all = [...new Set([...(previouslySeen || []), ...eligible])];
358 const CHUNK = 100000; let chunks = 0;
359 for (let i = 0; i < all.length || i === 0; i += CHUNK) {
360 await stateStore.setValue(`${key}-${chunks++}`, gzipSync(Buffer.from(JSON.stringify(all.slice(i, i + CHUNK)))), { contentType: 'application/gzip' });
361 if (all.length === 0) break;
362 }
363 await stateStore.setValue(`${key}-meta`, { chunks, urlCount: all.length, updatedAt: new Date().toISOString(), startUrls: starts });
364 summary.stateKey = key; summary.stateUrlCount = all.length;
365 }
366
367 const sitesWithSitemaps = summary.sites.filter((s) => s.sitemapsFound.length).length;
368 summary.finishedAt = new Date().toISOString();
369 summary.totals = { sites: starts.length, sitesWithSitemaps, sitemapsProcessed, urlsOutput: pushed, ...stats, chargeLimitHit };
370 summary.maxUrls = maxUrls;
371 summary.stoppedReason = chargeLimitHit ? 'Reached the maximum cost you set for this run.' : (maxUrls > 0 && accepted >= maxUrls ? `Reached maxUrls (${maxUrls}). Set maxUrls higher, or 0 for no limit.` : null);
372 summary.zeroResultsReason = pushed ? null : (summary.sites.map((s) => s.reason).filter(Boolean).join(' | ') || (onlyNewUrls && previouslySeen ? 'No new URLs since the last run.' : null));
373 if (pushed) summary.nextSteps = [`Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${Actor.getEnv().defaultDatasetId}"}`, ...(checkUrlStatus ? [] : ['Find broken URLs: run again with checkUrlStatus: true (adds HTTP status and final URL per row).'])];
374 await Actor.setValue('OUTPUT', summary);
375 const msg = !pushed && summary.zeroResultsReason ? `0 results: ${summary.zeroResultsReason}` : `Done: ${pushed} URLs from ${sitesWithSitemaps}/${starts.length} sites (${summary.sites.reduce((a, s) => a + s.sitemapsFound.length, 0)} sitemaps).`
376 + (maxUrls > 0 && accepted >= maxUrls ? ` Stopped at maxUrls (${maxUrls}); set it higher or 0 for all URLs.` : '')
377 + (chargeLimitHit ? ' Stopped at your maximum cost per run.' : '')
378 + (sitesWithSitemaps < summary.sites.length ? ' Some sites had no discoverable sitemap — see OUTPUT in the key-value store.' : '');
379 log.info(msg);
380 await Actor.exit(msg);
381} catch (e) {
382 if (e instanceof InputError) {
383 log.error(e.message);
384 await Actor.fail(`Invalid input: ${e.message}`);
385 } else {
386 log.exception(e, 'Unexpected error');
387 await Actor.fail(`Unexpected error: ${e.message}`);
388 }
389}
