1import { Actor, log } from 'apify';
2import {
3 fetchWithRetry, maybeGunzip, parseSitemap, parseRobotsSitemaps, looksLikeSitemapUrl,
4 normalizeInputUrl, registrableHost, COMMON_SITEMAP_PATHS,
5} from './sitemap.js';
6
7const EVENT_NAME = 'url-extracted';
8const PUSH_BATCH = 500;
9
10await Actor.init();
11
12const input = (await Actor.getInput()) ?? {};
13const {
14 startUrls = [{ url: 'https://crawlee.dev' }],
15 maxUrls = 1000,
16 includeRegex = '',
17 excludeRegex = '',
18 sameDomainOnly = true,
19 maxSitemaps = 500,
20 maxConcurrency = 10,
21 requestTimeoutSecs = 30,
22 discoverCommonPaths = true,
23} = input;
24
25const compile = (src, name) => {
26 if (!src) return null;
27 try { return new RegExp(src, 'i'); } catch (e) { throw new Error(`Invalid ${name}: ${e.message}`); }
28};
29
30let include; let exclude;
31try {
32 include = compile(includeRegex, 'includeRegex');
33 exclude = compile(excludeRegex, 'excludeRegex');
34} catch (e) {
35 await Actor.fail(e.message);
36}
37
38const inputs = [...new Set((startUrls || []).map((s) => normalizeInputUrl(typeof s === 'string' ? s : s?.url)).filter(Boolean))];
39if (!inputs.length) await Actor.fail('No valid URLs in "startUrls". Provide website URLs (https://example.com) or sitemap URLs.');
40
41const limit = maxUrls && maxUrls > 0 ? maxUrls : Infinity;
42const timeoutMs = Math.max(5, requestTimeoutSecs) * 1000;
43const chargingManager = Actor.getChargingManager();
44
45const stats = { inputs: inputs.length, sitemapsFetched: 0, sitemapsFailed: 0, urlsFound: 0, urlsFiltered: 0, duplicates: 0, urlsSaved: 0, errors: [] };
46const seenSitemaps = new Set();
47const seenUrls = new Set();
48const perInput = Object.fromEntries(inputs.map((u) => [u, 0]));
49const queue = [];
50let buffer = [];
51let accepted = 0;
52let stop = false;
53let budgetStop = false;
54
55const enqueueSitemap = (url, inputUrl, depth, discoveredVia) => {
56 if (seenSitemaps.has(url) || seenSitemaps.size >= maxSitemaps) return;
57 seenSitemaps.add(url);
58 queue.push({ url, inputUrl, depth, discoveredVia });
59};
60
61const flush = async () => {
62 if (!buffer.length) return;
63 let items = buffer; buffer = [];
64
65 const allowed = chargingManager.calculateMaxEventChargeCountWithinLimit(EVENT_NAME);
66 const budgetHit = items.length >= allowed;
67 if (items.length > allowed) items = items.slice(0, Math.max(0, allowed));
68 if (items.length) {
69 const res = await Actor.pushData(items, EVENT_NAME);
70 stats.urlsSaved += items.length;
71 if (res?.eventChargeLimitReached) budgetStop = true;
72 }
73 if (budgetHit || budgetStop) {
74 budgetStop = true;
75 log.warning('Maximum cost per run reached; stopping. Increase the run\'s max charge to get more URLs.');
76 stop = true;
77 }
78};
79
80
81async function discover(inputUrl) {
82 if (looksLikeSitemapUrl(inputUrl)) {
83 enqueueSitemap(inputUrl, inputUrl, 0, 'input');
84 return;
85 }
86 const origin = new URL(inputUrl).origin;
87 let found = [];
88 try {
89 const { buffer: buf } = await fetchWithRetry(`${origin}/robots.txt`, { timeoutMs, maxRetries: 2, log });
90 found = parseRobotsSitemaps(buf.toString('utf8'), origin);
91 log.info(`${origin}: robots.txt lists ${found.length} sitemap(s)`);
92 } catch (e) {
93 log.info(`${origin}: robots.txt not available (${e.message})`);
94 }
95 for (const sm of found) enqueueSitemap(sm, inputUrl, 0, 'robots.txt');
96 if (!found.length && discoverCommonPaths) {
97
98 for (const p of COMMON_SITEMAP_PATHS) enqueueSitemap(`${origin}${p}`, inputUrl, 0, 'common-path-probe');
99 }
100}
101
102
103async function processSitemap(job) {
104 const { url, inputUrl, depth, discoveredVia } = job;
105 let res;
106 try {
107 res = await fetchWithRetry(url, { timeoutMs, maxRetries: discoveredVia === 'common-path-probe' ? 1 : 3, log });
108 } catch (e) {
109 if (discoveredVia !== 'common-path-probe') {
110 stats.sitemapsFailed++;
111 stats.errors.push({ sitemap: url, error: e.message });
112 log.warning(`Failed to fetch ${url}: ${e.message}`);
113 }
114 return;
115 }
116 let parsed;
117 try {
118 parsed = parseSitemap(maybeGunzip(res.buffer));
119 } catch (e) {
120 if (discoveredVia !== 'common-path-probe') {
121 stats.sitemapsFailed++;
122 stats.errors.push({ sitemap: url, error: e.message });
123 log.warning(`Failed to parse ${url}: ${e.message}`);
124 }
125 return;
126 }
127 if (parsed.type === 'unknown') {
128 if (discoveredVia !== 'common-path-probe') {
129 stats.errors.push({ sitemap: url, error: 'Not a recognized sitemap format' });
130 log.warning(`${url} is not a recognized sitemap (urlset / sitemapindex / text / RSS / Atom).`);
131 }
132 return;
133 }
134
135 stats.sitemapsFetched++;
136 if (parsed.type === 'index') {
137 log.info(`Sitemap index ${url}: ${parsed.sitemaps.length} child sitemap(s)`);
138 for (const child of parsed.sitemaps) {
139 try { enqueueSitemap(new URL(child.loc, url).href, inputUrl, depth + 1, url); } catch { }
140 }
141 return;
142 }
143 const inputHost = registrableHost(inputUrl);
144 let kept = 0;
145 for (const u of parsed.urls) {
146 if (stop || accepted >= limit) { stop = true; break; }
147 stats.urlsFound++;
148 let loc;
149 try { loc = new URL(u.loc, url).href; } catch { stats.urlsFiltered++; continue; }
150 if (sameDomainOnly && registrableHost(loc) !== inputHost) { stats.urlsFiltered++; continue; }
151 if (include && !include.test(loc)) { stats.urlsFiltered++; continue; }
152 if (exclude && exclude.test(loc)) { stats.urlsFiltered++; continue; }
153 if (seenUrls.has(loc)) { stats.duplicates++; continue; }
154 seenUrls.add(loc);
155 const priority = u.priority != null && u.priority !== '' && !Number.isNaN(Number(u.priority)) ? Number(u.priority) : null;
156 buffer.push({
157 url: loc,
158 lastmod: u.lastmod ?? null,
159 changefreq: u.changefreq ?? null,
160 priority,
161 sourceSitemap: url,
162 sitemapType: parsed.type,
163 sitemapDepth: depth,
164 inputUrl,
165 imageCount: u.imageCount ?? 0,
166 hasVideo: u.hasVideo ?? false,
167 hasNews: u.hasNews ?? false,
168 alternateLanguageLinks: u.alternates ?? 0,
169 });
170 kept++;
171 accepted++;
172 perInput[inputUrl] = (perInput[inputUrl] || 0) + 1;
173 if (buffer.length >= PUSH_BATCH) await flush();
174 }
175 log.info(`Sitemap ${url}: ${parsed.urls.length} URL(s), ${kept} new after filters (total ${accepted})`);
176}
177
178const startedAt = Date.now();
179await Promise.all(inputs.map(discover));
180
181
182let active = 0;
183await new Promise((resolve) => {
184 const pump = () => {
185 if (stop) queue.length = 0;
186 while (active < maxConcurrency && queue.length) {
187 const job = queue.shift();
188 active++;
189 processSitemap(job).catch((e) => {
190 stats.errors.push({ sitemap: job.url, error: e.message });
191 log.exception(e, `Unexpected error on ${job.url}`);
192 }).finally(() => { active--; pump(); });
193 }
194 if (!active && !queue.length) resolve();
195 };
196 pump();
197});
198await flush();
199
200if (seenSitemaps.size >= maxSitemaps) log.warning(`Reached maxSitemaps (${maxSitemaps}); some sitemaps were not processed.`);
201const inputsWithoutUrls = Object.entries(perInput).filter(([, n]) => n === 0).map(([u]) => u);
202for (const u of inputsWithoutUrls) log.warning(`${u}: no sitemap URLs found (no sitemap published, unreachable, or everything filtered out).`);
203const summary = {
204 ...stats,
205 urlsPerInput: perInput,
206 inputsWithoutUrls,
207 sitemapsDiscovered: seenSitemaps.size,
208 durationSecs: Math.round((Date.now() - startedAt) / 100) / 10,
209 chargedEvents: chargingManager.getChargedEventCount(EVENT_NAME),
210 errors: stats.errors.slice(0, 200),
211};
212await Actor.setValue('SUMMARY', summary);
213log.info(`Done: ${summary.urlsSaved} URL(s) from ${summary.sitemapsFetched} sitemap(s) in ${summary.durationSecs}s`);
214
215if (summary.urlsSaved === 0) {
216 const msg = 'No URLs found. The site may not publish a sitemap, or all URLs were filtered out. See the SUMMARY record in the key-value store.';
217 await Actor.setStatusMessage(msg, { isStatusMessageTerminal: true });
218} else {
219 await Actor.setStatusMessage(`Extracted ${summary.urlsSaved} URL(s) from ${summary.sitemapsFetched} sitemap(s).`, { isStatusMessageTerminal: true });
220}
221await Actor.exit();
