1import { Actor } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4
5const DEFAULT_TIMEOUT_SECONDS = 10;
6const USER_AGENT = 'RobotsSitemapAuditor/0.1 (+https://apify.com)';
7
8function isPrivateIPv4(ip) {
9 const parts = ip.split('.').map(Number);
10 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;
11 const [a, b] = parts;
12 return a === 10
13 || (a === 172 && b >= 16 && b <= 31)
14 || (a === 192 && b === 168)
15 || a === 127
16 || a === 0
17 || a === 169 && b === 254;
18}
19
20function isPrivateIPv6(ip) {
21 const normalized = ip.toLowerCase();
22 return normalized === '::1'
23 || normalized.startsWith('fc')
24 || normalized.startsWith('fd')
25 || normalized.startsWith('fe80:');
26}
27
28export async function normalizeAndValidateUrl(rawUrl) {
29 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');
30 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;
31 const url = new URL(withScheme);
32 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');
33 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');
34
35 const literalType = net.isIP(url.hostname);
36 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');
37 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');
38
39 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });
40 for (const record of records) {
41 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');
42 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');
43 }
44
45 url.pathname = '/';
46 url.search = '';
47 url.hash = '';
48 return url;
49}
50
51async function fetchText(url, timeoutSeconds) {
52 const controller = new AbortController();
53 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);
54 try {
55 const response = await fetch(url, {
56 redirect: 'follow',
57 signal: controller.signal,
58 headers: { 'user-agent': USER_AGENT, accept: 'text/plain, application/xml, text/xml, */*;q=0.1' },
59 });
60 const text = await response.text();
61 return {
62 ok: response.ok,
63 status: response.status,
64 finalUrl: response.url,
65 contentType: response.headers.get('content-type') || '',
66 body: text.slice(0, 1_000_000),
67 };
68 } catch (error) {
69 return { ok: false, status: null, finalUrl: url, contentType: '', body: '', error: error.message };
70 } finally {
71 clearTimeout(timeout);
72 }
73}
74
75export function parseRobots(body) {
76 const lines = body.split(/\r?\n/);
77 const sitemaps = [];
78 const disallow = [];
79 const allow = [];
80 const crawlDelay = [];
81 for (const rawLine of lines) {
82 const line = rawLine.replace(/#.*/, '').trim();
83 if (!line || !line.includes(':')) continue;
84 const [rawKey, ...rest] = line.split(':');
85 const key = rawKey.trim().toLowerCase();
86 const value = rest.join(':').trim();
87 if (!value) continue;
88 if (key === 'sitemap') sitemaps.push(value);
89 if (key === 'disallow') disallow.push(value);
90 if (key === 'allow') allow.push(value);
91 if (key === 'crawl-delay') crawlDelay.push(value);
92 }
93 return { sitemaps: [...new Set(sitemaps)], disallow, allow, crawlDelay };
94}
95
96function summarizeFindings({ baseUrl, robotsResult, robotsParsed, fallbackSitemap }) {
97 const findings = [];
98 if (!robotsResult.ok) findings.push({ severity: 'warning', code: 'ROBOTS_NOT_OK', message: `robots.txt returned ${robotsResult.status ?? robotsResult.error}` });
99 if (robotsResult.ok && robotsParsed.disallow.includes('/')) findings.push({ severity: 'critical', code: 'SITEWIDE_DISALLOW', message: 'robots.txt contains Disallow: /' });
100 if (robotsResult.ok && robotsParsed.sitemaps.length === 0 && !fallbackSitemap.ok) findings.push({ severity: 'warning', code: 'NO_SITEMAP_DISCOVERED', message: 'No Sitemap directive and /sitemap.xml was not reachable' });
101 if (robotsParsed.crawlDelay.length > 0) findings.push({ severity: 'info', code: 'CRAWL_DELAY_SET', message: `Crawl-delay values found: ${robotsParsed.crawlDelay.join(', ')}` });
102 if (robotsParsed.disallow.some((path) => ['/wp-admin/', '/admin', '/cart', '/checkout'].includes(path))) findings.push({ severity: 'info', code: 'COMMON_PRIVATE_PATHS_BLOCKED', message: 'Common private/CMS paths are blocked from crawlers' });
103 if (findings.length === 0) findings.push({ severity: 'ok', code: 'BASIC_DISCOVERY_OK', message: `${baseUrl.hostname} has basic robots/sitemap discovery signals` });
104 return findings;
105}
106
107export async function auditWebsite(input) {
108 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);
109 const baseUrl = await normalizeAndValidateUrl(input.startUrl);
110 const robotsUrl = new URL('/robots.txt', baseUrl).href;
111 const fallbackSitemapUrl = new URL('/sitemap.xml', baseUrl).href;
112
113 const robotsResult = await fetchText(robotsUrl, timeoutSeconds);
114 const robotsParsed = robotsResult.body ? parseRobots(robotsResult.body) : { sitemaps: [], disallow: [], allow: [], crawlDelay: [] };
115 const sitemapUrls = robotsParsed.sitemaps.length ? robotsParsed.sitemaps : [fallbackSitemapUrl];
116 const sitemapChecks = [];
117 for (const sitemapUrl of sitemapUrls.slice(0, 10)) {
118 const result = await fetchText(sitemapUrl, timeoutSeconds);
119 sitemapChecks.push({ url: sitemapUrl, ok: result.ok, status: result.status, contentType: result.contentType, finalUrl: result.finalUrl, bytesSampled: result.body.length, error: result.error || null });
120 }
121
122 const fallbackSitemap = sitemapChecks.find((item) => item.url === fallbackSitemapUrl) || { ok: false };
123 return {
124 inputUrl: input.startUrl,
125 normalizedUrl: baseUrl.href,
126 checkedAt: new Date().toISOString(),
127 robots: {
128 url: robotsUrl,
129 ok: robotsResult.ok,
130 status: robotsResult.status,
131 finalUrl: robotsResult.finalUrl,
132 contentType: robotsResult.contentType,
133 sitemapDirectives: robotsParsed.sitemaps,
134 disallowCount: robotsParsed.disallow.length,
135 allowCount: robotsParsed.allow.length,
136 crawlDelay: robotsParsed.crawlDelay,
137 error: robotsResult.error || null,
138 },
139 sitemaps: sitemapChecks,
140 findings: summarizeFindings({ baseUrl, robotsResult, robotsParsed, fallbackSitemap }),
141 };
142}
143
144if (process.env.NODE_ENV !== 'test') {
145 await Actor.init();
146 try {
147 const input = await Actor.getInput();
148 const result = await auditWebsite(input || {});
149 await Actor.pushData(result);
150 Actor.setValue('OUTPUT', result);
151 Actor.log.info('Audit complete', { url: result.normalizedUrl, findings: result.findings.length });
152 } finally {
153 await Actor.exit();
154 }
155}
