1import { Actor } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4import { fileURLToPath } from 'node:url';
5
6const USER_AGENT = 'AiCrawlerRobotsAuditor/0.1 (+https://apify.com)';
7const DEFAULT_TIMEOUT_SECONDS = 10;
8const MAX_BODY_BYTES = 256 * 1024;
9
10
11
12
13export const AI_CRAWLERS = [
14 { name: 'GPTBot', operator: 'OpenAI', userAgent: 'GPTBot', purpose: 'Powers ChatGPT training and retrieval.' },
15 { name: 'OAI-SearchBot', operator: 'OpenAI', userAgent: 'OAI-SearchBot', purpose: 'Powers ChatGPT Search retrieval.' },
16 { name: 'ChatGPT-User', operator: 'OpenAI', userAgent: 'ChatGPT-User', purpose: 'Fetches pages for ChatGPT user prompts.' },
17 { name: 'ClaudeBot', operator: 'Anthropic', userAgent: 'ClaudeBot', purpose: 'Powers Claude model training.' },
18 { name: 'Claude-SearchBot', operator: 'Anthropic', userAgent: 'Claude-SearchBot', purpose: 'Real-time web search for Claude.' },
19 { name: 'anthropic-ai', operator: 'Anthropic', userAgent: 'anthropic-ai', purpose: 'Anthropic AI agent fetches.' },
20 { name: 'Google-Extended', operator: 'Google', userAgent: 'Google-Extended', purpose: 'Controls Google Gemini training and AI Overviews.' },
21 { name: 'Googlebot-Extended', operator: 'Google', userAgent: 'Googlebot-Extended', purpose: 'Controls inclusion in Google AI Overviews.' },
22 { name: 'PerplexityBot', operator: 'Perplexity', userAgent: 'PerplexityBot', purpose: 'Powers Perplexity answer retrieval.' },
23 { name: 'Perplexity-User', operator: 'Perplexity', userAgent: 'Perplexity-User', purpose: 'Fetches pages for Perplexity user prompts.' },
24 { name: 'CCBot', operator: 'Common Crawl', userAgent: 'CCBot', purpose: 'Open web crawl used by many AI models.' },
25 { name: 'Bytespider', operator: 'ByteDance', userAgent: 'Bytespider', purpose: 'Powers ByteDance/TikTok AI training.' },
26 { name: 'Meta-ExternalAgent', operator: 'Meta', userAgent: 'Meta-ExternalAgent', purpose: 'Powers Meta AI training and retrieval.' },
27 { name: 'AppleBot', operator: 'Apple', userAgent: 'AppleBot', purpose: 'Powers Apple Intelligence and Siri.' },
28 { name: 'Amazonbot', operator: 'Amazon', userAgent: 'Amazonbot', purpose: 'Powers Alexa and Amazon AI services.' },
29 { name: 'YouBot', operator: 'You.com', userAgent: 'YouBot', purpose: 'Powers You.com AI search.' },
30 { name: 'Diffbot', operator: 'Diffbot', userAgent: 'Diffbot', purpose: 'Knowledge graph services used by AI products.' },
31 { name: 'Omgilibot', operator: 'Omgili', userAgent: 'Omgilibot', purpose: 'Powers Omgili AI training.' },
32 { name: 'ImagesiftBot', operator: 'ImageSift', userAgent: 'ImagesiftBot', purpose: 'Image retrieval for AI services.' },
33 { name: 'Timpibot', operator: 'Timpi', userAgent: 'Timpibot', purpose: 'Powers Timpi AI search and training.' },
34];
35
36function isPrivateIpv4(ip) {
37 const parts = ip.split('.').map(Number);
38 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;
39 const [a, b] = parts;
40 return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254)
41 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168);
42}
43
44function isPrivateIpv6(ip) {
45 const value = ip.toLowerCase();
46 return value === '::1' || value.startsWith('fc') || value.startsWith('fd') || value.startsWith('fe80:');
47}
48
49export async function normalizeAndValidateUrl(rawUrl) {
50 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');
51 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {
52 throw new Error('Only HTTP and HTTPS URLs are supported');
53 }
54 const url = new URL(/^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`);
55 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');
56 if (!url.hostname || url.username || url.password) throw new Error('URL credentials are not allowed');
57
58 const literalType = net.isIP(url.hostname);
59 if (literalType === 4 && isPrivateIpv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');
60 if (literalType === 6 && isPrivateIpv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');
61
62 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });
63 for (const record of records) {
64 if (record.family === 4 && isPrivateIpv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');
65 if (record.family === 6 && isPrivateIpv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');
66 }
67 url.pathname = '/';
68 url.search = '';
69 url.hash = '';
70 return url;
71}
72
73async function fetchText(url, timeoutSeconds, redirectsRemaining = 3) {
74 await normalizeAndValidateUrl(url.href);
75 const controller = new AbortController();
76 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);
77 try {
78 const response = await fetch(url, {
79 redirect: 'manual',
80 signal: controller.signal,
81 headers: { 'user-agent': USER_AGENT, accept: 'text/plain,*/*;q=0.1' },
82 });
83 if ([301, 302, 303, 307, 308].includes(response.status)) {
84 if (redirectsRemaining <= 0) throw new Error('Too many redirects');
85 const location = response.headers.get('location');
86 if (!location) throw new Error('Redirect without Location header');
87 return fetchText(new URL(location, url.href), timeoutSeconds, redirectsRemaining - 1);
88 }
89 const text = await response.text();
90 return {
91 ok: response.ok,
92 status: response.status,
93 url: response.url || url.href,
94 text: text.slice(0, MAX_BODY_BYTES),
95 contentType: response.headers.get('content-type') || '',
96 error: null,
97 };
98 } catch (error) {
99 return { ok: false, status: null, url: url.href, text: '', contentType: '', error: error.message };
100 } finally {
101 clearTimeout(timeout);
102 }
103}
104
105
106
107
108
109export function parseRobotsForAiCrawlers(body) {
110 const lines = body.split(/\r?\n/);
111 const groups = [];
112 let current = null;
113 let collectingAgents = false;
114
115 for (const rawLine of lines) {
116 const line = rawLine.replace(/#.*/, '').trim();
117 if (!line) {
118 collectingAgents = false;
119 continue;
120 }
121 const match = /^([^:]+):\s*(.*)$/.exec(line);
122 if (!match) continue;
123 const key = match[1].trim().toLowerCase();
124 const value = match[2].trim();
125 if (!value) continue;
126
127 if (key === 'user-agent') {
128 if (!collectingAgents || (current && current.rules.length > 0)) {
129 current = { userAgents: [], rules: [], sitemaps: [], crawlDelay: null };
130 groups.push(current);
131 }
132 current.userAgents.push(value);
133 collectingAgents = true;
134 continue;
135 }
136
137 collectingAgents = false;
138 if (!current) {
139 current = { userAgents: ['*'], rules: [], sitemaps: [], crawlDelay: null };
140 groups.push(current);
141 }
142 if (key === 'disallow') current.rules.push({ type: 'disallow', value });
143 else if (key === 'allow') current.rules.push({ type: 'allow', value });
144 else if (key === 'sitemap') current.sitemaps.push(value);
145 else if (key === 'crawl-delay') current.crawlDelay = Number(value);
146 }
147 return groups;
148}
149
150export function analyzeCrawlers(groups) {
151 const crawlerResults = [];
152
153 for (const crawler of AI_CRAWLERS) {
154 let matched = null;
155 let rules = [];
156 for (const group of groups) {
157
158
159
160 if (group.userAgents.includes(crawler.userAgent)) {
161 matched = crawler.userAgent;
162 rules = group.rules;
163 break;
164 }
165 }
166 let status = 'no-rule';
167 if (matched) {
168 const disallowRoot = rules.some((r) => r.type === 'disallow' && (r.value === '/' || r.value === '/*' || r.value === ''));
169 const disallowAny = rules.some((r) => r.type === 'disallow');
170 status = disallowRoot ? 'blocked' : disallowAny ? 'partial' : 'allowed';
171 }
172 crawlerResults.push({
173 name: crawler.name,
174 operator: crawler.operator,
175 userAgent: crawler.userAgent,
176 purpose: crawler.purpose,
177 matchedToken: matched,
178 status,
179 disallowPaths: rules.filter((r) => r.type === 'disallow').map((r) => r.value),
180 allowPaths: rules.filter((r) => r.type === 'allow').map((r) => r.value),
181 });
182 }
183 return crawlerResults;
184}
185
186export function gradeFromScore(score) {
187 if (score >= 85) return 'A';
188 if (score >= 70) return 'B';
189 if (score >= 55) return 'C';
190 if (score >= 40) return 'D';
191 return 'F';
192}
193
194export function scoreAndRecommend(crawlerResults, robotsFound) {
195 if (!robotsFound) {
196 return {
197 score: 0,
198 grade: 'F',
199 recommendations: ['Publish a robots.txt file so AI crawlers can be directed.'],
200 findings: [{ name: 'robots.txt found', status: 'fail', points: 0 }],
201 };
202 }
203
204 const findings = [{ name: 'robots.txt found', status: 'pass', points: 20 }];
205 let coverageScore = 0;
206
207
208
209
210
211 for (const c of crawlerResults) {
212 if (c.status === 'allowed') coverageScore += 4;
213 else if (c.status === 'no-rule') coverageScore += 3;
214 else if (c.status === 'partial') coverageScore += 2;
215 }
216 coverageScore = Math.min(coverageScore, 80);
217 const score = 20 + coverageScore;
218 findings.push({
219 name: 'AI crawler coverage',
220 status: coverageScore >= 60 ? 'pass' : coverageScore >= 30 ? 'warn' : 'fail',
221 points: coverageScore,
222 });
223
224 const blocked = crawlerResults.filter((c) => c.status === 'blocked');
225 const noRule = crawlerResults.filter((c) => c.status === 'no-rule');
226 const partial = crawlerResults.filter((c) => c.status === 'partial');
227 const recommendations = [];
228
229
230 const aiVisibilityCrawlers = ['GPTBot', 'OAI-SearchBot', 'ChatGPT-User', 'ClaudeBot', 'Claude-SearchBot', 'Google-Extended', 'Googlebot-Extended', 'PerplexityBot', 'AppleBot'];
231
232 if (noRule.length === crawlerResults.length) {
233 recommendations.push('No AI crawler rules found. Add explicit User-agent directives for AI crawlers (e.g. GPTBot, ClaudeBot, Google-Extended) to control inclusion in AI products.');
234 }
235 for (const c of blocked.filter((c) => aiVisibilityCrawlers.includes(c.name))) {
236 recommendations.push(`${c.name} is explicitly blocked. Unblock it to allow inclusion in ${c.operator} AI products if your GEO/AI-SEO strategy favors visibility.`);
237 }
238 const keyDefaulted = noRule.filter((c) => aiVisibilityCrawlers.includes(c.name));
239 if (keyDefaulted.length) {
240 recommendations.push(`AI crawlers with no explicit rule (${keyDefaulted.map((c) => c.name).join(', ')}) are allowed by default. Make the choice explicit for clearer policy.`);
241 }
242 for (const c of partial) {
243 recommendations.push(`${c.name} has partial Disallow rules (${c.disallowPaths.join(', ')}). Confirm whether this path scoping is intended.`);
244 }
245 if (!recommendations.length) {
246 recommendations.push('robots.txt has clear AI crawler rules. Review periodically as new crawlers appear.');
247 }
248
249 return { score: Math.min(score, 100), grade: gradeFromScore(Math.min(score, 100)), recommendations, findings };
250}
251
252export async function auditAiCrawlers(input) {
253 const baseUrl = await normalizeAndValidateUrl(input.startUrl);
254 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);
255 const robotsUrl = new URL('/robots.txt', baseUrl).href;
256
257 const result = await fetchText(new URL('/robots.txt', baseUrl), timeoutSeconds);
258 const robotsFound = result.ok && /^\s*User-agent/im.test(result.text);
259
260 const crawlers = robotsFound ? analyzeCrawlers(parseRobotsForAiCrawlers(result.text)) : AI_CRAWLERS.map((c) => ({
261 name: c.name, operator: c.operator, userAgent: c.userAgent, purpose: c.purpose,
262 matchedToken: null, status: 'no-rule', disallowPaths: [], allowPaths: [],
263 }));
264
265 const sitemaps = robotsFound ? parseRobotsForAiCrawlers(result.text).flatMap((g) => g.sitemaps) : [];
266
267 const { score, grade, recommendations, findings } = scoreAndRecommend(crawlers, robotsFound);
268
269 return {
270 inputUrl: input.startUrl,
271 siteOrigin: baseUrl.origin,
272 robotsUrl,
273 robotsFound,
274 status: result.status,
275 contentType: result.contentType,
276 checkedAt: new Date().toISOString(),
277 score,
278 grade,
279 findings,
280 recommendations,
281 sitemaps: [...new Set(sitemaps)],
282 crawlers,
283 blocked: crawlers.filter((c) => c.status === 'blocked').map((c) => c.name),
284 allowed: crawlers.filter((c) => c.status === 'allowed').map((c) => c.name),
285 noRule: crawlers.filter((c) => c.status === 'no-rule').map((c) => c.name),
286 partial: crawlers.filter((c) => c.status === 'partial').map((c) => c.name),
287 error: result.error || null,
288 };
289}
290
291const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
292
293if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {
294 await Actor.init();
295 try {
296 const result = await auditAiCrawlers((await Actor.getInput()) || {});
297 await Actor.pushData(result);
298 await Actor.setValue('OUTPUT', result);
299 Actor.log.info('AI crawler audit complete', { siteOrigin: result.siteOrigin, robotsFound: result.robotsFound, score: result.score, grade: result.grade });
300 } finally {
301 await Actor.exit();
302 }
303}
