1import { Actor } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4import { fileURLToPath } from 'node:url';
5
6const USER_AGENT = 'LlmsTxtDiscoveryAuditor/0.1 (+https://apify.com)';
7const MAX_BYTES = 1_000_000;
8const DEFAULT_TIMEOUT_SECONDS = 10;
9
10function isPrivateIPv4(ip) {
11 const parts = ip.split('.').map(Number);
12 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;
13 const [a, b] = parts;
14 return a === 10
15 || (a === 172 && b >= 16 && b <= 31)
16 || (a === 192 && b === 168)
17 || a === 127
18 || a === 0
19 || (a === 169 && b === 254);
20}
21
22function isPrivateIPv6(ip) {
23 const normalized = ip.toLowerCase();
24 return normalized === '::1'
25 || normalized.startsWith('fc')
26 || normalized.startsWith('fd')
27 || normalized.startsWith('fe80:');
28}
29
30export async function normalizeAndValidateUrl(rawUrl) {
31 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');
32 if (/^[a-z][a-z0-9+.-]*:\/\//i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {
33 throw new Error('Only HTTP and HTTPS URLs are supported');
34 }
35 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;
36 const url = new URL(withScheme);
37 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');
38 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');
39
40 const literalType = net.isIP(url.hostname);
41 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');
42 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');
43
44 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });
45 for (const record of records) {
46 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');
47 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');
48 }
49 return url;
50}
51
52export function buildOriginUrl(url) {
53 return new URL(`${url.protocol}//${url.host}`);
54}
55
56export function buildFileUrl(originUrl, filePath) {
57 const url = new URL(originUrl.href);
58 url.pathname = filePath;
59 url.search = '';
60 url.hash = '';
61 return url;
62}
63
64async function fetchPublicText(initialUrl, timeoutSeconds, redirectsRemaining = 3) {
65 await normalizeAndValidateUrl(initialUrl.href);
66 const controller = new AbortController();
67 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);
68 try {
69 const response = await fetch(initialUrl, {
70 redirect: 'manual',
71 signal: controller.signal,
72 headers: {
73 'user-agent': USER_AGENT,
74 accept: 'text/markdown,text/plain,text/*,*/*;q=0.1',
75 },
76 });
77
78 if ([301, 302, 303, 307, 308].includes(response.status)) {
79 if (redirectsRemaining <= 0) throw new Error('Too many redirects');
80 const location = response.headers.get('location');
81 if (!location) throw new Error('Redirect without Location header');
82 const nextUrl = new URL(location, initialUrl.href);
83 await normalizeAndValidateUrl(nextUrl.href);
84 return fetchPublicText(nextUrl, timeoutSeconds, redirectsRemaining - 1);
85 }
86
87 const arrayBuffer = await response.arrayBuffer();
88 const bytes = Math.min(arrayBuffer.byteLength, MAX_BYTES);
89 const text = new TextDecoder('utf-8', { fatal: false }).decode(arrayBuffer.slice(0, bytes));
90 return {
91 ok: response.ok,
92 status: response.status,
93 finalUrl: response.url,
94 contentType: response.headers.get('content-type') || '',
95 byteLength: arrayBuffer.byteLength,
96 truncated: arrayBuffer.byteLength > MAX_BYTES,
97 text,
98 };
99 } catch (error) {
100 return { ok: false, status: null, finalUrl: initialUrl.href, contentType: '', byteLength: 0, truncated: false, text: '', error: error.message };
101 } finally {
102 clearTimeout(timeout);
103 }
104}
105
106function unique(values) {
107 return [...new Set(values.filter(Boolean))];
108}
109
110export function parseMarkdownSummary(text, baseUrl) {
111 const lines = text.split(/\r?\n/);
112 const headings = [];
113 const links = [];
114 const linkPattern = /\[([^\]]+)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
115
116 for (const line of lines) {
117 const headingMatch = /^(#{1,6})\s+(.+?)\s*$/.exec(line);
118 if (headingMatch) headings.push({ level: headingMatch[1].length, text: headingMatch[2].trim() });
119
120 let match;
121 while ((match = linkPattern.exec(line)) !== null) {
122 try {
123 const href = new URL(match[2], baseUrl.href);
124 if (['http:', 'https:'].includes(href.protocol)) {
125 links.push({ text: match[1].trim(), url: href.href, domain: href.hostname });
126 }
127 } catch {
128
129 }
130 }
131 }
132
133 const title = headings.find((h) => h.level === 1)?.text || null;
134 const domains = unique(links.map((link) => link.domain));
135 const likelyDocsLinks = links.filter((link) => /docs|documentation|api|developer|guide|reference|pricing|product/i.test(`${link.text} ${link.url}`));
136
137 return {
138 title,
139 headingCount: headings.length,
140 headings: headings.slice(0, 20),
141 linkCount: links.length,
142 links: links.slice(0, 100),
143 uniqueLinkedDomains: domains,
144 likelyDocsLinks: likelyDocsLinks.slice(0, 25),
145 lineCount: lines.length,
146 };
147}
148
149function analyzeFile(path, fetchResult, baseUrl) {
150 const warnings = [];
151 const present = fetchResult.ok && fetchResult.status === 200 && fetchResult.text.trim().length > 0;
152
153 if (!fetchResult.ok) warnings.push(fetchResult.error || `HTTP status ${fetchResult.status}`);
154 if (fetchResult.status === 404) warnings.push(`${path} is missing`);
155 if (fetchResult.ok && !fetchResult.text.trim()) warnings.push(`${path} is empty`);
156 if (fetchResult.truncated) warnings.push(`${path} exceeds ${MAX_BYTES} bytes and was truncated`);
157 if (fetchResult.contentType && /text\/html/i.test(fetchResult.contentType)) warnings.push(`${path} returned HTML; markdown or plain text is expected`);
158
159 const summary = present ? parseMarkdownSummary(fetchResult.text, baseUrl) : {
160 title: null,
161 headingCount: 0,
162 headings: [],
163 linkCount: 0,
164 links: [],
165 uniqueLinkedDomains: [],
166 likelyDocsLinks: [],
167 lineCount: 0,
168 };
169
170 if (present && !summary.title) warnings.push(`${path} should start with a clear H1 title`);
171 if (present && summary.linkCount === 0) warnings.push(`${path} has no markdown links for AI agents to follow`);
172 if (present && summary.headingCount < 2) warnings.push(`${path} has limited section structure`);
173
174 return {
175 path,
176 present,
177 status: fetchResult.status,
178 finalUrl: fetchResult.finalUrl,
179 contentType: fetchResult.contentType,
180 byteLength: fetchResult.byteLength,
181 truncated: fetchResult.truncated,
182 ...summary,
183 warnings,
184 };
185}
186
187export function scoreAudit(files) {
188 const primary = files.find((file) => file.path === '/llms.txt');
189 if (!primary?.present) return 0;
190 let score = 45;
191 if (primary.title) score += 10;
192 if (primary.headingCount >= 2) score += 10;
193 if (primary.linkCount >= 5) score += 15;
194 if (primary.likelyDocsLinks.length >= 2) score += 10;
195 if (files.some((file) => file.path === '/llms-full.txt' && file.present)) score += 10;
196 return Math.min(score, 100);
197}
198
199export function buildRecommendations(files, score) {
200 const recommendations = [];
201 const primary = files.find((file) => file.path === '/llms.txt');
202 const full = files.find((file) => file.path === '/llms-full.txt');
203 if (!primary?.present) recommendations.push('Create /llms.txt with a short H1, key sections, and curated links to docs, pricing, products, and support pages');
204 if (primary?.present && !primary.title) recommendations.push('Add a top-level H1 title that names the site or product');
205 if (primary?.present && primary.linkCount < 5) recommendations.push('Add more curated markdown links for AI agents and search tools to follow');
206 if (primary?.present && primary.likelyDocsLinks.length < 2) recommendations.push('Include documentation, API, developer, guide, reference, or product links');
207 if (!full?.present) recommendations.push('Add /llms-full.txt for complete long-form context when practical');
208 if (score >= 80) recommendations.push('Good baseline; schedule periodic checks after docs or navigation changes');
209 return recommendations;
210}
211
212export async function auditLlmsTxt(input) {
213 const startUrl = await normalizeAndValidateUrl(input.startUrl);
214 const originUrl = buildOriginUrl(startUrl);
215 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);
216 const paths = input.checkFullFile === false ? ['/llms.txt'] : ['/llms.txt', '/llms-full.txt'];
217 const files = [];
218
219 for (const path of paths) {
220 const fileUrl = buildFileUrl(originUrl, path);
221 const fetchResult = await fetchPublicText(fileUrl, timeoutSeconds);
222 files.push(analyzeFile(path, fetchResult, fileUrl));
223 }
224
225 const score = scoreAudit(files);
226 return {
227 inputUrl: input.startUrl,
228 siteOrigin: originUrl.href.replace(/\/$/, ''),
229 ok: files.some((file) => file.present),
230 score,
231 checkedAt: new Date().toISOString(),
232 files,
233 recommendations: buildRecommendations(files, score),
234 };
235}
236
237const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
238
239if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {
240 await Actor.init();
241 try {
242 const input = await Actor.getInput();
243 const result = await auditLlmsTxt(input || {});
244 await Actor.pushData(result);
245 await Actor.setValue('OUTPUT', result);
246 Actor.log.info('LLMs.txt audit complete', { siteOrigin: result.siteOrigin, score: result.score });
247 } finally {
248 await Actor.exit();
249 }
250}
