1import { Actor, log } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4
5await Actor.init();
6
7const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
8const USER_AGENT = 'Mozilla/5.0 (compatible; LahcenAI-Bot/1.0; +https://apify.com/lahcen_ai)';
9
10
11async function fetchRetry(url, options = {}, { retries = 2, timeoutMs = 20000 } = {}) {
12 let lastError;
13 for (let attempt = 0; attempt <= retries; attempt++) {
14 try {
15 const res = await fetch(url, {
16 redirect: 'follow',
17 ...options,
18 headers: { 'user-agent': USER_AGENT, ...(options.headers || {}) },
19 signal: AbortSignal.timeout(timeoutMs),
20 });
21 if ((res.status === 429 || res.status >= 500) && attempt < retries) {
22 const retryAfter = Number(res.headers.get('retry-after'));
23 await res.body?.cancel().catch(() => {});
24 await sleep(Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter, 30) * 1000 : 1500 * 2 ** attempt);
25 continue;
26 }
27 return res;
28 } catch (e) {
29 lastError = e;
30 if (attempt < retries) await sleep(1500 * 2 ** attempt);
31 }
32 }
33 throw lastError;
34}
35
36
37async function readText(res, maxBytes = 5_000_000) {
38 if (!res.body) return '';
39 const chunks = [];
40 let size = 0;
41 for await (const chunk of res.body) {
42 size += chunk.length;
43 if (size > maxBytes) {
44 chunks.push(Buffer.from(chunk).subarray(0, Math.max(0, chunk.length - (size - maxBytes))));
45 break;
46 }
47 chunks.push(Buffer.from(chunk));
48 }
49 return Buffer.concat(chunks).toString('utf8');
50}
51
52
53async function mapPool(items, concurrency, fn) {
54 const results = new Array(items.length);
55 let next = 0;
56 const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
57 while (next < items.length) {
58 const i = next++;
59 results[i] = await fn(items[i], i);
60 }
61 });
62 await Promise.all(workers);
63 return results;
64}
65
66
67function datasetBudget() {
68 try {
69 const charging = Actor.getChargingManager();
70 if (charging.getPricingInfo().isPayPerEvent) {
71 return charging.calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
72 }
73 } catch { }
74 return Infinity;
75}
76
77
78class Output {
79 constructor() {
80 this.budget = datasetBudget();
81 this.count = 0;
82 }
83
84 get full() { return this.count >= this.budget; }
85
86 async push(item) {
87 if (this.full) return false;
88 this.count++;
89 await Actor.pushData(item);
90 return true;
91 }
92
93
94 fail(item) {
95 this.failed = this.failed || [];
96 this.failed.push(item);
97 log.warning(`Failed: ${JSON.stringify(item).slice(0, 300)}`);
98 }
99
100 async finish(label) {
101 if (this.failed?.length) await Actor.setValue('FAILED_INPUTS', this.failed);
102 const msg = `Done: ${this.count} ${label}${this.failed?.length ? `, ${this.failed.length} failed (free, see FAILED_INPUTS)` : ''}.`;
103 if (!this.count && this.failed?.length) {
104 await Actor.fail(`No results: all ${this.failed.length} input(s) failed. First error: ${this.failed[0].error}`);
105 return;
106 }
107 if (this.full) log.warning('Stopped early: the maximum charge limit for this run was reached.');
108 await Actor.setStatusMessage(msg, { isStatusMessageTerminal: true });
109 await Actor.exit();
110 }
111}
112
113
114function toHostname(value, { stripWww = false } = {}) {
115 let s = String(value ?? '').trim().toLowerCase();
116 if (!s) return null;
117 if (s.includes('@') && !s.includes('/')) s = s.split('@').pop();
118 if (!/^[a-z][a-z0-9+.-]*:\/\//.test(s)) s = `http://${s}`;
119 try {
120 let host = new URL(s).hostname.replace(/\.$/, '');
121 if (stripWww) host = host.replace(/^www\./, '');
122 if (!/^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9-]{2,63}$/.test(host)) return null;
123 return host;
124 } catch {
125 return null;
126 }
127}
128
129
130function toUrl(value) {
131 let s = String(typeof value === 'object' && value ? value.url : value ?? '').trim();
132 if (!s) return null;
133 if (!/^https?:\/\//i.test(s)) s = `https://${s}`;
134 try {
135 const u = new URL(s);
136 if (!['http:', 'https:'].includes(u.protocol)) return null;
137 u.hash = '';
138 return u.toString();
139 } catch {
140 return null;
141 }
142}
143
144function isPrivateIp(ip) {
145 if (net.isIPv4(ip)) {
146 const [a, b] = ip.split('.').map(Number);
147 return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31)
148 || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224;
149 }
150 const v = ip.toLowerCase();
151 if (v.startsWith('::ffff:')) return isPrivateIp(v.slice(7));
152 return v === '::1' || v === '::' || v.startsWith('fc') || v.startsWith('fd') || v.startsWith('fe80');
153}
154
155
156async function assertPublicHost(hostname) {
157 if (net.isIP(hostname)) {
158 if (isPrivateIp(hostname)) throw new Error('Private or internal addresses are not allowed.');
159 return;
160 }
161 const records = await dns.lookup(hostname, { all: true });
162 if (!records.length || records.some((r) => isPrivateIp(r.address))) {
163 throw new Error('Host resolves to a private or internal address.');
164 }
165}
166
167
168function collectList(...values) {
169 const out = [];
170 for (const v of values) {
171 if (!v) continue;
172 const arr = Array.isArray(v) ? v : String(v).split(/[\n,;]+/);
173 for (const item of arr) {
174 const s = typeof item === 'object' && item ? (item.url ?? item.value ?? '') : String(item);
175 if (s.trim()) out.push(s.trim());
176 }
177 }
178 return [...new Set(out)];
179}
180
181function clampInt(value, fallback, min, max) {
182 const n = Number.isFinite(Number(value)) ? Math.trunc(Number(value)) : fallback;
183 return Math.min(max, Math.max(min, n));
184}
185
186
187function decodeEntities(s) {
188 return String(s ?? '')
189 .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(Number(d)))
190 .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16)))
191 .replace(/"/g, '"').replace(/'|'/g, "'").replace(/</g, '<').replace(/>/g, '>')
192 .replace(/ /g, ' ').replace(/&/g, '&');
193}
194
195function parseAttrs(tag) {
196 const attrs = {};
197 const re = /([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*(?:=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
198 const inner = tag.replace(/^<\s*[a-zA-Z0-9-]+/, '').replace(/\/?>$/, '');
199 let m;
200 while ((m = re.exec(inner))) attrs[m[1].toLowerCase()] = decodeEntities(m[2] ?? m[3] ?? m[4] ?? '');
201 return attrs;
202}
203
204function findTags(html, name) {
205 return [...String(html).matchAll(new RegExp(`<${name}\\b[^>]*>`, 'gi'))].map((m) => parseAttrs(m[0]));
206}
207
208function textOf(html) {
209 return decodeEntities(String(html)
210 .replace(/<script[\s\S]*?<\/script>/gi, ' ')
211 .replace(/<style[\s\S]*?<\/style>/gi, ' ')
212 .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
213 .replace(/<[^>]+>/g, ' '))
214 .replace(/\s+/g, ' ').trim();
215}
216
217function firstMatch(html, re) {
218 const m = String(html).match(re);
219 return m ? textOf(m[1]).trim() : null;
220}
221
222
223function extractMeta(html, baseUrl) {
224 const metas = findTags(html, 'meta');
225 const links = findTags(html, 'link');
226 const byName = (n) => metas.find((m) => (m.name || '').toLowerCase() === n)?.content ?? null;
227 const abs = (u) => { try { return u ? new URL(u, baseUrl).toString() : null; } catch { return null; } };
228 const og = {};
229 const twitter = {};
230 for (const m of metas) {
231 const key = (m.property || m.name || '').toLowerCase();
232 if (key.startsWith('og:') && m.content != null && !(key in og)) og[key.slice(3)] = m.content;
233 if (key.startsWith('twitter:') && m.content != null && !(key in twitter)) twitter[key.slice(8)] = m.content;
234 }
235 const jsonLd = [];
236 for (const m of String(html).matchAll(/<script\b[^>]*type\s*=\s*["']?application\/ld\+json["']?[^>]*>([\s\S]*?)<\/script>/gi)) {
237 try { jsonLd.push(JSON.parse(m[1].trim())); } catch { jsonLd.push({ parseError: true }); }
238 }
239 const schemaTypes = new Set();
240 const walk = (node) => {
241 if (Array.isArray(node)) return node.forEach(walk);
242 if (node && typeof node === 'object') {
243 const t = node['@type'];
244 if (t) (Array.isArray(t) ? t : [t]).forEach((x) => schemaTypes.add(String(x)));
245 for (const v of Object.values(node)) if (v && typeof v === 'object') walk(v);
246 }
247 };
248 walk(jsonLd);
249 const htmlTag = (String(html).match(/<html\b[^>]*>/i) || [''])[0];
250 const relHas = (l, r) => (l.rel || '').toLowerCase().split(/\s+/).includes(r);
251 return {
252 title: firstMatch(html, /<title[^>]*>([\s\S]*?)<\/title>/i),
253 metaDescription: byName('description'),
254 metaRobots: byName('robots'),
255 canonical: abs(links.find((l) => relHas(l, 'canonical'))?.href),
256 lang: htmlTag ? parseAttrs(htmlTag).lang ?? null : null,
257 h1: [...String(html).matchAll(/<h1\b[^>]*>([\s\S]*?)<\/h1>/gi)].map((m) => textOf(m[1])).filter(Boolean),
258 favicon: abs(links.find((l) => relHas(l, 'icon'))?.href) ?? abs('/favicon.ico'),
259 openGraph: og,
260 twitterCard: twitter,
261 jsonLd,
262 schemaTypes: [...schemaTypes],
263 alternateFeeds: links.filter((l) => relHas(l, 'alternate') && /rss|atom/i.test(l.type || '')).map((l) => abs(l.href)).filter(Boolean),
264 hreflang: links.filter((l) => relHas(l, 'alternate') && l.hreflang).map((l) => ({ hreflang: l.hreflang, href: abs(l.href) })),
265 };
266}
267
268
269function parseRobots(text) {
270 const groups = [];
271 let current = null;
272 let lastWasAgent = false;
273 const sitemaps = [];
274 for (const raw of String(text || '').split(/\r?\n/)) {
275 const line = raw.replace(/#.*$/, '').trim();
276 const m = line.match(/^([a-zA-Z-]+)\s*:\s*(.*)$/);
277 if (!m) continue;
278 const key = m[1].toLowerCase();
279 const value = m[2].trim();
280 if (key === 'user-agent') {
281 if (!lastWasAgent || !current) { current = { agents: [], rules: [] }; groups.push(current); }
282 current.agents.push(value.toLowerCase());
283 lastWasAgent = true;
284 } else if (key === 'allow' || key === 'disallow') {
285 if (current) current.rules.push({ allow: key === 'allow', path: value });
286 lastWasAgent = false;
287 } else if (key === 'sitemap') {
288 sitemaps.push(value);
289 } else {
290 lastWasAgent = false;
291 }
292 }
293 return { groups, sitemaps };
294}
295
296function robotsAllows(robots, agent, path = '/') {
297 const ua = agent.toLowerCase();
298 let matched = robots.groups.filter((g) => g.agents.some((a) => a !== '*' && ua.includes(a)));
299 let usedGroup = 'specific';
300 if (!matched.length) { matched = robots.groups.filter((g) => g.agents.includes('*')); usedGroup = matched.length ? 'wildcard' : 'none'; }
301 const rules = matched.flatMap((g) => g.rules).filter((r) => r.path !== '' || r.allow);
302 let best = null;
303 for (const r of rules) {
304 if (r.path === '') continue;
305 const pattern = r.path.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
306 const re = new RegExp(`^${pattern.endsWith('\\$') ? `${pattern.slice(0, -2)}$` : pattern}`);
307 if (re.test(path)) {
308 if (!best || r.path.length > best.path.length || (r.path.length === best.path.length && r.allow)) best = r;
309 }
310 }
311 return { allowed: best ? best.allow : true, matchedGroup: usedGroup, rule: best ? `${best.allow ? 'Allow' : 'Disallow'}: ${best.path}` : null };
312}
313
314const AI_BOTS = [
315 ['GPTBot', 'OpenAI', 'training'], ['OAI-SearchBot', 'OpenAI', 'search'], ['ChatGPT-User', 'OpenAI', 'user-browsing'],
316 ['ClaudeBot', 'Anthropic', 'training'], ['Claude-SearchBot', 'Anthropic', 'search'], ['Claude-User', 'Anthropic', 'user-browsing'],
317 ['PerplexityBot', 'Perplexity', 'search'], ['Perplexity-User', 'Perplexity', 'user-browsing'],
318 ['Google-Extended', 'Google', 'training (Gemini)'], ['Googlebot', 'Google', 'search (AI Overviews)'],
319 ['Bingbot', 'Microsoft', 'search (Copilot)'], ['Applebot-Extended', 'Apple', 'training'], ['Amazonbot', 'Amazon', 'search/assistant'],
320 ['meta-externalagent', 'Meta', 'training'], ['CCBot', 'Common Crawl', 'training'], ['Bytespider', 'ByteDance', 'training'],
321 ['cohere-ai', 'Cohere', 'training'], ['DuckAssistBot', 'DuckDuckGo', 'search'], ['MistralAI-User', 'Mistral', 'user-browsing'],
322];
323const OUR_AGENT = 'LahcenAI-Bot';
324
325async function getText(url, maxBytes = 3_000_000) {
326 const res = await fetchRetry(url, { headers: { accept: 'text/html,application/xhtml+xml;q=0.9,text/plain;q=0.8,*/*;q=0.1' } }, { retries: 1, timeoutMs: 15000 });
327 const body = res.ok ? await readText(res, maxBytes) : (await res.body?.cancel().catch(() => {}), '');
328 return { status: res.status, url: res.url, headers: res.headers, body, contentType: res.headers.get('content-type') || '' };
329}
330
331function sitemapLocs(xml) {
332 return [...String(xml).matchAll(/<loc>\s*(?:<!\[CDATA\[)?\s*([^<\]\s]+)\s*(?:\]\]>)?\s*<\/loc>/gi)].map((m) => decodeEntities(m[1]));
333}
334
335async function collectSitemapUrls(origin, robotsSitemaps, limit) {
336 const queue = robotsSitemaps.length ? [...robotsSitemaps] : [`${origin}/sitemap.xml`, `${origin}/sitemap_index.xml`];
337 const pages = new Set();
338 const seen = new Set();
339 let found = false;
340 while (queue.length && pages.size < limit && seen.size < 15) {
341 const sm = queue.shift();
342 if (seen.has(sm)) continue;
343 seen.add(sm);
344 try {
345 const r = await getText(sm, 10_000_000);
346 if (r.status !== 200 || !/<(urlset|sitemapindex)/i.test(r.body)) continue;
347 found = true;
348 const locs = sitemapLocs(r.body);
349 if (/<sitemapindex/i.test(r.body)) queue.push(...locs.filter((u) => !/\.gz$/i.test(u)));
350 else for (const u of locs) { if (pages.size >= limit) break; if (u.startsWith(origin)) pages.add(u.split('#')[0]); }
351 } catch { }
352 }
353 return { found, urls: [...pages] };
354}
355
356function mdEscape(s) { return String(s || '').replace(/\s+/g, ' ').replace(/[[\]]/g, '').trim(); }
357
358function buildLlmsTxt({ siteName, summary, pages, origin }) {
359 const sections = new Map();
360 for (const p of pages) {
361 let section = 'Main pages';
362 try {
363 const seg = new URL(p.url).pathname.split('/').filter(Boolean)[0];
364 if (seg && !/^[a-z]{2}(-[a-z]{2})?$/i.test(seg)) section = seg.replace(/[-_]+/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
365 } catch { }
366 if (!sections.has(section)) sections.set(section, []);
367 sections.get(section).push(p);
368 }
369 const lines = [`# ${mdEscape(siteName) || new URL(origin).hostname}`, ''];
370 if (summary) lines.push(`> ${mdEscape(summary)}`, '');
371 const ordered = [...sections.entries()].sort((a, b) => (a[0] === 'Main pages' ? -1 : b[0] === 'Main pages' ? 1 : b[1].length - a[1].length));
372 for (const [name, items] of ordered) {
373 lines.push(`## ${name}`, '');
374 for (const p of items) lines.push(`- [${mdEscape(p.title) || p.url}](${p.url})${p.description ? `: ${mdEscape(p.description)}` : ''}`);
375 lines.push('');
376 }
377 return lines.join('\n').trim() + '\n';
378}
379
380async function auditSite(startUrl, { generate, maxPages, kvsId }) {
381 const origin = new URL(startUrl).origin;
382 const host = new URL(startUrl).hostname;
383 await assertPublicHost(host);
384 const issues = [];
385 const add = (severity, code, message, fix) => issues.push({ severity, code, message, fix });
386 let score = 0;
387
388
389 const robotsRes = await getText(`${origin}/robots.txt`, 500_000).catch(() => ({ status: 0, body: '' }));
390 const robotsOk = robotsRes.status === 200 && !/<html/i.test(robotsRes.body.slice(0, 500));
391 const robots = parseRobots(robotsOk ? robotsRes.body : '');
392 const aiCrawlers = AI_BOTS.map(([agent, company, purpose]) => {
393 const r = robotsAllows(robots, agent, '/');
394 return { agent, company, purpose, allowed: r.allowed, rule: r.rule, explicitlyListed: r.matchedGroup === 'specific' };
395 });
396 const searchBots = aiCrawlers.filter((b) => /search|user/.test(b.purpose));
397 const blockedSearch = searchBots.filter((b) => !b.allowed).map((b) => b.agent);
398 score += Math.round(30 * (searchBots.length - blockedSearch.length) / searchBots.length);
399 if (blockedSearch.length) add('high', 'AI_SEARCH_BOTS_BLOCKED', `robots.txt blocks AI search/answer crawlers: ${blockedSearch.join(', ')}. Your site cannot be cited by these assistants.`, 'Allow these user agents in robots.txt if you want visibility in AI answers.');
400 const blockedTraining = aiCrawlers.filter((b) => /training/.test(b.purpose) && !b.allowed).map((b) => b.agent);
401 if (!robotsOk) add('low', 'NO_ROBOTS_TXT', 'No robots.txt found.', 'Publish robots.txt with a Sitemap line.');
402
403
404 const llms = await getText(`${origin}/llms.txt`, 2_000_000).catch(() => ({ status: 0, body: '' }));
405 const hasLlmsTxt = llms.status === 200 && /^\s*#\s+\S/.test(llms.body) && !/<html/i.test(llms.body.slice(0, 500));
406 const llmsFull = await getText(`${origin}/llms-full.txt`, 200_000).catch(() => ({ status: 0, body: '' }));
407 const hasLlmsFull = llmsFull.status === 200 && llmsFull.body.length > 50 && !/<html/i.test(llmsFull.body.slice(0, 500));
408 if (hasLlmsTxt) score += 10; else add('medium', 'NO_LLMS_TXT', 'No valid /llms.txt file (the emerging standard that guides AI assistants to your key pages).', 'Publish the generated llms.txt at the site root.');
409
410
411 const ourAccess = robotsAllows(robots, OUR_AGENT, new URL(startUrl).pathname || '/');
412 if (!ourAccess.allowed) throw new Error('robots.txt disallows crawling this page; audit stopped out of respect for the site policy.');
413 const started = Date.now();
414 const home = await getText(startUrl);
415 const responseTimeMs = Date.now() - started;
416 if (home.status >= 400 || !home.body) throw new Error(`Homepage returned HTTP ${home.status}`);
417 const meta = extractMeta(home.body, home.url);
418 const visibleText = textOf(home.body);
419 const wordCount = visibleText ? visibleText.split(/\s+/).length : 0;
420 const xRobots = home.headers.get('x-robots-tag') || '';
421 const noindex = /noindex/i.test(`${meta.metaRobots || ''} ${xRobots}`);
422 if (home.url.startsWith('https://')) score += 5; else add('high', 'NO_HTTPS', 'Site is not served over HTTPS.', 'Enable HTTPS.');
423 if (noindex) add('critical', 'NOINDEX', 'Page is marked noindex: it will not appear in search or AI answers.', 'Remove noindex.'); else score += 5;
424 if (meta.title) score += 5; else add('medium', 'NO_TITLE', 'Missing <title>.', 'Add a descriptive title.');
425 if (meta.metaDescription) score += 5; else add('medium', 'NO_META_DESCRIPTION', 'Missing meta description (often used as the summary by AI engines).', 'Add a 120–160 character meta description.');
426 if (meta.schemaTypes.length) score += 10; else add('medium', 'NO_STRUCTURED_DATA', 'No JSON-LD structured data found.', 'Add Organization/WebSite (and Product/Article/FAQPage where relevant) schema.org JSON-LD.');
427 if (meta.schemaTypes.some((t) => /Organization|LocalBusiness|Corporation|Person/.test(t))) score += 5; else add('low', 'NO_ENTITY_SCHEMA', 'No Organization/LocalBusiness entity schema: AI engines rely on it to understand who you are.', 'Add Organization schema with name, logo, sameAs links.');
428 if (Object.keys(meta.openGraph).length) score += 5; else add('low', 'NO_OPEN_GRAPH', 'No Open Graph tags.', 'Add og:title, og:description, og:image.');
429 if (wordCount >= 150) score += 10; else add('high', 'LOW_SERVER_TEXT', `Only ${wordCount} words in the server-rendered HTML. Most AI crawlers do not execute JavaScript.`, 'Server-side render or pre-render key content.');
430 if (meta.h1.length === 1) score += 3; else add('low', meta.h1.length ? 'MULTIPLE_H1' : 'NO_H1', `${meta.h1.length} <h1> headings found.`, 'Use exactly one descriptive H1.');
431 if (meta.lang) score += 2; else add('low', 'NO_LANG', 'Missing <html lang>.', 'Declare the page language.');
432
433 const sm = await collectSitemapUrls(origin, robots.sitemaps.filter((s) => { try { return new URL(s).origin === origin || new URL(s).hostname.endsWith(host.replace(/^www\./, '')); } catch { return false; } }), generate ? maxPages : 1);
434 if (sm.found) score += 5; else add('medium', 'NO_SITEMAP', 'No XML sitemap found (robots.txt or /sitemap.xml).', 'Publish a sitemap and reference it in robots.txt.');
435
436
437 let generatedLlmsTxt = null;
438 let generatedLlmsTxtUrl = null;
439 if (generate) {
440 const pages = [{ url: home.url, title: meta.title, description: meta.metaDescription }];
441 const candidates = sm.urls.filter((u) => u !== home.url && robotsAllows(robots, OUR_AGENT, new URL(u).pathname).allowed).slice(0, maxPages - 1);
442 await mapPool(candidates, 4, async (u) => {
443 try {
444 const p = await getText(u, 1_500_000);
445 if (p.status === 200 && /html/i.test(p.contentType)) {
446 const m = extractMeta(p.body, p.url);
447 if (!/noindex/i.test(m.metaRobots || '')) pages.push({ url: p.url, title: m.title, description: m.metaDescription });
448 }
449 } catch { }
450 await sleep(250);
451 });
452 const siteName = meta.openGraph.site_name || (meta.title || '').split(/\s[|–—-]\s/).pop() || host;
453 generatedLlmsTxt = buildLlmsTxt({ siteName, summary: meta.metaDescription || meta.openGraph.description, pages, origin });
454 const key = `llms-${host.replace(/[^a-z0-9.-]/gi, '_')}.txt`;
455 await Actor.setValue(key, generatedLlmsTxt, { contentType: 'text/plain; charset=utf-8' });
456 generatedLlmsTxtUrl = kvsId ? `https://api.apify.com/v2/key-value-stores/${kvsId}/records/${key}` : key;
457 }
458
459 score = Math.min(100, score);
460 const order = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
461 issues.sort((a, b) => order[a.severity] - order[b.severity]);
462 return {
463 url: home.url,
464 domain: host,
465 geoScore: score,
466 grade: score >= 85 ? 'A' : score >= 70 ? 'B' : score >= 55 ? 'C' : score >= 40 ? 'D' : 'F',
467 aiSearchBotsBlocked: blockedSearch,
468 aiTrainingBotsBlocked: blockedTraining,
469 aiCrawlers,
470 hasRobotsTxt: robotsOk,
471 hasLlmsTxt,
472 hasLlmsFullTxt: hasLlmsFull,
473 hasSitemap: sm.found,
474 sitemapUrlsSampled: sm.urls.length,
475 noindex,
476 title: meta.title,
477 metaDescription: meta.metaDescription,
478 canonical: meta.canonical,
479 lang: meta.lang,
480 h1: meta.h1.slice(0, 5),
481 schemaTypes: meta.schemaTypes,
482 openGraph: meta.openGraph,
483 serverRenderedWordCount: wordCount,
484 responseTimeMs,
485 issueCount: issues.length,
486 issues,
487 generatedLlmsTxtUrl,
488 generatedLlmsTxt,
489 checkedAt: new Date().toISOString(),
490 };
491}
492
493const input = (await Actor.getInput()) ?? {};
494const urls = [...new Set(collectList(input.startUrls, input.urls).map(toUrl).filter(Boolean))];
495if (!urls.length) throw new Error('Please provide at least one website URL in "startUrls" (for example "https://apify.com").');
496const generate = input.generateLlmsTxt !== false;
497const maxPages = clampInt(input.maxPagesForLlmsTxt, 25, 1, 200);
498const kvsId = Actor.getEnv().defaultKeyValueStoreId;
499const out = new Output();
500await mapPool(urls, clampInt(input.maxConcurrency, 3, 1, 10), async (u) => {
501 if (out.full) return;
502 let item;
503 try { item = await auditSite(u, { generate, maxPages, kvsId }); } catch (e) {
504 out.fail({ url: u, error: e.message });
505 return;
506 }
507 await out.push(item);
508});
509await out.finish('site(s) audited');
