1import { Actor, log } from 'apify';
2import { domainsFromDataset, DOMAIN_FIELDS } from './dataset.js';
3import {
4 BOTS, BOT_INDEX, MAX_ROBOTS_BYTES, parseRobots, isAllowed, botAccess, verdict, looksLikeHtml, parseLlmsTxt, metaRobots, directives, toTarget,
5} from './robots.js';
6
7const EVENT = 'domain-checked';
8const UA = 'Mozilla/5.0 (compatible; AICrawlerAccessChecker/1.0; +https://apify.com/swillz/ai-crawler-access-checker)';
9const OWN_TOKEN = 'AICrawlerAccessChecker';
10class InputError extends Error {}
11class NetError extends Error {}
12
13
14async function readCapped(res, max) {
15 if (!res.body) return { text: '', bytes: 0, truncated: false };
16 const reader = res.body.getReader(); const chunks = []; let bytes = 0; let truncated = false;
17 try {
18 for (;;) {
19 const { done, value } = await reader.read();
20 if (done) break;
21 chunks.push(value); bytes += value.length;
22 if (bytes >= max) { truncated = true; break; }
23 }
24 } finally { try { await reader.cancel(); } catch { } }
25 const buf = Buffer.concat(chunks.map((c) => Buffer.from(c)));
26 return { text: buf.subarray(0, max).toString('utf8'), bytes, truncated };
27}
28
29async function get(url, timeoutMs, max) {
30 let res;
31 try {
32 res = await fetch(url, { redirect: 'follow', signal: AbortSignal.timeout(timeoutMs), headers: { 'user-agent': UA, accept: 'text/plain,text/markdown,text/html;q=0.9,*/*;q=0.5', 'accept-language': 'en' } });
33 } catch (e) {
34 const c = e?.cause?.code || e?.name || '';
35 throw new NetError(/ENOTFOUND|EAI_AGAIN/.test(c) ? 'DNS lookup failed (domain does not resolve)' : /Timeout|Abort/i.test(c) ? `no response within ${Math.round(timeoutMs / 1000)} s` : /CERT|TLS|SSL/i.test(c) ? 'TLS/certificate error' : `connection failed (${c || e.message})`);
36 }
37 const body = await readCapped(res, max);
38 return { status: res.status, url: res.url || url, headers: res.headers, ...body };
39}
40
41await Actor.init();
42try {
43 const input = (await Actor.getInput()) ?? {};
44 const {
45 domains = [], bots = [], customUserAgents = [], testPath = '/', checkLlmsTxt = true, checkHomepage = true, includeBotDetails = true,
46 concurrency = 10, timeoutSecs = 15,
47 } = input;
48 const raw = (Array.isArray(domains) ? domains : String(domains).split(/[\n,;\s]+/))
49 .map((w) => (typeof w === 'object' && w ? (w.url ?? w.domain ?? w.website) : w)).map((w) => String(w ?? '').trim()).filter(Boolean);
50 const capRaw = input.maxDomains ?? input.maxItems ?? input.maxResults;
51 let cap = Number.isFinite(Number(capRaw)) && Number(capRaw) > 0 ? Math.floor(Number(capRaw)) : 1000;
52 const envCap = Number(process.env.ACTOR_MAX_PAID_DATASET_ITEMS);
53 if (Number.isFinite(envCap) && envCap > 0) cap = Math.min(cap, envCap);
54
55 const dsId = String(input.datasetId || input?.payload?.resource?.defaultDatasetId || input?.resource?.defaultDatasetId || '').trim();
56 if (dsId) {
57 const fromDs = await domainsFromDataset(Actor.newClient(), dsId, String(input.domainField || '').trim(), Math.max(1, Math.floor(Number(input.datasetMaxItems) || 1000)));
58 raw.push(...fromDs.values.filter((v) => !v.includes('@')));
59 log.info(`Read ${fromDs.values.length} values from dataset ${dsId} (field "${fromDs.field ?? 'none found'}", ${fromDs.itemsScanned} items scanned).`);
60 if (!raw.length) throw new InputError(`Dataset ${dsId} has no domain or website values${fromDs.field ? ` in field "${fromDs.field}"` : ''}. Set "domainField" (tried: ${DOMAIN_FIELDS.filter((f) => !/mail/i.test(f)).join(', ')}).`);
61 }
62 if (!raw.length) throw new InputError('Add at least one domain or website URL, e.g. "nytimes.com", or pass a datasetId from another Actor run.');
63
64
65 const wanted = (Array.isArray(bots) ? bots : String(bots).split(',')).map((b) => String(b).trim()).filter(Boolean);
66 const unknownBots = wanted.filter((b) => !BOT_INDEX.has(b.toLowerCase()));
67 if (unknownBots.length) throw new InputError(`Unknown bot(s) ${unknownBots.map((b) => `"${b}"`).join(', ')} in "bots". Use names from the list (e.g. "GPTBot", "ClaudeBot") or put other user-agent tokens in "customUserAgents".`);
68 const botList = wanted.length ? wanted.map((b) => BOT_INDEX.get(b.toLowerCase())) : BOTS;
69 const custom = (Array.isArray(customUserAgents) ? customUserAgents : String(customUserAgents).split(','))
70 .map((c) => String(c).trim().split('/')[0]).filter((c) => /^[A-Za-z0-9._-]{2,60}$/.test(c) && !BOT_INDEX.has(c.toLowerCase()));
71 const checkList = [...botList, ...[...new Set(custom)].slice(0, 20).map((c) => ({ bot: c, operator: null, purpose: 'custom', product: null }))];
72 let path = String(testPath || '/').trim();
73 if (/^https?:\/\//i.test(path)) { try { const u = new URL(path); path = u.pathname + u.search; } catch { path = '/'; } }
74 if (!path.startsWith('/')) path = `/${path}`;
75 const timeoutMs = Math.max(3, Math.min(60, Number(timeoutSecs) || 15)) * 1000;
76
77 const seen = new Set(); const targets = []; const invalid = [];
78 for (const r of raw) {
79 const t = toTarget(r);
80 if (!t) { invalid.push(r.includes('@') ? '(email address)' : r.slice(0, 200)); continue; }
81 const key = `${t.host}:${t.port}`;
82 if (seen.has(key)) continue; seen.add(key); targets.push({ ...t, input: r.slice(0, 300) });
83 }
84 const overCap = targets.length > cap ? targets.splice(cap) : [];
85 if (overCap.length) log.warning(`maxDomains is ${cap}: ${overCap.length} extra domains were not checked.`);
86
87 const charging = Actor.getChargingManager();
88 const counts = {}; const verdicts = {}; const blockedTally = {}; let llmsFound = 0; let chargeLimitHit = false; const skipped = [];
89
90 async function fetchRobots(t) {
91 const port = t.port ? `:${t.port}` : '';
92 const origins = t.scheme ? [`${t.scheme}://${t.host}${port}`]
93 : [`https://${t.host}${port}`, ...(!t.host.startsWith('www.') && t.host.split('.').length === 2 ? [`https://www.${t.host}${port}`] : []), `http://${t.host}${port}`];
94 let lastErr = null;
95 for (const origin of origins) {
96 for (let attempt = 0; attempt < 2; attempt++) {
97 try {
98 const r = await get(`${origin}/robots.txt`, timeoutMs, MAX_ROBOTS_BYTES);
99 if (r.status >= 500 && attempt === 0) { await new Promise((ok) => setTimeout(ok, 1500)); continue; }
100 return { ...r, origin };
101 } catch (e) {
102 lastErr = e;
103 if (/DNS/.test(e.message)) break;
104 if (attempt === 0) continue;
105 }
106 }
107 }
108 throw lastErr ?? new NetError('connection failed');
109 }
110
111 async function check(t) {
112 const base = { domain: t.host, input: t.input };
113 const now = () => new Date().toISOString();
114 let rb;
115 try { rb = await fetchRobots(t); } catch (e) {
116 return { ...base, status: 'unreachable', error: `Could not reach the site: ${e.message}. Not charged.`, checkedAt: now() };
117 }
118 const finalOrigin = new URL(rb.url).origin;
119 const robotsTxtUrl = rb.url;
120 let note = null; let parsed = { groups: [], sitemaps: [], lines: 0, unknownLines: 0 }; let found = false;
121 if (rb.status === 401 || rb.status === 403 || rb.status === 429) {
122 return { ...base, status: 'robots_inaccessible', robotsTxtUrl, robotsTxtStatus: rb.status,
123 error: `robots.txt answered HTTP ${rb.status} to our checker (usually a firewall or bot protection such as Cloudflare). Its rules could not be read, and the firewall may also block AI crawlers directly. Not charged.`, checkedAt: now() };
124 }
125 if (rb.status >= 500) {
126 return { ...base, status: 'unreachable', robotsTxtUrl, robotsTxtStatus: rb.status, error: `robots.txt answered HTTP ${rb.status} twice (server error). Under RFC 9309 crawlers treat this as "disallow everything" until it recovers. Not charged; try again later.`, checkedAt: now() };
127 }
128 if (rb.status >= 200 && rb.status < 300) {
129 if (looksLikeHtml(rb.text, rb.headers.get('content-type') || '')) note = 'robots.txt returned an HTML page instead of a robots file, so no rules apply (every crawler is allowed).';
130 else { parsed = parseRobots(rb.text); found = true; if (!parsed.groups.length) note = 'robots.txt has no user-agent groups, so every crawler is allowed.'; }
131 if (rb.truncated) note = `robots.txt is larger than ${MAX_ROBOTS_BYTES / 1000} KB; only the first part was read (as Google does).`;
132 } else note = `No robots.txt (HTTP ${rb.status}), so every crawler is allowed.`;
133
134 const botRows = checkList.map((b) => {
135 const a = botAccess(parsed, b.bot, path);
136 return { bot: b.bot, operator: b.operator, purpose: b.purpose, access: a.access, allowed: a.allowed, matchedUserAgent: a.matchedUserAgent, matchedRule: a.matchedRule };
137 });
138 const star = isAllowed(parsed, '*', '/');
139 const v = verdict(botRows);
140 const ai = botRows.filter((r) => r.purpose !== 'search_engine' && r.purpose !== 'custom');
141 const row = {
142 ...base, status: 'checked', robotsTxtUrl, robotsTxtStatus: rb.status, robotsTxtFound: found, robotsTxtNote: note,
143 checkedPath: path, aiAccessVerdict: v, botsChecked: botRows.length,
144 blockedBots: botRows.filter((r) => r.access === 'blocked').map((r) => r.bot),
145 partiallyBlockedBots: botRows.filter((r) => r.access === 'partial').map((r) => r.bot),
146 allowedBots: botRows.filter((r) => r.access === 'allowed').map((r) => r.bot),
147 trainingBotsBlocked: ai.filter((r) => r.purpose === 'training' && !r.allowed).length,
148 answerBotsBlocked: ai.filter((r) => (r.purpose === 'search' || r.purpose === 'user') && !r.allowed).length,
149 aiSearchAllowed: ai.filter((r) => r.purpose === 'search' || r.purpose === 'user').every((r) => r.allowed),
150 googlebotAllowed: botAccess(parsed, 'Googlebot', path).allowed,
151 bingbotAllowed: botAccess(parsed, 'Bingbot', path).allowed,
152 contentSignal: star.contentSignal,
153 crawlDelaySecs: star.crawlDelay,
154 sitemaps: parsed.sitemaps,
155 bots: includeBotDetails ? botRows : null,
156 };
157 for (const b of row.blockedBots) blockedTally[b] = (blockedTally[b] ?? 0) + 1;
158
159 const ownOk = (p) => isAllowed(parsed, OWN_TOKEN, p).allowed;
160 if (checkLlmsTxt) {
161 const one = async (file, max) => {
162 const url = `${finalOrigin}/${file}`;
163 if (!ownOk(`/${file}`)) return { found: null, url, httpStatus: null, note: `robots.txt disallows /${file} for generic crawlers, so it was not fetched.` };
164 try {
165 const r = await get(url, timeoutMs, max);
166 const ok = r.status === 200 && r.text.trim().length > 0 && !looksLikeHtml(r.text, r.headers.get('content-type') || '');
167 const len = Number(r.headers.get('content-length'));
168 return { found: ok, url: ok ? r.url : url, httpStatus: r.status, sizeBytes: ok ? (r.truncated ? (Number.isFinite(len) && len > 0 ? len : null) : r.bytes) : null, text: ok ? r.text : null,
169 note: ok ? null : r.status === 200 ? 'Returned an HTML page (soft 404), not a text file.' : null };
170 } catch (e) { return { found: null, url, httpStatus: null, note: `Could not fetch: ${e.message}.` }; }
171 };
172 const [l, lf] = await Promise.all([one('llms.txt', 200000), one('llms-full.txt', 65536)]);
173 const meta = l.found ? parseLlmsTxt(l.text) : {};
174 row.llmsTxt = { found: l.found, url: l.url, httpStatus: l.httpStatus, sizeBytes: l.sizeBytes ?? null, title: meta.title ?? null, summary: meta.summary ?? null,
175 sections: meta.sections ?? null, linkCount: meta.linkCount ?? null, followsSpec: l.found ? meta.validFormat : null, note: l.note ?? null };
176 row.llmsFullTxt = { found: lf.found, url: lf.url, httpStatus: lf.httpStatus, sizeBytes: lf.sizeBytes ?? null, note: lf.note ?? null };
177 if (l.found) llmsFound++;
178 } else { row.llmsTxt = null; row.llmsFullTxt = null; }
179
180 if (checkHomepage) {
181 if (!ownOk(path)) row.homepage = { fetched: false, note: `robots.txt disallows ${path} for generic crawlers, so the page was not fetched.` };
182 else {
183 try {
184 const r = await get(`${finalOrigin}${path}`, timeoutMs, 1000000);
185 const html = looksLikeHtml(r.text, r.headers.get('content-type') || '');
186 const metas = html ? metaRobots(r.text) : [];
187 const xr = r.headers.get('x-robots-tag');
188 const d = directives(metas, xr);
189 row.homepage = { fetched: true, url: r.url, httpStatus: r.status, metaRobots: metas, xRobotsTag: xr ? xr.slice(0, 300) : null, noindex: d.noindex, noai: d.noai, noimageai: d.noimageai,
190 note: r.status === 403 || r.status === 429 ? `The page answered HTTP ${r.status} to our checker; a firewall or bot protection may also block AI crawlers here even though robots.txt allows them.` : null };
191 } catch (e) { row.homepage = { fetched: false, note: `Could not fetch the page: ${e.message}.` }; }
192 }
193 } else row.homepage = null;
194 row.checkedAt = now();
195 return row;
196 }
197
198 async function save(row) {
199 counts[row.status] = (counts[row.status] ?? 0) + 1;
200 if (row.status !== 'checked') { await Actor.pushData(row); return; }
201 if (chargeLimitHit) { counts.checked--; skipped.push(row.domain); return; }
202 if (charging) {
203 const allowed = charging.calculateMaxEventChargeCountWithinLimit(EVENT);
204 if (Number.isFinite(allowed) && allowed < 1) { chargeLimitHit = true; counts.checked--; skipped.push(row.domain); log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.'); return; }
205 }
206 verdicts[row.aiAccessVerdict ?? 'custom_only'] = (verdicts[row.aiAccessVerdict ?? 'custom_only'] ?? 0) + 1;
207 const r = await Actor.pushData(row, EVENT);
208 if (r?.eventChargeLimitReached) chargeLimitHit = true;
209 }
210
211 const conc = Math.max(1, Math.min(30, Math.floor(Number(concurrency) || 10)));
212 let idx = 0; let done = 0;
213 const worker = async () => {
214 while (idx < targets.length && !chargeLimitHit) {
215 const t = targets[idx++];
216 let row;
217 try { row = await check(t); } catch (e) { row = { domain: t.host, input: t.input, status: 'unreachable', error: `${e.message}. Not charged.`, checkedAt: new Date().toISOString() }; }
218 await save(row);
219 if (++done % 50 === 0) log.info(`Checked ${done}/${targets.length} domains`);
220 }
221 };
222 await Promise.all(Array.from({ length: Math.min(conc, targets.length) }, worker));
223
224 for (const t of targets.slice(idx)) skipped.push(t.host);
225 for (const d of skipped) await Actor.pushData({ domain: d, status: 'skipped_charge_limit', error: 'Not checked: the run reached the maximum cost you set. Raise the limit and run these again. Not charged.', checkedAt: new Date().toISOString() });
226 if (skipped.length) counts.skipped_charge_limit = skipped.length;
227 for (const v of invalid) await Actor.pushData({ domain: null, input: v, status: 'invalid_input', error: 'Not a valid domain or http(s) website URL. Not charged.', checkedAt: new Date().toISOString() });
228 if (invalid.length) counts.invalid_input = invalid.length;
229
230 const checked = counts.checked ?? 0;
231 const dsOut = Actor.getEnv().defaultDatasetId;
232 const summary = {
233 domainsRequested: targets.length + overCap.length + invalid.length, domainsChecked: checked, byStatus: counts, byVerdict: verdicts,
234 mostBlockedBots: Object.entries(blockedTally).sort((a, b) => b[1] - a[1]).slice(0, 10).map(([bot, n]) => ({ bot, domains: n })),
235 llmsTxtFound: checkLlmsTxt ? llmsFound : null, botsChecked: checkList.map((b) => b.bot), checkedPath: path,
236 notCheckedOverMaxDomains: overCap.length, chargeLimitReached: chargeLimitHit, sourceDatasetId: dsId || null,
237 zeroResultsReason: checked ? null : `No domain could be checked: ${Object.entries(counts).map(([k, n]) => `${n} ${k}`).join(', ') || 'no valid input'}. See the error field of each row.`,
238 finishedAt: new Date().toISOString(),
239 };
240 if (checked) summary.nextSteps = [
241 'See whether ChatGPT actually mentions and cites these brands: run swillz/ai-brand-visibility-tracker with {"brand": "<brand>", "domain": "<domain>"}.',
242 `Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${dsOut}"}.`,
243 'List every page an AI crawler can reach: run swillz/sitemap-url-extractor with {"startUrls": [{"url": "https://<domain>"}]}.',
244 ];
245 await Actor.setValue('OUTPUT', summary);
246 if (targets.length >= 3 && !checked && (counts.unreachable ?? 0) === targets.length) throw new Error(`Could not reach any of the ${targets.length} sites. This looks like a network problem; try again.`);
247 const freeParts = Object.entries(counts).filter(([k, n]) => k !== 'checked' && n).map(([k, n]) => `${n} ${k}`);
248 await Actor.exit(`${checked ? '' : '0 results: '}Checked ${checked} of ${targets.length} ${targets.length === 1 ? 'site' : 'sites'} (${Object.entries(verdicts).map(([k, n]) => `${n} ${k}`).join(', ') || 'none'})${freeParts.length ? `; ${freeParts.join(', ')} not charged` : ''}.`);
249} catch (e) {
250 if (e instanceof InputError || e?.isInputError) { log.error(e.message); await Actor.fail(`Invalid input: ${e.message}`); }
251 else { log.exception(e, 'Run failed'); await Actor.fail(e.message); }
252}
