AI Crawler Access Checker
Pricing
$0.05 / site-audit
AI assistants now send their own crawlers (GPTBot, ClaudeBot, PerplexityBot, Google-Extended…). If robots.txt blocks them — or an old noai meta tag is still in your templates — your site is invisible in AI answers and nothing in Google Search Console will ever tell you.
{ "title": "Input schema for AI Crawler Access Checker", "type": "object", "schemaVersion": 1, "properties": { "websiteUrls": { "title": "Website URLs", "type": "array", "editor": "stringList", "description": "One or more website origins to audit, e.g. https://example.com. Multiple URLs are audited in parallel.", "minItems": 1, "maxItems": 50, "prefill": [ "https://www.example.com" ] }, "bots": { "title": "AI crawlers to check", "type": "array", "editor": "stringList", "description": "robots.txt user-agent tokens to evaluate. Defaults cover the major AI crawlers as of November 2026. You can add or remove tokens freely.", "prefill": [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "Claude-SearchBot", "PerplexityBot", "Perplexity-User", "Google-Extended", "Applebot-Extended", "meta-externalagent", "Bytespider", "Amazonbot", "CCBot", "cohere-ai" ] }, "checkLlmsTxt": { "title": "Check llms.txt", "type": "boolean", "editor": "checkbox", "description": "Also probe /llms.txt and /llms-full.txt — the emerging standard that helps AI models index your site better.", "default": true }, "checkHtmlSignals": { "title": "Check HTML AI opt-out signals", "type": "boolean", "editor": "checkbox", "description": "Inspect the homepage for AI opt-out meta tags (noai, noimageai, googlebot) and X-Robots-Tag response headers.", "default": true }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "type": "integer", "editor": "number", "description": "Per-request timeout in seconds.", "default": 15, "minimum": 5, "maximum": 60 }, "generateMarkdownReport": { "title": "Generate markdown report", "type": "boolean", "editor": "checkbox", "description": "Save a human-readable report.md to the key-value store, ready to share with clients or colleagues.", "default": true } }, "required": [ "websiteUrls" ]}{ "actorSpecification": 1.0, "name": "ai-crawler-audit", "title": "AI Crawler Access & GPTBot Checker", "description": "See your website the way AI does. Checks which AI crawlers (GPTBot, ClaudeBot, PerplexityBot, Google-Extended, ...) your robots.txt allows, whether llms.txt exists, and whether AI opt-out meta tags or headers are present. Returns a per-bot report, an AI Visibility Score and a prioritized fix list.", "version": "0.1.0", "buildTag": "latest", "input": "./INPUT_SCHEMA.json", "output": "./output_schema.json", "dockerfile": "./Dockerfile", "minMemoryMbytes": 256, "maxMemoryMbytes": 512}{ "actorOutputSchemaVersion": 1, "title": "Output schema of AI Crawler Access Checker", "description": "Per-site AI crawler audit results and a shareable markdown report.", "type": "object", "properties": { "siteAudits": { "type": "string", "title": "Site audits", "description": "One audit item per website: per-bot access matrix, llms.txt status, AI visibility score and prioritized recommendations.", "template": "{{links.apiDefaultDatasetUrl}}/items" }, "markdownReport": { "type": "string", "title": "Markdown report", "description": "Human-readable report stored as report.md in the default key-value store.", "template": "{{links.apiDefaultKeyValueStoreUrl}}/records/report.md" } }}1
1"""Core audit logic for the AI Crawler Access Checker Actor.2
3This module is intentionally free of Apify SDK imports so the audit engine4can be unit-tested and reused locally without the Apify platform.5"""6
7from __future__ import annotations8
9import re10from dataclasses import dataclass, field11from datetime import datetime, timezone12from typing import Any13
14import httpx15from bs4 import BeautifulSoup16
17AUDIT_VERSION = "0.1.0"18BROWSER_UA = "Mozilla/5.0 (compatible; AICrawlerAudit/0.1)"19MAX_ROBOTS_BYTES = 200_00020
21# Knowledge base: the major AI crawlers as of November 2026.22# Categories are used for scoring: search > chat > training > aggregator.23DEFAULT_BOTS = [24 "GPTBot",25 "OAI-SearchBot",26 "ChatGPT-User",27 "ClaudeBot",28 "Claude-User",29 "Claude-SearchBot",30 "PerplexityBot",31 "Perplexity-User",32 "Google-Extended",33 "Applebot-Extended",34 "meta-externalagent",35 "Bytespider",36 "Amazonbot",37 "CCBot",38 "cohere-ai",39]40
41BOT_DB: dict[str, dict[str, str]] = {42 "GPTBot": {"operator": "OpenAI", "product": "ChatGPT model training", "category": "training"},43 "OAI-SearchBot": {"operator": "OpenAI", "product": "ChatGPT search index", "category": "search"},44 "ChatGPT-User": {"operator": "OpenAI", "product": "Live browsing requested by ChatGPT users", "category": "chat"},45 "ClaudeBot": {"operator": "Anthropic", "product": "Claude model training", "category": "training"},46 "Claude-User": {"operator": "Anthropic", "product": "Live browsing requested by Claude users", "category": "chat"},47 "Claude-SearchBot": {"operator": "Anthropic", "product": "Claude search index", "category": "search"},48 "PerplexityBot": {"operator": "Perplexity", "product": "Perplexity answer-engine index", "category": "search"},49 "Perplexity-User": {"operator": "Perplexity", "product": "Live browsing requested by Perplexity users", "category": "chat"},50 "Google-Extended": {"operator": "Google", "product": "Gemini model training (does not affect Google Search)", "category": "training"},51 "Applebot-Extended": {"operator": "Apple", "product": "Apple Intelligence model training", "category": "training"},52 "meta-externalagent": {"operator": "Meta", "product": "Meta AI model training", "category": "training"},53 "Bytespider": {"operator": "ByteDance", "product": "ByteDance AI data collection", "category": "training"},54 "Amazonbot": {"operator": "Amazon", "product": "Amazon AI data collection (Alexa, Rufus)", "category": "training"},55 "CCBot": {"operator": "Common Crawl", "product": "Open web corpus reused by many model trainers", "category": "aggregator"},56 "cohere-ai": {"operator": "Cohere", "product": "Cohere model training", "category": "training"},57}58
59BLOCK_PENALTY = {"search": 20, "chat": 10, "training": 8, "aggregator": 5}60
61AI_OPT_OUT_TOKENS = ("noai", "noimageai", "noai-image", "noai_textdataai")62
63
64# ---------------------------------------------------------------------------65# robots.txt parsing (RFC 9309 flavour: groups, wildcards, $ anchor)66# ---------------------------------------------------------------------------67
68@dataclass69class RobotsRule:70 allow: bool71 pattern: str72
73
74@dataclass75class RobotsGroup:76 agents: list[str] = field(default_factory=list)77 allow: list[RobotsRule] = field(default_factory=list)78 disallow: list[RobotsRule] = field(default_factory=list)79 crawl_delay: float | None = None80
81
82def parse_robots(text: str) -> list[RobotsGroup]:83 """Parse robots.txt into user-agent groups.84
85 Consecutive ``User-agent`` lines merge into one group. The first86 directive after a group closes it, per RFC 9309 grouping.87 """88 groups: list[RobotsGroup] = []89 current: RobotsGroup | None = None90 last_line_was_agent = False91
92 for raw in text.splitlines():93 line = raw.split("#", 1)[0].strip()94 if not line or ":" not in line:95 continue96 key, _, value = line.partition(":")97 key = key.strip().lower()98 value = value.strip()99
100 if key == "user-agent":101 if current is None or not last_line_was_agent:102 current = RobotsGroup()103 groups.append(current)104 current.agents.append(value)105 last_line_was_agent = True106 continue107
108 last_line_was_agent = False109 if current is None:110 current = RobotsGroup()111 groups.append(current)112 if key == "disallow" and value:113 current.disallow.append(RobotsRule(allow=False, pattern=value))114 elif key == "allow" and value:115 current.allow.append(RobotsRule(allow=True, pattern=value))116 elif key == "crawl-delay":117 try:118 current.crawl_delay = float(value)119 except ValueError:120 pass121 return groups122
123
124def _pattern_to_regex(pattern: str) -> re.Pattern[str]:125 """Translate a robots.txt path pattern into a prefix-matching regex."""126 anchored = pattern.endswith("$")127 body = pattern[:-1] if anchored else pattern128 parts = body.split("*")129 rx = "^" + ".*".join(re.escape(p) for p in parts)130 if anchored:131 rx += "$"132 return re.compile(rx)133
134
135def _pattern_specificity(pattern: str) -> int:136 """Specificity = length of the pattern excluding wildcard characters."""137 return len(pattern.replace("*", "").rstrip("$"))138
139
140def permission_for(groups: list[RobotsGroup], bot: str, path: str = "/") -> dict[str, Any]:141 """Decide whether ``bot`` may fetch ``path`` according to ``groups``.142
143 Returns a dict with ``allowed`` (bool), ``matched_rule`` and ``group_source``144 so the caller can explain *why* in the report.145 """146 chosen: RobotsGroup | None = None147 source = "none"148 for g in groups:149 if any(a.lower() == bot.lower() for a in g.agents):150 chosen = g151 source = "dedicated group for this bot"152 break153 if chosen is None:154 for g in groups:155 if "*" in g.agents:156 chosen = g157 source = "wildcard group (User-agent: *)"158 break159
160 if chosen is None:161 return {"allowed": True, "matched_rule": None, "group_source": "no rules found — allowed by default"}162
163 best: tuple[int, bool, str] | None = None164 for rule in chosen.allow + chosen.disallow:165 if not _pattern_to_regex(rule.pattern).match(path):166 continue167 specificity = _pattern_specificity(rule.pattern)168 if (169 best is None170 or specificity > best[0]171 or (specificity == best[0] and rule.allow and not best[1])172 ):173 best = (specificity, rule.allow, rule.pattern)174
175 if best is None:176 return {"allowed": True, "matched_rule": None, "group_source": f"{source}; no matching rule — allowed by default"}177 kind = "Allow" if best[1] else "Disallow"178 return {"allowed": best[1], "matched_rule": f"{kind}: {best[2]}", "group_source": source}179
180
181# ---------------------------------------------------------------------------182# Audit model183# ---------------------------------------------------------------------------184
185@dataclass186class AuditConfig:187 bots: list[str] = field(default_factory=lambda: list(DEFAULT_BOTS))188 check_llms: bool = True189 check_html: bool = True190 timeout: int = 15191 include_markdown: bool = True192
193
194@dataclass195class SiteAudit:196 url: str197 fetched_at: str198 robots_status: str = "unknown"199 robots_raw: str = ""200 robots_groups: list[RobotsGroup] = field(default_factory=list)201 sitemaps: list[str] = field(default_factory=list)202 bot_results: list[dict[str, Any]] = field(default_factory=list)203 llms_txt: str = "not_checked"204 llms_full_txt: str = "not_checked"205 llms_txt_title: str | None = None206 meta_robots: str | None = None207 ai_opt_out_tags: list[str] = field(default_factory=list)208 x_robots_tag: str | None = None209 score: int = 100210 verdict: str = ""211 recommendations: list[dict[str, str]] = field(default_factory=list)212 notes: list[str] = field(default_factory=list)213
214 def blocked_bots(self) -> list[str]:215 return [b["bot"] for b in self.bot_results if not b["allowed"]]216
217 def summary_item(self) -> dict[str, Any]:218 return {219 "url": self.url,220 "auditedAt": self.fetched_at,221 "aiVisibilityScore": self.score,222 "verdict": self.verdict,223 "robotsTxt": self.robots_status,224 "llmsTxt": self.llms_txt,225 "llmsFullTxt": self.llms_full_txt,226 "aiOptOutTags": self.ai_opt_out_tags,227 "xRobotsTag": self.x_robots_tag,228 "blockedBots": self.blocked_bots(),229 "botResults": self.bot_results,230 "recommendations": self.recommendations,231 "notes": self.notes,232 "auditVersion": AUDIT_VERSION,233 }234
235
236# ---------------------------------------------------------------------------237# Fetching238# ---------------------------------------------------------------------------239
240def _normalize(url: str) -> str:241 url = url.strip()242 if not re.match(r"^https?://", url, re.I):243 url = "https://" + url244 return url.rstrip("/")245
246
247async def _get(client: httpx.AsyncClient, url: str) -> httpx.Response:248 return await client.get(url)249
250
251# ---------------------------------------------------------------------------252# Main audit routine253# ---------------------------------------------------------------------------254
255async def audit_site(url: str, config: AuditConfig) -> SiteAudit:256 audit = SiteAudit(url=_normalize(url), fetched_at=datetime.now(timezone.utc).isoformat())257 origin = audit.url258 headers = {"User-Agent": BROWSER_UA, "Accept-Language": "en"}259
260 async with httpx.AsyncClient(261 headers=headers,262 timeout=config.timeout,263 follow_redirects=True,264 ) as client:265 # --- robots.txt -----------------------------------------------------266 try:267 resp = await _get(client, f"{origin}/robots.txt")268 if resp.status_code == 200 and resp.text.strip():269 audit.robots_status = "found"270 audit.robots_raw = resp.text[:MAX_ROBOTS_BYTES]271 audit.robots_groups = parse_robots(audit.robots_raw)272 audit.sitemaps = [273 v for line in resp.text.splitlines()274 if (lv := line.split(":", 1)) and lv[0].strip().lower() == "sitemap"275 for v in [lv[1].strip()] if v276 ]277 elif resp.status_code == 404:278 audit.robots_status = "missing"279 else:280 audit.robots_status = f"http_{resp.status_code}"281 except httpx.HTTPError as exc:282 audit.robots_status = "error"283 audit.notes.append(f"robots.txt fetch failed: {exc!r}")284
285 # --- llms.txt -------------------------------------------------------286 if config.check_llms:287 for key, path in (("llms_txt", "/llms.txt"), ("llms_full_txt", "/llms-full.txt")):288 try:289 resp = await _get(client, f"{origin}{path}")290 if resp.status_code == 200 and resp.text.strip():291 setattr(audit, key, "found")292 if key == "llms_txt":293 for line in resp.text.splitlines():294 if line.startswith("#"):295 audit.llms_txt_title = line.lstrip("# ").strip()296 break297 else:298 setattr(audit, key, "missing")299 except httpx.HTTPError:300 setattr(audit, key, "error")301
302 # --- homepage + HTML signals ----------------------------------------303 home_status = 0304 try:305 resp = await _get(client, origin)306 home_status = resp.status_code307 if config.check_html and resp.status_code == 200:308 soup = BeautifulSoup(resp.text, "html.parser")309 metas = []310 for tag in soup.find_all("meta"):311 name = (tag.get("name") or "").strip().lower()312 if name in ("robots", "googlebot", "googlebot-news", "bingbot"):313 metas.append((tag.get("content") or "").strip())314 audit.meta_robots = ", ".join(m for m in metas if m) or None315 xrt = resp.headers.get("x-robots-tag")316 audit.x_robots_tag = xrt317 haystack = " ".join(metas).lower() + " " + (xrt or "").lower()318 audit.ai_opt_out_tags = [t for t in AI_OPT_OUT_TOKENS if t in haystack]319 if any("noindex" in m.lower() for m in metas):320 audit.notes.append("Homepage carries a noindex directive — invisible to both search and AI systems.")321 except httpx.HTTPError as exc:322 audit.notes.append(f"homepage fetch failed: {exc!r}")323
324 # --- per-bot permissions ------------------------------------------------325 for bot in config.bots:326 info = BOT_DB.get(bot, {"operator": "Unknown", "product": "Unknown AI agent", "category": "other"})327 if audit.robots_status == "found":328 perm = permission_for(audit.robots_groups, bot, "/")329 else:330 perm = {"allowed": True, "matched_rule": None, "group_source": "no robots.txt — allowed by default"}331 audit.bot_results.append({332 "bot": bot,333 "operator": info["operator"],334 "product": info["product"],335 "category": info["category"],336 "allowed": perm["allowed"],337 "matched_rule": perm["matched_rule"],338 "group_source": perm["group_source"],339 })340
341 _score(audit)342 _recommend(audit, home_status)343 return audit344
345
346# ---------------------------------------------------------------------------347# Scoring & recommendations348# ---------------------------------------------------------------------------349
350def _score(audit: SiteAudit) -> None:351 score = 100.0352 for b in audit.bot_results:353 if not b["allowed"]:354 score -= BLOCK_PENALTY.get(b["category"], 5)355 if audit.ai_opt_out_tags:356 score -= 15357 if audit.llms_txt == "missing":358 score -= 10359 audit.score = int(max(0, min(100, score)))360 if audit.score >= 85:361 audit.verdict = "Open to AI"362 elif audit.score >= 60:363 audit.verdict = "Partially open"364 elif audit.score >= 30:365 audit.verdict = "Mostly blocked"366 else:367 audit.verdict = "Closed to AI"368
369
370def _recommend(audit: SiteAudit, home_status: int) -> None:371 recs: list[dict[str, str]] = []372
373 if audit.robots_status == "missing":374 recs.append({375 "issue": "No robots.txt found",376 "impact": "AI crawlers are allowed by default, but you have no explicit control.",377 "how_to_fix": "Publish a robots.txt at the site root. Start from the official bot lists of OpenAI, Anthropic and Perplexity.",378 })379 elif audit.robots_status != "found":380 recs.append({381 "issue": f"robots.txt returned '{audit.robots_status}'",382 "impact": "AI crawlers may be blocked by accident if your server rejects unknown agents.",383 "how_to_fix": "Make sure /robots.txt returns HTTP 200 for every user agent.",384 })385
386 blocked = [b for b in audit.bot_results if not b["allowed"]]387 for b in blocked:388 recs.append({389 "issue": f"{b['bot']} is blocked ({b['matched_rule']})",390 "impact": f"{b['operator']} uses this bot for: {b['product']}.",391 "how_to_fix": "If this is unintended, remove the matching Disallow line from the bot's group in robots.txt.",392 })393
394 if audit.llms_txt == "missing":395 recs.append({396 "issue": "No /llms.txt",397 "impact": "AI models get no curated map of your site and may index it poorly.",398 "how_to_fix": "Add an llms.txt file at the site root describing your key pages in Markdown.",399 })400 if audit.ai_opt_out_tags:401 recs.append({402 "issue": f"AI opt-out signals present: {', '.join(audit.ai_opt_out_tags)}",403 "impact": "You are explicitly telling AI systems not to use your content.",404 "how_to_fix": "Remove the noai/noimageai meta tags or X-Robots-Tag header if you want AI visibility.",405 })406 if home_status and home_status >= 400:407 recs.append({408 "issue": f"Homepage returned HTTP {home_status}",409 "impact": "Neither users nor AI agents can see your site.",410 "how_to_fix": "Fix the server error on the homepage.",411 })412 if not recs:413 recs.append({414 "issue": "No blocking issues detected",415 "impact": "Major AI crawlers can access your homepage and llms.txt is present.",416 "how_to_fix": "Re-run this audit periodically — bot lists change often.",417 })418 audit.recommendations = recs419
420
421# ---------------------------------------------------------------------------422# Markdown report423# ---------------------------------------------------------------------------424
425def render_report(audits: list[tuple[str, SiteAudit]]) -> str:426 lines: list[str] = [427 "# AI Crawler Access Report",428 "",429 f"_Generated {datetime.now(timezone.utc).strftime('%Y-%m-%d %H:%M UTC')} by AI Crawler Access Checker v{AUDIT_VERSION}_",430 "",431 ]432 for _, a in audits:433 lines += [f"## {a.url}", "", f"**AI Visibility Score: {a.score}/100 — {a.verdict}**", ""]434 lines += [435 "| AI crawler | Operator | Purpose | Access to / | Matched rule |",436 "|---|---|---|---|---|",437 ]438 for b in a.bot_results:439 access = "✅ allowed" if b["allowed"] else "⛔ blocked"440 rule = b["matched_rule"] or "—"441 lines.append(f"| {b['bot']} | {b['operator']} | {b['product']} | {access} | {rule} |")442 lines += [443 "",444 f"- robots.txt: **{a.robots_status}**"445 + (f" · sitemaps: {', '.join(a.sitemaps)}" if a.sitemaps else ""),446 f"- llms.txt: **{a.llms_txt}**" + (f" — “{a.llms_txt_title}”" if a.llms_txt_title else ""),447 f"- llms-full.txt: **{a.llms_full_txt}**",448 f"- AI opt-out signals: **{', '.join(a.ai_opt_out_tags) if a.ai_opt_out_tags else 'none'}**",449 "",450 "### Recommendations",451 "",452 ]453 for r in a.recommendations:454 lines += [f"- **{r['issue']}** — {r['impact']}", f" - Fix: {r['how_to_fix']}"]455 if a.notes:456 lines += ["", "### Notes", ""]457 lines += [f"- {n}" for n in a.notes]458 lines.append("")459 return "\n".join(lines)1"""AI Crawler Access & GPTBot Checker — Apify Actor entry point."""2
3from __future__ import annotations4
5import asyncio6
7from apify import Actor8
9from .audit import AuditConfig, DEFAULT_BOTS, audit_site, render_report10
11MAX_CONCURRENCY = 512
13
14async def main() -> None:15 async with Actor:16 inp = await Actor.get_input() or {}17
18 urls = [str(u).strip() for u in (inp.get("websiteUrls") or []) if str(u).strip()]19 if not urls:20 raise RuntimeError("Input 'websiteUrls' must contain at least one URL.")21
22 config = AuditConfig(23 bots=[str(b).strip() for b in (inp.get("bots") or DEFAULT_BOTS) if str(b).strip()],24 check_llms=bool(inp.get("checkLlmsTxt", True)),25 check_html=bool(inp.get("checkHtmlSignals", True)),26 timeout=int(inp.get("requestTimeoutSecs", 15)),27 include_markdown=bool(inp.get("generateMarkdownReport", True)),28 )29
30 Actor.log.info(f"Auditing {len(urls)} site(s) against {len(config.bots)} AI crawlers ...")31
32 semaphore = asyncio.Semaphore(MAX_CONCURRENCY)33
34 async def run_one(url: str):35 async with semaphore:36 Actor.log.info(f"Auditing {url} ...")37 return await audit_site(url, config)38
39 results = await asyncio.gather(*(run_one(u) for u in urls), return_exceptions=True)40
41 audits = []42 for url, result in zip(urls, results):43 if isinstance(result, Exception):44 Actor.log.error(f"Audit failed for {url}: {result!r}")45 await Actor.push_data({"url": url, "status": "error", "error": repr(result)})46 continue47 audits.append((url, result))48 await Actor.push_data(result.summary_item())49
50 if config.include_markdown and audits:51 kvs = await Actor.open_key_value_store()52 await kvs.set_value("report.md", render_report(audits), "text/markdown")53 Actor.log.info("Markdown report saved to key 'report.md' in the default key-value store.")54
55 Actor.log.info(f"Done — {len(audits)}/{len(urls)} site(s) audited successfully.")56
57
58if __name__ == "__main__":59 asyncio.run(main())FROM apify/actor-python:3.13
# Verified in a clean environment (2026-11-01): apify 4.2.0 + crawlee 1.11.0# + pydantic 2.14.2 + httpx 0.28.1 + bs4 4.15.0 - clean install, full import, local E2E run OK.# httpx is installed EXPLICITLY: apify 4.x does NOT pull it on a clean base image.RUN pip install --no-cache-dir "apify>=4,<5" "httpx>=0.27,<1" "beautifulsoup4>=4.12,<5"
# Copy the Actor source codeCOPY . /usr/src/app
CMD python3 -m src.main1apify>=2.0,<32httpx>=0.27,<13beautifulsoup4>=4.12,<5