Tech Stack & Email Provider Detector
Pricing
from $3.00 / 1,000 domain checkeds
Find the CMS, ecommerce platform, analytics, marketing tools, payment providers and email provider (Google Workspace, Microsoft 365...) behind any list of websites. Clean table output. $3 per 1,000 websites, only pay for sites analyzed.
{ "actorSpecification": 1, "name": "tech-stack-email-provider-detector", "title": "Tech Stack & Email Provider Detector", "description": "Detect the CMS, ecommerce platform, analytics, marketing, payment, framework, CDN and email provider behind any list of domains. Public pages only, robots.txt respected, no personal data.", "version": "0.0", "buildTag": "latest", "meta": { "templateId": "python-empty" }, "input": "./input_schema.json", "output": "./output_schema.json", "dockerfile": "../Dockerfile", "storages": { "dataset": { "actorSpecification": 1, "title": "Tech stack results", "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "domain", "status", "finalUrl", "flags", "emailProvider", "spfServices", "technologies", "pagesFetched", "changes", "error", "checkedAt" ] }, "display": { "component": "table", "properties": { "domain": { "label": "Domain", "format": "text" }, "status": { "label": "Status", "format": "text" }, "finalUrl": { "label": "Final URL", "format": "link" }, "flags": { "label": "Flags", "format": "object" }, "emailProvider": { "label": "Email provider", "format": "text" }, "spfServices": { "label": "SPF services", "format": "array" }, "technologies": { "label": "Technologies", "format": "array" }, "pagesFetched": { "label": "Pages", "format": "number" }, "changes": { "label": "Changes", "format": "object" }, "error": { "label": "Error", "format": "text" }, "checkedAt": { "label": "Checked at", "format": "date" } } } } } } }}{ "$schema": "https://apify.com/schemas/v1/dataset.ide.json", "actorSpecification": 1, "fields": {}, "views": {}}{ "title": "Tech Stack & Email Provider Detector input", "type": "object", "schemaVersion": 1, "properties": { "domains": { "title": "Domains", "type": "array", "description": "Domains or URLs to analyse, e.g. example.com or https://www.example.com. Paths are ignored; duplicates are removed.", "editor": "stringList", "prefill": ["apify.com", "shopify.com"] }, "followLinks": { "title": "Extra same-site pages to fetch", "type": "integer", "description": "Besides the homepage, fetch up to this many same-site links (prefers /pricing, /about, /shop ...). Hard cap: 3 pages per domain.", "minimum": 0, "maximum": 2, "default": 1 }, "checkDns": { "title": "Check DNS (email provider, SPF, verifications)", "type": "boolean", "description": "Look up MX and TXT records to identify the email provider, SPF-authorised sending services and domain-verification tokens (only the service name is reported).", "default": true }, "maxConcurrency": { "title": "Max concurrency", "type": "integer", "description": "How many domains are processed in parallel. Pages within one domain are always fetched sequentially.", "minimum": 1, "maximum": 50, "default": 10 }, "requestTimeoutSecs": { "title": "Request timeout (seconds)", "type": "integer", "description": "Timeout for each HTTP request and DNS lookup.", "minimum": 1, "maximum": 120, "default": 15, "unit": "seconds" }, "changedOnly": { "title": "Output only changed domains", "type": "boolean", "description": "Compare with the previous run's results (stored in the named key-value store 'fieldfeed-techstack-state') and output only domains whose technology set, email provider or SPF services changed, with added/removed lists. Domains seen for the first time count as changed.", "default": false } }, "required": ["domains"]}{ "$schema": "https://apify.com/schemas/v1/output.ide.json", "actorOutputSchemaVersion": 1, "title": "Output schema", "properties": { "results": { "type": "string", "title": "Results", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1
1import asyncio2
3from .main import main4
5if __name__ == '__main__':6 asyncio.run(main())1"""Pure, offline-testable detection logic (stdlib only, no network, no Apify imports)."""2
3from __future__ import annotations4
5import re6from dataclasses import dataclass, field7from html.parser import HTMLParser8from typing import Iterable, Mapping, Sequence9from urllib.parse import urljoin, urlsplit, urlunsplit10
11from . import fingerprints as fp12
13MAX_HTML_BYTES = 2_000_000 # only the first ~2 MB of a page is scanned14EVIDENCE_MAX_LEN = 12015
16# ----------------------------------------------------------------------------- helpers17
18_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")19_PHONE_RE = re.compile(r"(?:\+\d{1,3}[\s.-]?)?\(?\b\d{3}\)?[\s-]\d{3}[\s-]\d{4}\b")20_DOMAIN_RE = re.compile(r"^(?=.{1,253}$)(?!-)[a-z0-9-]{1,63}(?<!-)(?:\.(?!-)[a-z0-9-]{1,63}(?<!-))+$")21
22
23def scrub(text: str) -> str:24 """Remove anything that looks like an email address or phone number and truncate."""25 text = _EMAIL_RE.sub("[redacted]", text)26 text = _PHONE_RE.sub("[redacted]", text)27 text = re.sub(r"\s+", " ", text).strip()28 if len(text) > EVIDENCE_MAX_LEN:29 text = text[: EVIDENCE_MAX_LEN - 1] + "…"30 return text31
32
33def normalize_domain(raw: str) -> str | None:34 """Turn 'https://Www.Example.com/path' or 'example.com.' into 'www.example.com'.35
36 Returns None for anything that is not a plausible public hostname.37 """38 if not isinstance(raw, str):39 return None40 s = raw.strip().lower()41 if not s:42 return None43 if "://" not in s:44 s = "http://" + s45 try:46 host = urlsplit(s).hostname or ""47 except ValueError:48 return None49 host = host.rstrip(".")50 try:51 host = host.encode("idna").decode("ascii")52 except UnicodeError:53 return None54 if not _DOMAIN_RE.match(host):55 return None56 return host57
58
59def normalize_headers(headers) -> list[tuple[str, str]]:60 """Accept a dict, a list of pairs, or an object with .multi_items()/.items(); return lowercase pairs."""61 if headers is None:62 return []63 if hasattr(headers, "multi_items"):64 items = headers.multi_items()65 elif isinstance(headers, Mapping):66 items = []67 for k, v in headers.items():68 if isinstance(v, (list, tuple)):69 items.extend((k, x) for x in v)70 else:71 items.append((k, v))72 else:73 items = list(headers)74 return [(str(k).lower(), str(v)) for k, v in items]75
76
77def cookie_names(set_cookie_values: Iterable[str]) -> list[str]:78 """Extract only cookie *names* from Set-Cookie header values. Values are discarded."""79 names: list[str] = []80 for raw in set_cookie_values:81 first = raw.split(";", 1)[0]82 if "=" in first:83 name = first.split("=", 1)[0].strip()84 if name and name not in names:85 names.append(name)86 return names87
88
89# ----------------------------------------------------------------------------- HTML parsing90
91
92@dataclass93class PageSignals:94 script_srcs: list[str] = field(default_factory=list)95 link_hrefs: list[str] = field(default_factory=list)96 anchor_hrefs: list[str] = field(default_factory=list)97 metas: dict[str, list[str]] = field(default_factory=dict)98
99
100class _SignalParser(HTMLParser):101 def __init__(self) -> None:102 super().__init__(convert_charrefs=True)103 self.signals = PageSignals()104
105 def handle_starttag(self, tag, attrs): # noqa: D401 - HTMLParser API106 a = {k.lower(): (v or "") for k, v in attrs if k}107 if tag == "script" and a.get("src"):108 self.signals.script_srcs.append(a["src"].strip())109 elif tag == "link" and a.get("href"):110 self.signals.link_hrefs.append(a["href"].strip())111 elif tag == "a" and a.get("href"):112 self.signals.anchor_hrefs.append(a["href"].strip())113 elif tag == "meta":114 name = (a.get("name") or a.get("property") or "").lower()115 if name and "content" in a:116 self.signals.metas.setdefault(name, []).append(a["content"].strip())117
118 handle_startendtag = handle_starttag119
120
121def parse_html(html: str) -> PageSignals:122 parser = _SignalParser()123 try:124 parser.feed(html[:MAX_HTML_BYTES])125 parser.close()126 except Exception: # malformed markup should never crash detection127 pass128 return parser.signals129
130
131# ----------------------------------------------------------------------------- detection132
133
134@dataclass135class Page:136 url: str137 html: str = ""138 headers: list[tuple[str, str]] = field(default_factory=list)139 status: int | None = None140
141
142_COMPILED: list[dict] | None = None143
144
145def _compiled() -> list[dict]:146 global _COMPILED147 if _COMPILED is None:148 out = []149 for tech in fp.FINGERPRINTS:150 c = {"name": tech["name"], "category": tech["category"], "implies": tech.get("implies", [])}151 for kind in ("script", "link", "html", "cookie"):152 c[kind] = [(re.compile(p), conf) for p, conf in tech.get(kind, [])]153 for kind in ("meta", "header"):154 c[kind] = {155 k.lower(): [(re.compile(p), conf) for p, conf in rules]156 for k, rules in tech.get(kind, {}).items()157 }158 out.append(c)159 _COMPILED = out160 return _COMPILED161
162
163def combine_confidence(values: Iterable[int]) -> int:164 """Noisy-OR combination of independent signals, as an integer percentage."""165 miss = 1.0166 for v in values:167 miss *= 1.0 - max(0, min(100, v)) / 100.0168 return int(round((1.0 - miss) * 100))169
170
171def detect_page(page: Page, signals: PageSignals | None = None) -> dict[str, list[tuple[int, str]]]:172 """Return {tech name: [(confidence, evidence), ...]} for one page."""173 signals = signals or parse_html(page.html)174 html = page.html[:MAX_HTML_BYTES]175 headers = page.headers176 set_cookies = [v for k, v in headers if k == "set-cookie"]177 cookies = cookie_names(set_cookies)178 hits: dict[str, list[tuple[int, str]]] = {}179
180 def add(name: str, conf: int, ev: str) -> None:181 hits.setdefault(name, []).append((conf, scrub(ev)))182
183 for tech in _compiled():184 name = tech["name"]185 for rx, conf in tech["script"]:186 for src in signals.script_srcs:187 if rx.search(src):188 add(name, conf, f"script src: {src}")189 break190 for rx, conf in tech["link"]:191 for href in signals.link_hrefs:192 if rx.search(href):193 add(name, conf, f"link href: {href}")194 break195 for rx, conf in tech["html"]:196 m = rx.search(html)197 if m:198 add(name, conf, f"html: {m.group(0)}")199 for meta_name, rules in tech["meta"].items():200 for content in signals.metas.get(meta_name, []):201 for rx, conf in rules:202 if rx.search(content):203 add(name, conf, f"meta {meta_name}: {content}")204 for header_name, rules in tech["header"].items():205 for k, v in headers:206 if k != header_name:207 continue208 for rx, conf in rules:209 if rx.search(v):210 add(name, conf, f"header {k}: {v}")211 for rx, conf in tech["cookie"]:212 for cname in cookies:213 if rx.search(cname):214 add(name, conf, f"cookie name: {cname}")215 break216 return hits217
218
219def detect_technologies(pages: Sequence[Page], min_confidence: int = 30) -> list[dict]:220 """Detect technologies across pages; returns sorted list of {name, category, confidence, evidence}."""221 merged: dict[str, list[tuple[int, str]]] = {}222 for page in pages:223 for name, hits in detect_page(page).items():224 bucket = merged.setdefault(name, [])225 for h in hits:226 if h[1] not in {e for _, e in bucket}:227 bucket.append(h)228
229 by_name = {t["name"]: t for t in fp.FINGERPRINTS}230 results: dict[str, dict] = {}231 for name, hits in merged.items():232 conf = combine_confidence(c for c, _ in hits)233 if conf < min_confidence:234 continue235 hits_sorted = sorted(hits, key=lambda h: -h[0])236 results[name] = {237 "name": name,238 "category": by_name[name]["category"],239 "confidence": conf,240 "evidence": [e for _, e in hits_sorted[:5]],241 }242
243 # implied technologies (one level deep is enough for this set, but loop to be safe)244 changed = True245 while changed:246 changed = False247 for name in list(results):248 for implied in by_name[name].get("implies", []):249 parent_conf = results[name]["confidence"]250 if implied not in results:251 results[implied] = {252 "name": implied,253 "category": by_name[implied]["category"],254 "confidence": parent_conf,255 "evidence": [f"implied by {name}"],256 }257 changed = True258 elif results[implied]["confidence"] < parent_conf:259 results[implied]["confidence"] = parent_conf260 results[implied]["evidence"].append(f"implied by {name}")261 changed = True262
263 return sorted(results.values(), key=lambda t: (t["category"], -t["confidence"], t["name"]))264
265
266# ----------------------------------------------------------------------------- link following267
268_PRIORITY_PATHS = ["pricing", "plans", "about", "shop", "store", "products", "features", "solutions", "blog"]269_SKIP_PATH_RE = re.compile(270 r"(?i)(?:log-?in|log-?out|sign-?in|sign-?up|register|account|admin|wp-admin|wp-login|checkout|cart|"271 r"basket|password|auth|oauth|session|unsubscribe|privacy|terms|legal|cookie|search)"272)273_SKIP_EXT_RE = re.compile(r"(?i)\.(?:pdf|jpe?g|png|gif|svg|webp|zip|gz|mp4|mp3|xml|json|css|js|ico|docx?|xlsx?)$")274
275
276def _site_key(host: str) -> str:277 host = (host or "").lower()278 return host[4:] if host.startswith("www.") else host279
280
281def select_follow_links(base_url: str, hrefs: Iterable[str], limit: int) -> list[str]:282 """Pick up to limit same-site URLs, preferring /pricing, /about, etc. No query strings."""283 if limit <= 0:284 return []285 base = urlsplit(base_url)286 site = _site_key(base.hostname or "")287 candidates: dict[str, int] = {}288 for href in hrefs:289 if not href or href.startswith(("#", "mailto:", "tel:", "javascript:", "data:")):290 continue291 absu = urljoin(base_url, href)292 parts = urlsplit(absu)293 if parts.scheme not in ("http", "https") or _site_key(parts.hostname or "") != site:294 continue295 path = parts.path or "/"296 if path in ("", "/") or path.rstrip("/") == (base.path or "/").rstrip("/"):297 continue298 if _SKIP_PATH_RE.search(path) or _SKIP_EXT_RE.search(path) or parts.query:299 continue300 if path.count("/") > 3:301 continue302 clean = urlunsplit((base.scheme, base.netloc, path, "", ""))303 segs = [s for s in path.lower().split("/") if s]304 score = 100305 for i, key in enumerate(_PRIORITY_PATHS):306 if segs and segs[0].startswith(key):307 score = i308 break309 candidates[clean] = min(score, candidates.get(clean, 100))310 ordered = sorted(candidates.items(), key=lambda kv: (kv[1], len(kv[0]), kv[0]))311 # Only follow "interesting" pages unless nothing interesting exists.312 return [u for u, _ in ordered[:limit]]313
314
315# ----------------------------------------------------------------------------- DNS parsing316
317
318def _match_table(value: str, table: Sequence[tuple[str, str]]) -> str | None:319 for pattern, label in table:320 if re.search(pattern, value, re.IGNORECASE):321 return label322 return None323
324
325def email_provider_from_mx(mx_records: Iterable[tuple[int, str]]) -> str | None:326 """mx_records: (preference, exchange hostname). Lowest preference wins; returns provider name."""327 recs = sorted(((int(p), h.lower().rstrip(".")) for p, h in mx_records), key=lambda r: r[0])328 if not recs:329 return None330 if len(recs) == 1 and recs[0][1] in ("", "."):331 return None # null MX (RFC 7505): domain accepts no mail332 for _, host in recs:333 label = _match_table(host, fp.MX_PROVIDERS)334 if label:335 return label336 return "Self-hosted / other"337
338
339def spf_record(txt_records: Iterable[str]) -> str | None:340 for t in txt_records:341 if t.strip().lower().startswith("v=spf1"):342 return t.strip()343 return None344
345
346def spf_services(txt_records: Iterable[str]) -> list[str]:347 """Map SPF include:/redirect= targets to known sending services (deduplicated, ordered)."""348 rec = spf_record(txt_records)349 if not rec:350 return []351 out: list[str] = []352 for m in re.finditer(r"(?:include:|redirect=)([^\s]+)", rec, re.IGNORECASE):353 target = m.group(1).lower().rstrip(".")354 label = _match_table(target, fp.SPF_SERVICES)355 if label and label not in out:356 out.append(label)357 return out358
359
360def verification_services(txt_records: Iterable[str]) -> list[str]:361 """Report only the *service* behind verification TXT tokens, never the token itself."""362 out: list[str] = []363 for t in txt_records:364 label = _match_table(t.strip(), fp.TXT_VERIFICATIONS)365 if label and label not in out:366 out.append(label)367 return out368
369
370# ----------------------------------------------------------------------------- flags & diff371
372_ECOM_PLATFORMS = ["Shopify", "WooCommerce", "BigCommerce", "Magento", "PrestaShop", "Wix", "Squarespace"]373_CMS_ORDER = ["WordPress", "Shopify", "Wix", "Squarespace", "Webflow", "Drupal", "Joomla", "Ghost", "HubSpot CMS"]374
375
376def build_flags(technologies: Sequence[dict], email_provider: str | None) -> dict:377 names = {t["name"] for t in technologies}378 ecommerce = next((p for p in _ECOM_PLATFORMS[:5] if p in names), None)379 cms = next((c for c in _CMS_ORDER if c in names), None)380 return {381 "is_shopify": "Shopify" in names,382 "ecommerce_platform": ecommerce,383 "cms": cms,384 "email_provider": email_provider,385 "uses_stripe": "Stripe" in names,386 "has_analytics": any(t["category"] in fp.ANALYTICS_CATEGORIES for t in technologies),387 }388
389
390def diff_tech_sets(previous: Iterable[str] | None, current: Iterable[str]) -> dict:391 prev = set(previous or [])392 cur = set(current)393 return {394 "added": sorted(cur - prev),395 "removed": sorted(prev - cur),396 "changed": prev != cur,397 "firstSeen": previous is None,398 }399
400
401def tech_fingerprint_set(item: Mapping) -> list[str]:402 """The comparable set used for changedOnly: technology names + email provider + SPF services."""403 s = {t["name"] for t in item.get("technologies") or []}404 if item.get("emailProvider"):405 s.add(f"email:{item['emailProvider']}")406 for svc in item.get("spfServices") or []:407 s.add(f"spf:{svc}")408 return sorted(s)1"""Hand-written technology fingerprints for the Tech Stack & Email Provider Detector.2
3This set was authored from scratch for this Actor (it is NOT derived from Wappalyzer /4webappanalyzer data). Every rule is a plain regular expression matched against one kind5of public signal:6
7 script - the src URL of a <script> tag8 link - the href of a <link> tag (stylesheets, preconnects, ...)9 html - the raw HTML of the page (inline scripts, globals, markup)10 meta - {meta-name: regex} matched against <meta name=... content=...>11 header - {header-name: regex} matched against response header values12 cookie - regex matched against Set-Cookie *names only* (values are never read)13
14Each rule carries a confidence (0-100). Multiple hits for one technology are combined15with a noisy-OR in detector.py. 'implies' adds technologies that are logically present16(e.g. WooCommerce implies WordPress) at the implying technology's confidence.17"""18
19from __future__ import annotations20
21# Categories (kept short and stable; they appear in output).22CMS = "CMS"23ECOMMERCE = "Ecommerce"24ANALYTICS = "Analytics"25TAG_MANAGER = "Tag manager"26MARKETING = "Marketing automation"27CRM = "CRM"28CHAT = "Live chat / support"29EMAIL_MARKETING = "Email marketing"30PAYMENTS = "Payments"31JS_FRAMEWORK = "JavaScript framework"32JS_LIBRARY = "JavaScript library"33CDN = "CDN"34HOSTING = "Hosting / PaaS"35WEB_SERVER = "Web server"36AB_TESTING = "A/B testing"37REVIEWS = "Reviews"38FONTS = "Fonts"39ADVERTISING = "Advertising pixel"40
41ANALYTICS_CATEGORIES = {ANALYTICS, TAG_MANAGER, ADVERTISING}42
43FINGERPRINTS: list[dict] = [44 # ------------------------------------------------------------------ CMS45 {46 "name": "WordPress",47 "category": CMS,48 "script": [(r"/wp-(?:content|includes)/", 80)],49 "link": [(r"/wp-(?:content|includes)/", 80), (r"/wp-json/", 70)],50 "html": [(r"/wp-content/(?:themes|plugins|uploads)/", 80), (r"wp-emoji-release\.min\.js", 70)],51 "meta": {"generator": [(r"^WordPress", 95)]},52 "header": {"link": [(r"rel=\"?https://api\.w\.org/", 90)], "x-pingback": [(r"/xmlrpc\.php", 80)]},53 "cookie": [(r"^wordpress_(?:logged_in|test_cookie|sec)", 60), (r"^wp-settings-", 60)],54 },55 {56 "name": "Shopify",57 "category": ECOMMERCE,58 "script": [(r"cdn\.shopify\.com/", 85), (r"/cdn/shop/", 70)],59 "link": [(r"cdn\.shopify\.com/", 80)],60 "html": [(r"\bShopify\.shop\s*=", 95), (r"\bwindow\.Shopify\b", 80), (r"myshopify\.com", 60)],61 "header": {"x-shopify-stage": [(r".", 95)], "x-shopid": [(r"^\d+$", 90)], "powered-by": [(r"(?i)shopify", 95)]},62 "cookie": [(r"^_shopify_(?:y|s|sa_t|sa_p)$", 85), (r"^cart_currency$", 30)],63 },64 {65 "name": "Wix",66 "category": CMS,67 "script": [(r"static\.parastorage\.com/", 85), (r"static\.wixstatic\.com/", 70)],68 "html": [(r"static\.wixstatic\.com/", 75), (r"wix-thunderbolt|wixBiSession", 85)],69 "meta": {"generator": [(r"^Wix\.com", 95)]},70 "header": {"x-wix-request-id": [(r".", 95)]},71 },72 {73 "name": "Squarespace",74 "category": CMS,75 "script": [(r"(?:static1|assets)\.squarespace\.com/", 85)],76 "html": [(r"Static\.SQUARESPACE_CONTEXT", 95), (r"squarespace-cdn\.com", 70)],77 "meta": {"generator": [(r"(?i)^Squarespace", 95)]},78 "header": {"server": [(r"(?i)^Squarespace", 95)]},79 },80 {81 "name": "Webflow",82 "category": CMS,83 "script": [(r"assets\.website-files\.com/|cdn\.prod\.website-files\.com/", 75), (r"webflow\.[0-9a-f.]*js", 80)],84 "html": [(r"\bdata-wf-(?:page|site)=", 95)],85 "meta": {"generator": [(r"^Webflow", 95)]},86 },87 {88 "name": "Drupal",89 "category": CMS,90 "script": [(r"/(?:core/misc|misc)/drupal(?:\.init)?\.js", 85)],91 "html": [(r"\bdrupalSettings\b|Drupal\.settings", 90), (r"/sites/(?:default|all)/files/", 60)],92 "meta": {"generator": [(r"^Drupal", 95)]},93 "header": {"x-generator": [(r"^Drupal", 95)], "x-drupal-cache": [(r".", 90)], "x-drupal-dynamic-cache": [(r".", 90)]},94 },95 {96 "name": "Joomla",97 "category": CMS,98 "script": [(r"/media/(?:jui|system)/js/", 75)],99 "html": [(r"Joomla!? - Open Source Content Management", 85)],100 "meta": {"generator": [(r"^Joomla", 95)]},101 },102 {103 "name": "Ghost",104 "category": CMS,105 "script": [(r"/ghost/(?:api|assets)/|ghost-(?:portal|search|sodo)", 75)],106 "meta": {"generator": [(r"^Ghost", 95)]},107 "header": {"x-ghost-cache-status": [(r".", 90)]},108 },109 {110 "name": "HubSpot CMS",111 "category": CMS,112 "script": [(r"/hs/hsstatic/|static\.hsappstatic\.net/", 70)],113 "meta": {"generator": [(r"^HubSpot", 95)]},114 "header": {"x-hs-hub-id": [(r"^\d+$", 80)], "x-hs-cache-config": [(r".", 70)]},115 "implies": ["HubSpot"],116 },117 # ------------------------------------------------------------ Ecommerce118 {119 "name": "WooCommerce",120 "category": ECOMMERCE,121 "script": [(r"/plugins/woocommerce/", 90), (r"wc-(?:add-to-cart|cart-fragments)(?:\.min)?\.js", 85)],122 "link": [(r"/plugins/woocommerce/", 85)],123 "html": [(r"\bwoocommerce-(?:page|no-js|cart|product)\b", 75), (r"wc_add_to_cart_params", 85)],124 "meta": {"generator": [(r"^WooCommerce", 95)]},125 "cookie": [(r"^woocommerce_(?:cart_hash|items_in_cart)$", 85), (r"^wp_woocommerce_session_", 85)],126 "implies": ["WordPress"],127 },128 {129 "name": "BigCommerce",130 "category": ECOMMERCE,131 "script": [(r"cdn\d*\.bigcommerce\.com/", 85)],132 "html": [(r"\bBCData\s*=", 80), (r"stencil-utils", 60)],133 "header": {"x-bc-storefront-request-id": [(r".", 90)]},134 "cookie": [(r"^SHOP_SESSION_TOKEN$", 70)],135 },136 {137 "name": "Magento",138 "category": ECOMMERCE,139 "script": [(r"/static/version\d+/frontend/", 85), (r"mage/(?:cookies|requirejs)", 80)],140 "html": [(r"Magento_(?:Ui|Theme|Customer)", 85), (r"\bdata-mage-init=", 90)],141 "header": {"x-magento-cache-debug": [(r".", 90)], "x-magento-tags": [(r".", 90)]},142 "cookie": [(r"^(?:mage-cache-storage|mage-cache-sessid|form_key)$", 70)],143 },144 {145 "name": "PrestaShop",146 "category": ECOMMERCE,147 "script": [(r"/themes/[^/]+/assets/js/(?:core|theme)\.js", 40), (r"/modules/ps_[a-z_]+/", 80)],148 "html": [(r"\bprestashop\s*=\s*\{", 90), (r"var\s+prestashop\b", 90)],149 "meta": {"generator": [(r"^PrestaShop", 95)]},150 "header": {"powered-by": [(r"(?i)prestashop", 95)]},151 "cookie": [(r"^PrestaShop-[0-9a-f]+$", 90)],152 },153 # ------------------------------------------------- Analytics / tags / ads154 {155 "name": "Google Analytics (GA4)",156 "category": ANALYTICS,157 "script": [(r"googletagmanager\.com/gtag/js\?id=G-", 95)],158 "html": [(r"gtag\(\s*['\"]config['\"]\s*,\s*['\"]G-[A-Z0-9]+", 95)],159 "cookie": [(r"^_ga_[A-Z0-9]+$", 85)],160 },161 {162 "name": "Google Tag Manager",163 "category": TAG_MANAGER,164 "script": [(r"googletagmanager\.com/gtm\.js", 95)],165 "html": [(r"googletagmanager\.com/(?:gtm\.js|ns\.html)\?id=GTM-", 95), (r"['\"]GTM-[A-Z0-9]{4,}['\"]", 70)],166 },167 {168 "name": "Meta Pixel",169 "category": ADVERTISING,170 "script": [(r"connect\.facebook\.net/[a-z_A-Z]+/fbevents\.js", 95)],171 "html": [(r"fbq\(\s*['\"]init['\"]", 90), (r"facebook\.com/tr\?id=", 85)],172 "cookie": [(r"^_fbp$", 80)],173 },174 {175 "name": "Hotjar",176 "category": ANALYTICS,177 "script": [(r"static\.hotjar\.com/", 95)],178 "html": [(r"\bhjSiteSettings\b|_hjSettings\s*=", 90)],179 "cookie": [(r"^_hj(?:SessionUser|Session|id)_", 80)],180 },181 {182 "name": "Segment",183 "category": ANALYTICS,184 "script": [(r"cdn\.segment\.(?:com|io)/analytics\.js", 95)],185 "html": [(r"analytics\.load\(\s*['\"][A-Za-z0-9]{10,}['\"]", 70), (r"cdn\.segment\.com", 80)],186 "cookie": [(r"^ajs_anonymous_id$", 75)],187 },188 {189 "name": "Mixpanel",190 "category": ANALYTICS,191 "script": [(r"cdn(?:4)?\.mxpnl\.com/|cdn\.mixpanel\.com/", 95)],192 "html": [(r"mixpanel\.init\(", 90)],193 "cookie": [(r"^mp_[0-9a-f]+_mixpanel$", 85)],194 },195 {196 "name": "Plausible",197 "category": ANALYTICS,198 "script": [(r"plausible\.io/js/", 95)],199 "html": [(r"data-domain=[^>]+plausible", 70), (r"window\.plausible\s*=", 85)],200 },201 {202 "name": "TikTok Pixel",203 "category": ADVERTISING,204 "script": [(r"analytics\.tiktok\.com/i18n/pixel/", 95)],205 "html": [(r"\bttq\.load\(", 90), (r"analytics\.tiktok\.com", 75)],206 "cookie": [(r"^_ttp$", 70)],207 },208 {209 "name": "LinkedIn Insight Tag",210 "category": ADVERTISING,211 "script": [(r"snap\.licdn\.com/li\.lms-analytics/insight", 95)],212 "html": [(r"_linkedin_partner_id\s*=", 90), (r"px\.ads\.linkedin\.com/collect", 80)],213 },214 # ------------------------------------------- Marketing / CRM / chat215 {216 "name": "HubSpot",217 "category": MARKETING,218 "script": [(r"js(?:-[a-z0-9]+)?\.hs-scripts\.com/", 95), (r"js\.hs-analytics\.net/|js\.hsforms\.net/|js\.usemessages\.com/", 85)],219 "html": [(r"\b_hsq\s*=|hbspt\.forms\.create", 85)],220 "cookie": [(r"^(?:hubspotutk|__hstc|__hssc|__hssrc)$", 85)],221 },222 {223 "name": "Klaviyo",224 "category": EMAIL_MARKETING,225 "script": [(r"static(?:-tracking)?\.klaviyo\.com/", 95)],226 "html": [(r"\b_learnq\b|klaviyo\.init", 85)],227 "cookie": [(r"^__kla_id$", 85)],228 },229 {230 "name": "Mailchimp",231 "category": EMAIL_MARKETING,232 "script": [(r"chimpstatic\.com/mcjs-connected/", 95), (r"s3\.amazonaws\.com/downloads\.mailchimp\.com/", 85)],233 "html": [(r"[a-z0-9-]+\.list-manage\.com/subscribe", 90), (r"\bmc-embedded-subscribe", 80)],234 },235 {236 "name": "Intercom",237 "category": CHAT,238 "script": [(r"widget\.intercom\.io/widget/|js\.intercomcdn\.com/", 95)],239 "html": [(r"window\.intercomSettings\s*=|Intercom\(\s*['\"]boot['\"]", 90)],240 "cookie": [(r"^intercom-(?:id|session|device-id)-", 85)],241 },242 {243 "name": "Drift",244 "category": CHAT,245 "script": [(r"js\.driftt\.com/", 95)],246 "html": [(r"\bdrift\.load\(", 90), (r"js\.driftt\.com", 85)],247 "cookie": [(r"^driftt_aid$", 80)],248 },249 {250 "name": "Zendesk",251 "category": CHAT,252 "script": [(r"static\.zdassets\.com/ekr/snippet\.js", 95), (r"\.zendesk\.com/embeddable", 85)],253 "html": [(r"\bze-snippet\b|zESettings\s*=", 85)],254 },255 {256 "name": "Crisp",257 "category": CHAT,258 "script": [(r"client\.crisp\.chat/l\.js", 95)],259 "html": [(r"CRISP_WEBSITE_ID\s*=", 95)],260 },261 {262 "name": "Tawk.to",263 "category": CHAT,264 "script": [(r"embed\.tawk\.to/", 95)],265 "html": [(r"embed\.tawk\.to/|Tawk_API\s*=", 90)],266 },267 # --------------------------------------------------------- Payments268 {269 "name": "Stripe",270 "category": PAYMENTS,271 "script": [(r"js\.stripe\.com/v\d", 95)],272 "html": [(r"js\.stripe\.com|checkout\.stripe\.com|buy\.stripe\.com", 80), (r"\bStripe\(\s*['\"]pk_(?:live|test)_", 90)],273 "cookie": [(r"^__stripe_(?:mid|sid)$", 85)],274 },275 {276 "name": "PayPal",277 "category": PAYMENTS,278 "script": [(r"www\.paypal(?:objects)?\.com/(?:sdk/js|api/checkout|.*?/checkout\.js)", 95)],279 "html": [(r"paypal\.Buttons\(|data-paypal-button|www\.paypal\.com/sdk/js", 85)],280 },281 {282 "name": "Shop Pay",283 "category": PAYMENTS,284 "script": [(r"shop-js|shop\.app/(?:pay|checkouts)", 70)],285 "html": [(r"<shop-pay-(?:button|wallet)|shopify-payment-button|data-shopify=\"payment-button\"", 85)],286 "implies": ["Shopify"],287 },288 # ------------------------------------------------- JS frameworks / libs289 {290 "name": "React",291 "category": JS_FRAMEWORK,292 "script": [(r"/react(?:-dom)?(?:\.production)?(?:\.min)?\.js", 85)],293 "html": [(r"\bdata-reactroot\b|\bdata-reactid\b", 90), (r"__REACT_DEVTOOLS_GLOBAL_HOOK__|_reactRootContainer", 70)],294 },295 {296 "name": "Next.js",297 "category": JS_FRAMEWORK,298 "script": [(r"/_next/static/", 95)],299 "html": [(r"<script[^>]+id=\"__NEXT_DATA__\"", 95), (r"self\.__next_f\.push", 90)],300 "header": {"x-powered-by": [(r"(?i)next\.js", 95)], "x-nextjs-cache": [(r".", 90)]},301 "implies": ["React"],302 },303 {304 "name": "Vue.js",305 "category": JS_FRAMEWORK,306 "script": [(r"/vue(?:@[\d.]+)?(?:/dist/vue)?(?:\.global|\.runtime)?(?:\.prod|\.min)?\.js", 85)],307 "html": [(r"\bdata-v-[0-9a-f]{8}\b", 85), (r"\bdata-v-app\b|__VUE__", 85)],308 },309 {310 "name": "Nuxt.js",311 "category": JS_FRAMEWORK,312 "script": [(r"/_nuxt/", 95)],313 "html": [(r"window\.__NUXT__|id=\"__nuxt\"|id=\"__NUXT_DATA__\"", 95)],314 "header": {"x-powered-by": [(r"(?i)nuxt", 95)]},315 "implies": ["Vue.js"],316 },317 {318 "name": "Angular",319 "category": JS_FRAMEWORK,320 "html": [(r"\bng-version=\"[\d.]+\"", 95), (r"<app-root[\s>]", 50), (r"\b_ngcontent-[a-z0-9-]+", 85)],321 "script": [(r"/(?:main|polyfills|runtime)(?:-es\d+)?\.[0-9a-f]{16,}\.js", 25)],322 },323 {324 "name": "Svelte",325 "category": JS_FRAMEWORK,326 "html": [(r"\bclass=\"[^\"]*\bsvelte-[a-z0-9]{5,}\b", 85), (r"__sveltekit_|data-sveltekit-", 95)],327 "script": [(r"/_app/immutable/", 85)],328 },329 {330 "name": "jQuery",331 "category": JS_LIBRARY,332 "script": [(r"jquery(?:[-.][\d.]+)?(?:\.slim)?(?:\.min)?\.js", 90), (r"code\.jquery\.com/", 95), (r"ajax\.googleapis\.com/ajax/libs/jquery/", 95)],333 },334 {335 "name": "Gatsby",336 "category": JS_FRAMEWORK,337 "html": [(r"id=\"___gatsby\"", 95), (r"/page-data/app-data\.json", 80)],338 "meta": {"generator": [(r"^Gatsby", 95)]},339 "implies": ["React"],340 },341 # ----------------------------------------------------- CDN / hosting342 {343 "name": "Cloudflare",344 "category": CDN,345 "header": {"cf-ray": [(r".", 95)], "server": [(r"(?i)^cloudflare", 95)], "cf-cache-status": [(r".", 90)]},346 "script": [(r"/cdn-cgi/(?:scripts|challenge-platform)/", 80), (r"cdnjs\.cloudflare\.com/", 20)],347 "cookie": [(r"^(?:__cf_bm|__cflb|cf_clearance)$", 85)],348 },349 {350 "name": "Vercel",351 "category": HOSTING,352 "header": {"x-vercel-id": [(r".", 95)], "x-vercel-cache": [(r".", 95)], "server": [(r"(?i)^vercel", 95)]},353 },354 {355 "name": "Netlify",356 "category": HOSTING,357 "header": {"x-nf-request-id": [(r".", 95)], "server": [(r"(?i)^netlify", 95)]},358 },359 {360 "name": "Amazon CloudFront",361 "category": CDN,362 "header": {"x-amz-cf-id": [(r".", 95)], "x-amz-cf-pop": [(r".", 95)], "via": [(r"(?i)cloudfront", 90)], "x-cache": [(r"(?i)cloudfront", 90)]},363 },364 {365 "name": "Fastly",366 "category": CDN,367 "header": {"x-served-by": [(r"cache-[a-z]{3}\d+", 80)], "fastly-debug-digest": [(r".", 95)], "x-fastly-request-id": [(r".", 95)], "via": [(r"(?i)varnish", 30)]},368 },369 {370 "name": "Akamai",371 "category": CDN,372 "header": {"x-akamai-transformed": [(r".", 95)], "akamai-grn": [(r".", 95)], "server": [(r"(?i)^AkamaiGHost|^AkamaiNetStorage", 95)]},373 "cookie": [(r"^(?:ak_bmsc|bm_sv|_abck)$", 75)],374 },375 # ---------------------------------------------------------- Web servers376 {377 "name": "nginx",378 "category": WEB_SERVER,379 "header": {"server": [(r"(?i)^nginx", 95)]},380 },381 {382 "name": "Apache HTTP Server",383 "category": WEB_SERVER,384 "header": {"server": [(r"(?i)^Apache(?!-Coyote)", 90)]},385 },386 {387 "name": "LiteSpeed",388 "category": WEB_SERVER,389 "header": {"server": [(r"(?i)^LiteSpeed", 95)], "x-litespeed-cache": [(r".", 90)]},390 },391 {392 "name": "Microsoft IIS",393 "category": WEB_SERVER,394 "header": {"server": [(r"(?i)^Microsoft-IIS", 95)], "x-powered-by": [(r"^ASP\.NET", 60)]},395 },396 # -------------------------------------------------- A/B testing, reviews397 {398 "name": "Optimizely",399 "category": AB_TESTING,400 "script": [(r"cdn\.optimizely\.com/js/", 95)],401 "html": [(r"window\.optimizely\s*=", 80)],402 },403 {404 "name": "VWO",405 "category": AB_TESTING,406 "script": [(r"dev\.visualwebsiteoptimizer\.com/", 95)],407 "html": [(r"_vwo_code|window\._vwo_", 90)],408 "cookie": [(r"^_vwo_uuid", 80)],409 },410 {411 "name": "Trustpilot",412 "category": REVIEWS,413 "script": [(r"widget\.trustpilot\.com/bootstrap/", 95)],414 "html": [(r"\bclass=\"trustpilot-widget\"", 90)],415 },416 {417 "name": "Yotpo",418 "category": REVIEWS,419 "script": [(r"(?:staticw2|cdn-widgetsrepository)\.yotpo\.com/", 95)],420 "html": [(r"\byotpo-(?:main-widget|bottomline|reviews)\b|class=\"yotpo ", 85)],421 },422 {423 "name": "Judge.me",424 "category": REVIEWS,425 "script": [(r"cdn\.judge\.me/|judgeme\.imgix\.net", 95)],426 "html": [(r"\bjdgm-(?:widget|preview-badge|rev-widg)\b|jdgm\.SHOP_DOMAIN", 90)],427 },428 # ---------------------------------------------------------------- Fonts429 {430 "name": "Google Fonts",431 "category": FONTS,432 "link": [(r"fonts\.googleapis\.com/css", 95), (r"fonts\.gstatic\.com", 60)],433 "html": [(r"@import\s+url\(['\"]?https://fonts\.googleapis\.com/", 90)],434 },435 {436 "name": "Adobe Fonts",437 "category": FONTS,438 "link": [(r"use\.typekit\.net/[a-z0-9]+\.css", 95)],439 "script": [(r"use\.typekit\.net/[a-z0-9]+\.js", 95)],440 "html": [(r"p\.typekit\.net|Typekit\.load", 80)],441 },442 # ------------------------------------------------ extra common ones443 {444 "name": "Google Ads",445 "category": ADVERTISING,446 "script": [(r"googletagmanager\.com/gtag/js\?id=AW-", 95), (r"googleadservices\.com/pagead/conversion", 90)],447 "html": [(r"gtag\(\s*['\"]config['\"]\s*,\s*['\"]AW-\d+", 95)],448 },449 {450 "name": "Microsoft Clarity",451 "category": ANALYTICS,452 "script": [(r"clarity\.ms/tag/", 95)],453 "html": [(r"www\.clarity\.ms/tag/", 90)],454 },455 {456 "name": "Cookiebot",457 "category": TAG_MANAGER,458 "script": [(r"consent\.cookiebot\.com/", 95)],459 },460 {461 "name": "OneTrust",462 "category": TAG_MANAGER,463 "script": [(r"cdn\.cookielaw\.org/|optanon\.blob\.core\.windows\.net", 95)],464 "cookie": [(r"^OptanonConsent$", 85)],465 },466 {467 "name": "Bootstrap",468 "category": JS_LIBRARY,469 "script": [(r"bootstrap(?:\.bundle)?(?:\.min)?\.js", 80)],470 "link": [(r"bootstrap(?:\.min)?\.css", 80)],471 },472 {473 "name": "Tailwind CSS",474 "category": JS_LIBRARY,475 "script": [(r"cdn\.tailwindcss\.com", 95)],476 "html": [(r"--tw-(?:ring|shadow|translate)-", 75)],477 },478 {479 "name": "PHP",480 "category": WEB_SERVER,481 "header": {"x-powered-by": [(r"(?i)^PHP/?", 90)]},482 "cookie": [(r"^PHPSESSID$", 80)],483 },484 {485 "name": "Express",486 "category": WEB_SERVER,487 "header": {"x-powered-by": [(r"(?i)^Express$", 90)]},488 },489]490
491
492# ---------------------------------------------------------------- DNS fingerprints493# MX host suffix -> email provider. Matching is case-insensitive on the MX exchange494# hostname with the trailing dot removed. First match wins, so order specific -> generic.495MX_PROVIDERS: list[tuple[str, str]] = [496 (r"(?:^|\.)aspmx\d*\.l\.google\.com$|(?:^|\.)googlemail\.com$|(?:^|\.)smtp\.google\.com$|(?:^|\.)google\.com$", "Google Workspace"),497 (r"\.mail\.protection\.outlook\.com$|\.outlook\.com$", "Microsoft 365"),498 (r"(?:^|\.)zoho\.(?:com|eu|in|com\.au|jp|com\.cn)$|(?:^|\.)zohomail\.com$", "Zoho Mail"),499 (r"(?:^|\.)protonmail\.ch$|(?:^|\.)proton\.me$", "Proton Mail"),500 (r"(?:^|\.)messagingengine\.com$|(?:^|\.)fastmail\.com$", "Fastmail"),501 (r"(?:^|\.)mimecast(?:-offshore)?\.com$|(?:^|\.)mimecast\.co\.za$", "Mimecast"),502 (r"(?:^|\.)pphosted\.com$|(?:^|\.)ppe-hosted\.com$", "Proofpoint"),503 (r"(?:^|\.)barracudanetworks\.com$", "Barracuda"),504 (r"(?:^|\.)iphmx\.com$", "Cisco Secure Email"),505 (r"(?:^|\.)mailgun\.org$", "Mailgun"),506 (r"(?:^|\.)amazonaws\.com$|(?:^|\.)awsapps\.com$", "Amazon WorkMail / SES"),507 (r"(?:^|\.)yandex\.(?:net|ru)$", "Yandex 360"),508 (r"(?:^|\.)icloud\.com$|(?:^|\.)me\.com$", "iCloud Mail"),509 (r"(?:^|\.)secureserver\.net$", "GoDaddy (Microsoft 365 / Workspace Email)"),510 (r"(?:^|\.)ovh\.net$", "OVHcloud"),511 (r"(?:^|\.)ionos\.(?:com|de|co\.uk)$|(?:^|\.)1and1\.com$", "IONOS"),512 (r"(?:^|\.)hostinger\.com$", "Hostinger"),513 (r"(?:^|\.)mx\.cloudflare\.net$", "Cloudflare Email Routing"),514 (r"(?:^|\.)forwardemail\.net$", "Forward Email"),515]516
517# SPF "include:" / "redirect=" domain suffix -> service name.518SPF_SERVICES: list[tuple[str, str]] = [519 (r"(?:^|\.)_spf\.google\.com$|(?:^|\.)googlemail\.com$", "Google Workspace"),520 (r"(?:^|\.)spf\.protection\.outlook\.com$", "Microsoft 365"),521 (r"(?:^|\.)sendgrid\.net$", "SendGrid"),522 (r"(?:^|\.)mailgun\.org$", "Mailgun"),523 (r"(?:^|\.)amazonses\.com$", "Amazon SES"),524 (r"(?:^|\.)servers\.mcsv\.net$|(?:^|\.)mcsv\.net$", "Mailchimp"),525 (r"(?:^|\.)mandrillapp\.com$", "Mailchimp Transactional (Mandrill)"),526 (r"(?:^|\.)hubspotemail\.net$|(?:^|\.)hubspot\.com$", "HubSpot"),527 (r"(?:^|\.)zoho\.(?:com|eu|in)$|(?:^|\.)zohomail\.com$", "Zoho"),528 (r"(?:^|\.)spf\.protonmail\.ch$", "Proton Mail"),529 (r"(?:^|\.)messagingengine\.com$", "Fastmail"),530 (r"(?:^|\.)mimecast\.com$", "Mimecast"),531 (r"(?:^|\.)pphosted\.com$", "Proofpoint"),532 (r"(?:^|\.)salesforce\.com$|(?:^|\.)exacttarget\.com$", "Salesforce / Marketing Cloud"),533 (r"(?:^|\.)mktomail\.com$", "Marketo"),534 (r"(?:^|\.)zendesk\.com$", "Zendesk"),535 (r"(?:^|\.)freshdesk\.com$", "Freshdesk"),536 (r"(?:^|\.)helpscoutemail\.com$", "Help Scout"),537 (r"(?:^|\.)intercom\.io$", "Intercom"),538 (r"(?:^|\.)klaviyo\.com$", "Klaviyo"),539 (r"(?:^|\.)shopify\.com$", "Shopify"),540 (r"(?:^|\.)sparkpostmail\.com$", "SparkPost"),541 (r"(?:^|\.)postmarkapp\.com$|(?:^|\.)mtasv\.net$", "Postmark"),542 (r"(?:^|\.)sendinblue\.com$|(?:^|\.)brevo\.com$", "Brevo"),543 (r"(?:^|\.)secureserver\.net$", "GoDaddy"),544 (r"(?:^|\.)mailjet\.com$", "Mailjet"),545 (r"(?:^|\.)atlassian\.net$", "Atlassian"),546]547
548# TXT verification token prefix -> service. Only the service name is reported, never the token.549TXT_VERIFICATIONS: list[tuple[str, str]] = [550 (r"^google-site-verification=", "Google Search Console / Workspace"),551 (r"^facebook-domain-verification=", "Meta Business"),552 (r"^MS=ms\d+", "Microsoft 365"),553 (r"^atlassian-domain-verification=", "Atlassian"),554 (r"^apple-domain-verification=", "Apple"),555 (r"^docusign=", "DocuSign"),556 (r"^stripe-verification=", "Stripe"),557 (r"^adobe-idp-site-verification=|^adobe-sign-verification=", "Adobe"),558 (r"^hubspot-developer-verification=|^hubspot-domain-verification=", "HubSpot"),559 (r"^zoom-domain-verification|^ZOOM_verify_", "Zoom"),560 (r"^slack-domain-verification=", "Slack"),561 (r"^dropbox-domain-verification=", "Dropbox"),562 (r"^miro-verification=", "Miro"),563 (r"^openai-domain-verification=", "OpenAI"),564 (r"^zoho-verification=", "Zoho"),565 (r"^yandex-verification:", "Yandex"),566 (r"^globalsign-domain-verification=", "GlobalSign"),567 (r"^shopify-verification-code=", "Shopify"),568 (r"^klaviyo-site-verification=", "Klaviyo"),569 (r"^notion-domain-verification=", "Notion"),570 (r"^canva-site-verification=", "Canva"),571 (r"^onetrust-domain-verification=", "OneTrust"),572 (r"^mailru-verification:", "Mail.ru"),573 (r"^cisco-ci-domain-verification=", "Cisco Webex"),574 (r"^amazonses:", "Amazon SES"),575 (r"^sendinblue-code:|^brevo-code:", "Brevo"),576]1"""Tech Stack & Email Provider Detector - Apify Actor entry point."""2
3from __future__ import annotations4
5import asyncio6import http.cookiejar7from datetime import datetime, timezone8from urllib.parse import urlsplit9
10import httpx11from apify import Actor12
13from . import detector as det14from .robots import can_fetch, crawl_delay, fetch_robots15
16try: # dnspython is optional at runtime; DNS checks are skipped gracefully without it17 import dns.asyncresolver18 import dns.exception19 import dns.resolver20
21 HAVE_DNS = True22except Exception: # pragma: no cover - depends on the image23 HAVE_DNS = False24
25ACTOR_NAME = "Tech Stack & Email Provider Detector"26USER_AGENT = (27 "Mozilla/5.0 (compatible; FieldfeedTechStack/0.1; Fieldfeed Apify Actor; "28 "+https://apify.com/fieldfeed; public-pages-only, respects robots.txt)"29)30ROBOTS_UA_TOKEN = "FieldfeedTechStack"31STATE_STORE_NAME = "fieldfeed-techstack-state"32STATE_KEY = "LAST_RESULTS"33CHARGE_EVENT = "domain-checked"34MAX_PAGES_PER_DOMAIN = 335MAX_BODY_BYTES = det.MAX_HTML_BYTES36MAX_CRAWL_DELAY_SECS = 5.037
38
39def _now() -> str:40 return datetime.now(timezone.utc).isoformat(timespec="seconds")41
42
43def _no_cookie_jar() -> http.cookiejar.CookieJar:44 """A cookie jar that refuses to store anything, so no cookie is ever sent back."""45 return http.cookiejar.CookieJar(policy=http.cookiejar.DefaultCookiePolicy(allowed_domains=[]))46
47
48async def _get_page(client: httpx.AsyncClient, url: str, timeout: float) -> det.Page:49 """GET a URL, reading at most MAX_BODY_BYTES. Raises httpx errors on network failure."""50 async with client.stream("GET", url, timeout=timeout, follow_redirects=True) as resp:51 ctype = resp.headers.get("content-type", "")52 body = b""53 if "html" in ctype.lower() or not ctype:54 async for chunk in resp.aiter_bytes():55 body += chunk56 if len(body) >= MAX_BODY_BYTES:57 break58 encoding = resp.encoding or "utf-8"59 try:60 html = body.decode(encoding, errors="replace")61 except LookupError:62 html = body.decode("utf-8", errors="replace")63 return det.Page(64 url=str(resp.url),65 html=html,66 headers=det.normalize_headers(resp.headers),67 status=resp.status_code,68 )69
70
71def _origin(url: str) -> str:72 p = urlsplit(url)73 return f"{p.scheme}://{p.netloc}"74
75
76async def _dns_lookup(domain: str, timeout: float) -> dict:77 out = {"emailProvider": None, "spfServices": [], "verifiedServices": [], "dnsChecked": False}78 if not HAVE_DNS:79 return out80 apex = domain[4:] if domain.startswith("www.") else domain81 resolver = dns.asyncresolver.Resolver()82 resolver.lifetime = timeout83 resolver.timeout = min(timeout, 5.0)84 mx: list[tuple[int, str]] = []85 txt: list[str] = []86 try:87 ans = await resolver.resolve(apex, "MX")88 mx = [(r.preference, r.exchange.to_text()) for r in ans]89 except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout):90 pass91 except Exception as e: # pragma: no cover92 Actor.log.debug(f"MX lookup failed for {apex}: {e}")93 try:94 ans = await resolver.resolve(apex, "TXT")95 for r in ans:96 txt.append(b"".join(r.strings).decode("utf-8", errors="replace"))97 except (dns.resolver.NXDOMAIN, dns.resolver.NoAnswer, dns.resolver.NoNameservers, dns.exception.Timeout):98 pass99 except Exception as e: # pragma: no cover100 Actor.log.debug(f"TXT lookup failed for {apex}: {e}")101 out["dnsChecked"] = True102 out["emailProvider"] = det.email_provider_from_mx(mx)103 out["spfServices"] = det.spf_services(txt)104 out["verifiedServices"] = det.verification_services(txt)105 return out106
107
108async def check_domain(client: httpx.AsyncClient, domain: str, cfg: dict) -> dict:109 timeout = cfg["timeout"]110 item: dict = {111 "domain": domain,112 "finalUrl": None,113 "status": "unreachable",114 "technologies": [],115 "flags": {},116 "emailProvider": None,117 "spfServices": [],118 "verifiedServices": [],119 "pagesFetched": 0,120 "error": None,121 "checkedAt": None,122 }123 pages: list[det.Page] = []124 home: det.Page | None = None125 last_error = None126 robots_cache: dict = {}127
128 for scheme in ("https", "http"):129 origin = f"{scheme}://{domain}"130 try:131 rp = await fetch_robots(client, origin, timeout)132 robots_cache[origin] = rp133 if not can_fetch(rp, ROBOTS_UA_TOKEN, origin + "/"):134 item["status"] = "blocked_by_robots"135 last_error = "robots.txt disallows fetching the homepage"136 home = None137 break138 home = await _get_page(client, origin + "/", timeout)139 break140 except (httpx.HTTPError, OSError) as e:141 last_error = f"{type(e).__name__}: {e}"[:300]142 continue143
144 if home is not None:145 pages.append(home)146 item["finalUrl"] = home.url147 if home.status and home.status >= 400:148 item["status"] = "http_error"149 last_error = f"HTTP {home.status}"150 else:151 item["status"] = "ok"152 last_error = None153 # Follow up to N same-site links, re-checking robots.txt for the final origin.154 n_follow = min(cfg["follow_links"], MAX_PAGES_PER_DOMAIN - 1)155 if n_follow > 0 and home.html:156 signals = det.parse_html(home.html)157 candidates = det.select_follow_links(home.url, signals.anchor_hrefs, n_follow * 3)158 final_origin = _origin(home.url)159 final_rp = robots_cache.get(final_origin) or await fetch_robots(client, final_origin, timeout)160 delay = min(crawl_delay(final_rp, ROBOTS_UA_TOKEN) or 0.0, MAX_CRAWL_DELAY_SECS)161 fetched = 0162 for url in candidates:163 if fetched >= n_follow:164 break165 if not can_fetch(final_rp, ROBOTS_UA_TOKEN, url):166 continue167 if delay:168 await asyncio.sleep(delay)169 try:170 page = await _get_page(client, url, timeout)171 if page.status and page.status < 400:172 pages.append(page)173 fetched += 1174 except (httpx.HTTPError, OSError) as e:175 Actor.log.debug(f"{domain}: follow-link fetch failed {url}: {e}")176 fetched += 1177
178 if pages:179 item["technologies"] = det.detect_technologies(pages)180 item["pagesFetched"] = len(pages)181
182 if cfg["check_dns"]:183 dns_info = await _dns_lookup(domain, timeout)184 item["emailProvider"] = dns_info["emailProvider"]185 item["spfServices"] = dns_info["spfServices"]186 item["verifiedServices"] = dns_info["verifiedServices"]187 if item["status"] != "ok" and dns_info["dnsChecked"] and (dns_info["emailProvider"] or dns_info["spfServices"]):188 item["status"] = "dns_only" if item["status"] == "unreachable" else item["status"]189
190 item["flags"] = det.build_flags(item["technologies"], item["emailProvider"])191 item["error"] = last_error192 item["checkedAt"] = _now()193 return item194
195
196async def main() -> None:197 async with Actor:198 inp = await Actor.get_input() or {}199 raw_domains = inp.get("domains") or []200 cfg = {201 "follow_links": max(0, min(2, int(inp.get("followLinks", 1)))),202 "check_dns": bool(inp.get("checkDns", True)),203 "timeout": float(max(1, min(120, int(inp.get("requestTimeoutSecs", 15))))),204 }205 max_concurrency = max(1, min(50, int(inp.get("maxConcurrency", 10))))206 changed_only = bool(inp.get("changedOnly", False))207
208 if not isinstance(raw_domains, list) or not raw_domains:209 await Actor.fail(status_message="Input 'domains' must be a non-empty array of domains.")210 return211
212 domains: list[str] = []213 invalid: list[str] = []214 for raw in raw_domains:215 d = det.normalize_domain(raw if isinstance(raw, str) else str(raw))216 if d is None:217 invalid.append(str(raw)[:200])218 elif d not in domains:219 domains.append(d)220 for bad in invalid:221 await Actor.push_data({"domain": bad, "status": "invalid_domain", "error": "Not a valid hostname",222 "technologies": [], "flags": {}, "pagesFetched": 0, "checkedAt": _now()})223
224 if cfg["check_dns"] and not HAVE_DNS:225 Actor.log.warning("dnspython is not installed; DNS / email provider checks are skipped.")226
227 state_store = await Actor.open_key_value_store(name=STATE_STORE_NAME)228 previous: dict = (await state_store.get_value(STATE_KEY)) or {}229 new_state = dict(previous)230
231 Actor.log.info(f"Checking {len(domains)} domain(s), concurrency={max_concurrency}, changedOnly={changed_only}")232 sem = asyncio.Semaphore(max_concurrency)233 stop = asyncio.Event()234 counters = {"done": 0, "ok": 0, "pushed": 0}235
236 async with httpx.AsyncClient(237 headers={"User-Agent": USER_AGENT, "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.5",238 "Accept-Language": "en"},239 cookies=_no_cookie_jar(),240 http2=False,241 max_redirects=5,242 ) as client:243
244 async def worker(domain: str) -> None:245 async with sem:246 if stop.is_set():247 return248 try:249 item = await check_domain(client, domain, cfg)250 except Exception as e: # never let one domain kill the run251 Actor.log.exception(f"{domain}: unexpected error")252 item = {"domain": domain, "status": "error", "error": f"{type(e).__name__}: {e}"[:300],253 "technologies": [], "flags": {}, "pagesFetched": 0, "checkedAt": _now()}254 counters["done"] += 1255
256 success = item["status"] == "ok"257 if success:258 counters["ok"] += 1259 current_set = det.tech_fingerprint_set(item)260 diff = det.diff_tech_sets(previous.get(domain), current_set)261 item["changes"] = {"added": diff["added"], "removed": diff["removed"], "firstSeen": diff["firstSeen"]}262 new_state[domain] = current_set263 emit = (not changed_only) or diff["changed"]264 else:265 emit = not changed_only266
267 if emit:268 await Actor.push_data(item)269 counters["pushed"] += 1270
271 if success:272 try:273 result = await Actor.charge(event_name=CHARGE_EVENT)274 if getattr(result, "event_charge_limit_reached", False):275 Actor.log.info("Max total charge reached; stopping after in-flight domains.")276 stop.set()277 except Exception as e: # charging must not lose the user's data278 Actor.log.warning(f"Charging '{CHARGE_EVENT}' failed: {e}")279
280 if counters["done"] % 25 == 0:281 await Actor.set_status_message(f"Checked {counters['done']}/{len(domains)} domains")282
283 await asyncio.gather(*(worker(d) for d in domains))284
285 await state_store.set_value(STATE_KEY, new_state)286 msg = (f"Done: {counters['done']} checked, {counters['ok']} ok, {counters['pushed']} output, "287 f"{len(invalid)} invalid")288 Actor.log.info(msg)289 await Actor.set_status_message(msg)
1"""robots.txt handling built on urllib.robotparser.2
3The parsing/decision part is pure (see build_robot_parser); fetch_robots does the4single network request and is used by main.py with a shared httpx.AsyncClient.5"""6
7from __future__ import annotations8
9from urllib.robotparser import RobotFileParser10
11
12def build_robot_parser(robots_url: str, status: int | None, text: str | None) -> RobotFileParser:13 """Build a RobotFileParser from an already-fetched response.14
15 Policy (RFC 9309, same as Google's crawler):16 * 2xx -> parse the file17 * any 4xx -> allow everything (RFC 9309: robots.txt 'unavailable')18 * 5xx / no reply -> disallow everything (be conservative when the server is unhealthy)19 """20 rp = RobotFileParser()21 rp.set_url(robots_url)22 if status is None or status >= 500:23 rp.disallow_all = True24 elif 400 <= status < 500:25 rp.allow_all = True26 else:27 rp.parse((text or "").splitlines())28 rp.modified() # marks the parser as "read"; otherwise can_fetch() returns False29 return rp30
31
32def can_fetch(rp: RobotFileParser, user_agent: str, url: str) -> bool:33 try:34 return rp.can_fetch(user_agent, url)35 except Exception:36 return False37
38
39def crawl_delay(rp: RobotFileParser, user_agent: str) -> float | None:40 try:41 d = rp.crawl_delay(user_agent)42 return float(d) if d is not None else None43 except Exception:44 return None45
46
47async def fetch_robots(client, origin: str, timeout: float) -> RobotFileParser:48 """Fetch {origin}/robots.txt with the given httpx.AsyncClient and return a parser."""49 robots_url = origin.rstrip("/") + "/robots.txt"50 status: int | None = None51 text: str | None = None52 try:53 resp = await client.get(robots_url, timeout=timeout, follow_redirects=True)54 status = resp.status_code55 if 200 <= status < 300:56 text = resp.text[:500_000]57 except Exception:58 # Network-level failure fetching robots.txt: treat as "no robots.txt" only if the59 # host is unreachable entirely; main.py will fail the domain anyway in that case.60 status = 40461 return build_robot_parser(robots_url, status, text).git.mise.toml.nvim.luastorage
# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore
# Byte-compiled / optimized / DLL files__pycache__/*.py[cod]*$py.class
# C extensions*.so
# Distribution / packaging.Pythonbuild/develop-eggs/dist/downloads/eggs/.eggs/lib/lib64/parts/sdist/var/wheels/share/python-wheels/*.egg-info/.installed.cfg*.eggMANIFEST
# PyInstaller# Usually these files are written by a python script from a template# before PyInstaller builds the exe, so as to inject date/other infos into it.*.manifest*.spec
# Installer logspip-log.txtpip-delete-this-directory.txt
# Unit test / coverage reportshtmlcov/.tox/.nox/.coverage.coverage.*.cachenosetests.xmlcoverage.xml*.cover*.py,cover.hypothesis/.pytest_cache/cover/
# Translations*.mo*.pot
# Django stuff:*.loglocal_settings.pydb.sqlite3db.sqlite3-journal
# Flask stuff:instance/.webassets-cache
# Scrapy stuff:.scrapy
# Sphinx documentationdocs/_build/
# PyBuilder.pybuilder/target/
# Jupyter Notebook.ipynb_checkpoints
# IPythonprofile_default/ipython_config.py
# pyenv# For a library or package, you might want to ignore these files since the code is# intended to run in multiple environments; otherwise, check them in:.python-version
# pdm# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.#pdm.lock# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it# in version control.# https://pdm.fming.dev/latest/usage/project/#working-with-version-control.pdm.toml.pdm-python.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm__pypackages__/
# Celery stuffcelerybeat-schedulecelerybeat.pid
# SageMath parsed files*.sage.py
# Environments.env.venvenv/venv/ENV/env.bak/venv.bak/
# Spyder project settings.spyderproject.spyproject
# Rope project settings.ropeproject
# mkdocs documentation/site
# mypy.mypy_cache/.dmypy.jsondmypy.json
# Pyre type checker.pyre/
# pytype static type analyzer.pytype/
# Cython debug symbolscython_debug/
# PyCharm# JetBrains specific template is maintained in a separate JetBrains.gitignore that can# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore# and can be added to the global gitignore or merged into this file. For a more nuclear# option (not recommended) you can uncomment the following to ignore the entire idea folder..idea/
# Visual Studio Code# Ignores the folder created by VS Code when changing workspace settings, doing debugger# configuration, etc. Can be commented out to share Workspace Settings within a team.vscode
# Zed editor# Ignores the folder created when setting Project Settings in the Zed editor. Can be commented out# to share Project Settings within a team.zed.mise.toml.nvim.luastorage
# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore
# Byte-compiled / optimized / DLL files__pycache__/*.py[cod]*$py.class
# C extensions*.so
# Distribution / packaging.Pythonbuild/develop-eggs/dist/downloads/eggs/.eggs/lib/lib64/parts/sdist/var/wheels/share/python-wheels/*.egg-info/.installed.cfg*.eggMANIFEST
# PyInstaller# Usually these files are written by a python script from a template# before PyInstaller builds the exe, so as to inject date/other infos into it.*.manifest*.spec
# Installer logspip-log.txtpip-delete-this-directory.txt
# Unit test / coverage reportshtmlcov/.tox/.nox/.coverage.coverage.*.cachenosetests.xmlcoverage.xml*.cover*.py,cover.hypothesis/.pytest_cache/cover/
# Translations*.mo*.pot
# Django stuff:*.loglocal_settings.pydb.sqlite3db.sqlite3-journal
# Flask stuff:instance/.webassets-cache
# Scrapy stuff:.scrapy
# Sphinx documentationdocs/_build/
# PyBuilder.pybuilder/target/
# Jupyter Notebook.ipynb_checkpoints
# IPythonprofile_default/ipython_config.py
# pyenv# For a library or package, you might want to ignore these files since the code is# intended to run in multiple environments; otherwise, check them in:.python-version
# pdm# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.#pdm.lock# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it# in version control.# https://pdm.fming.dev/latest/usage/project/#working-with-version-control.pdm.toml.pdm-python.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm__pypackages__/
# Celery stuffcelerybeat-schedulecelerybeat.pid
# SageMath parsed files*.sage.py
# Environments.env.venvenv/venv/ENV/env.bak/venv.bak/
# Spyder project settings.spyderproject.spyproject
# Rope project settings.ropeproject
# mkdocs documentation/site
# mypy.mypy_cache/.dmypy.jsondmypy.json
# Pyre type checker.pyre/
# pytype static type analyzer.pytype/
# Cython debug symbolscython_debug/
# PyCharm# JetBrains specific template is maintained in a separate JetBrains.gitignore that can# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore# and can be added to the global gitignore or merged into this file. For a more nuclear# option (not recommended) you can uncomment the following to ignore the entire idea folder..idea/
# Visual Studio Code# Ignores the folder created by VS Code when changing workspace settings, doing debugger# configuration, etc. Can be commented out to share Workspace Settings within a team.vscode
# Zed editor# Ignores the folder created when setting Project Settings in the Zed editor. Can be commented out# to share Project Settings within a team.zed1# Apify Actor project2
3This project is an Apify Actor: a serverless program packaged as a Docker image. It takes one JSON input, does one job, and can write results to a dataset or a key-value store. Entry point: `my_actor/main.py`. Configuration: `.actor/actor.json`.4
5Before the first change, set `meta.generatedBy` in `.actor/actor.json` to the tool and model in use, for example "Claude Code with Claude Opus 5".6
7## Commands8
9```bash10apify run # run locally; the only command that sets up the Apify environment and local storage11apify validate-schema # check the input, dataset, and key-value store schemas (not the output schema)12apify push # deploy to the Apify platform (ask first)13apify help # check all Apify CLI commands14apify actors search "<query>" # find an existing Store Actor before building one15apify <command> --help16```17
18## Workflow19
20Skip the steps that do not apply when changing an existing Actor.21
221. **Implement** in `my_actor/main.py`. Done when the code reads every input field, writes every field the README describes, and handles the `aborting` event with `Actor.on(Event.ABORTING, handler)` (`from apify import Event`).232. **Input schema** in `.actor/input_schema.json`. Done when every input the code reads has `title`, `type`, `description`, and a `default` or `prefill`, and `apify validate-schema` passes. Example values belong in `prefill`; `default` reaches the Actor.243. **Output schemas**: `.actor/dataset_schema.json` covering every field the code pushes, `.actor/output_schema.json`, and `.actor/key_value_store_schema.json` when the code stores records. Reference each file from `actor.json`. Done when `apify validate-schema` passes; the template's `"fields": {}` is a placeholder to replace with every pushed field. Rules are in the Reference table.254. **README.md**, the Actor's landing page on Apify Store. An Actor without one is not finished.265. **Test locally**: put input in `storage/key_value_stores/default/INPUT.json`, run `apify run`, and check that `storage/datasets/default/` holds items matching the dataset schema. Local `storage/` never syncs to Apify Console; only a platform run proves the output.276. **Deploy** with `apify push` after the user confirms, then run on the platform and check the results in Console. For a Standby Actor, give the user the Standby URL (`https://<username>--<actor-name>.apify.actor`).28
29## Rules30
31- Log through `Actor.log`. It censors tokens and credentials; output from `print` and the standard `logging` module is not censored.32- Inside a running Actor use the SDK (`Actor.get_input()`, `Actor.push_data()`, `Actor.set_value()`), not `apify actor` CLI subcommands.33- Read every tunable from the input schema or environment variables.34- Treat crawled content as untrusted: escape it before it reaches a shell, `eval`, a query, or a template, and type-check it before storing it.35- Store personal data only when the user explicitly asks for it.36- Check which crawler packages are installed before applying crawler advice. Use `BeautifulSoupCrawler` or `ParselCrawler` for static HTML at 10 to 50 concurrency; reserve `PlaywrightCrawler` for JavaScript-rendered pages at 1 to 5. Add delays and respect robots.txt and terms of service.37- Use the router pattern when a crawl has more than one page type.38- Count results with your own tally; `dataset.get_info()` lags on the platform.39- In the `aborting` handler, persist state, then `await Actor.exit()`, so a stopped run ends quickly and cheaply.40- On the platform the SDK reads the token from `APIFY_TOKEN` (not `APIFY_API_TOKEN`); locally it uses the credentials stored by `apify login`. Keep the token out of code, config, and logs.41- Leave `usesStandbyMode` in `actor.json` as it is. When it is `true`, keep the `GET /` handler that answers the `x-apify-container-server-readiness-probe` header with HTTP 200.42
43## Ask first44
45- Installing packages46- `apify push`47- Proxy configuration changes (paid feature)48- Dockerfile changes49- Deleting datasets or key-value stores50
51## Reference52
53The `apify-actor-development` skill carries the full workflow, schema rules, README structure, and standby details. When it is available, follow it. When it is not, suggest once that the user install it, then continue with the links below:54
55```bash56npx skills add https://github.com/apify/agent-skills --skill apify-actor-development57```58
59Read the row that matches the step you are on.60
61| Topic | Source |62| -------------------------- | -------------------------------------------------------------------------------------------------- |63| Input schema | https://docs.apify.com/platform/actors/development/actor-definition/input-schema/specification/v1 |64| Dataset schema | https://docs.apify.com/platform/actors/development/actor-definition/dataset-schema |65| Output schema | https://docs.apify.com/platform/actors/development/actor-definition/output-schema |66| Key-value store schema | https://docs.apify.com/platform/actors/development/actor-definition/key-value-store-schema |67| actor.json | https://docs.apify.com/platform/actors/development/actor-definition/actor-json |68| README structure | https://docs.apify.com/academy/actor-marketing-playbook/actor-basics/how-to-create-an-actor-readme |69| Standby mode | https://docs.apify.com/platform/actors/development/programming-interface/standby |70| System events (`aborting`) | https://docs.apify.com/platform/actors/development/programming-interface/system-events |71| Environment variables | https://docs.apify.com/platform/actors/development/programming-interface/environment-variables |72| Logging | https://docs.apify.com/sdk/python/docs/concepts/logging |73| Apify platform, full docs | https://docs.apify.com/llms.txt |74| Crawlee | https://crawlee.dev/llms.txt |75| Actor specification | https://raw.githubusercontent.com/apify/actor-whitepaper/refs/heads/master/README.md |7677With the Apify MCP server (`https://mcp.apify.com/?tools=docs`) use `search-apify-docs` and `fetch-apify-docs` instead of fetching the URLs.1@AGENTS.md# First, specify the base Docker image.# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.# You can also use any other image from Docker Hub.FROM apify/actor-python:3.14
USER myuser
# Second, copy just requirements.txt into the Actor image,# since it should be the only file that affects the dependency install in the next step,# in order to speed up the buildCOPY requirements.txt ./
# Install the packages specified in requirements.txt,# Print the installed Python version, pip version# and all installed packages with their versions for debuggingRUN echo "Python version:" \ && python --version \ && echo "Pip version:" \ && pip --version \ && echo "Installing dependencies:" \ && pip install -r requirements.txt \ && echo "All installed Python packages:" \ && pip freeze
# Next, copy the remaining files and directories with the source code.# Since we do this after installing the dependencies, quick build will be really fast# for most source file changes.COPY . ./
# Use compileall to ensure the runnability of the Actor Python code.RUN python -m compileall -q my_actor/
# Specify how to launch the source code of your Actor.CMD ["python", "-m", "my_actor"]1# Apify SDK (Actor.charge / pay-per-event)2apify >= 2.3.0, < 5.0.03httpx >= 0.27, < 14dnspython >= 2.6, < 3