1"""Pure extraction helpers - no I/O, so they are easy to test and cheap to run."""
2
3from __future__ import annotations
4
5import re
6from typing import Any
7from urllib.parse import urljoin, urlparse
8
9from bs4 import BeautifulSoup
10
11EMAIL_RE = re.compile(
12 r'\b[A-Za-z0-9._%+\-]+@[A-Za-z0-9.\-]+\.[A-Za-z]{2,24}\b'
13)
14
15PHONE_RE = re.compile(
16 r'(?<![\w.])(\+\d{1,3}[\s.\-()]?(?:\(?\d{1,4}\)?[\s.\-]?){1,5}\d{2,6})(?![\w.])'
17)
18
19_ASSET_SUFFIX = (
20 '.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico',
21 '.woff', '.woff2', '.ttf', '.eot', '.css', '.js', '.mp4', '.pdf',
22)
23_NOREPLY = ('noreply', 'no-reply', 'donotreply', 'do-not-reply')
24_PLACEHOLDER = (
25 'example.com', 'yourdomain', 'domain.com', 'email.com', 'sentry.io',
26 'wixpress.com', 'squarespace.com', '@2x', 'u003e',
27)
28
29
30
31_STOP = r'[^"\'\s?#<>&\\]'
32
33SOCIAL_PATTERNS = {
34 'linkedin': re.compile(rf'https?://(?:[a-z]{{2,3}}\.)?linkedin\.com/(?:company|school|in)/{_STOP}+', re.I),
35 'twitter': re.compile(r'https?://(?:www\.)?(?:twitter|x)\.com/(?!intent|share|home)[A-Za-z0-9_]{1,25}', re.I),
36 'facebook': re.compile(rf'https?://(?:www\.|[a-z]{{2}}-[a-z]{{2}}\.)?facebook\.com/(?!sharer|dialog|tr\?){_STOP}+', re.I),
37 'instagram': re.compile(rf'https?://(?:www\.)?instagram\.com/(?!p/|explore){_STOP}+', re.I),
38 'youtube': re.compile(rf'https?://(?:www\.)?youtube\.com/(?:c/|channel/|user/|@){_STOP}+', re.I),
39 'github': re.compile(r'https?://(?:www\.)?github\.com/[A-Za-z0-9\-_.]{1,39}(?:/[A-Za-z0-9\-_.]+)?', re.I),
40 'tiktok': re.compile(rf'https?://(?:www\.)?tiktok\.com/@{_STOP}+', re.I),
41}
42
43
44TECH_SIGNALS: tuple[tuple[str, tuple[str, ...]], ...] = (
45 ('WordPress', ('/wp-content/', '/wp-includes/', 'name="generator" content="wordpress')),
46 ('Shopify', ('cdn.shopify.com', 'shopify.theme', 'myshopify.com')),
47 ('Wix', ('static.parastorage.com', 'wix.com/website-template', 'X-Wix-')),
48 ('Squarespace', ('squarespace.com/universal', 'static1.squarespace.com')),
49 ('Webflow', ('assets.website-files.com', 'data-wf-page', 'assets-global.website-files.com')),
50 ('HubSpot', ('js.hs-scripts.com', 'js.hsforms.net', 'hs-analytics')),
51 ('Google Tag Manager', ('googletagmanager.com/gtm.js', 'googletagmanager.com/ns.html')),
52 ('Google Analytics 4', ('googletagmanager.com/gtag/js', 'gtag(')),
53 ('Segment', ('cdn.segment.com/analytics.js',)),
54 ('Intercom', ('widget.intercom.io', 'intercomcdn.com')),
55 ('Drift', ('js.driftt.com',)),
56 ('Zendesk', ('static.zdassets.com', 'zendesk.com/embeddable')),
57 ('Stripe', ('js.stripe.com',)),
58 ('PayPal', ('paypalobjects.com', 'paypal.com/sdk/js')),
59 ('Cloudflare', ('cdn-cgi/', 'cloudflareinsights.com')),
60 ('Next.js', ('/_next/static', '__NEXT_DATA__')),
61 ('Nuxt', ('/_nuxt/', '__NUXT__')),
62 ('React', ('react-dom', 'data-reactroot', '__reactContainer')),
63 ('Vue.js', ('vue.runtime', 'data-v-app')),
64 ('Angular', ('ng-version=', 'angular.min.js')),
65 ('Svelte / SvelteKit', ('/_app/immutable/', 'svelte-')),
66 ('Bootstrap', ('bootstrap.min.css', 'bootstrap.bundle')),
67 ('Tailwind CSS', ('tailwind', 'tw-')),
68 ('jQuery', ('jquery.min.js', 'jquery-3')),
69 ('Font Awesome', ('fontawesome', 'font-awesome')),
70 ('Google Fonts', ('fonts.googleapis.com',)),
71 ('Mailchimp', ('chimpstatic.com', 'list-manage.com')),
72 ('Klaviyo', ('static.klaviyo.com',)),
73 ('Hotjar', ('static.hotjar.com',)),
74 ('Meta Pixel', ('connect.facebook.net/en_US/fbevents.js', 'fbq(')),
75 ('LinkedIn Insight', ('snap.licdn.com',)),
76 ('Plausible', ('plausible.io/js',)),
77 ('Matomo', ('matomo.js', 'piwik.js')),
78 ('Sentry', ('browser.sentry-cdn.com', '@sentry/')),
79 ('Algolia', ('algolianet.com', 'algolia.net')),
80 ('Contentful', ('images.ctfassets.net',)),
81 ('Sanity', ('cdn.sanity.io',)),
82 ('Vercel', ('vercel-insights', '/_vercel/')),
83 ('Calendly', ('assets.calendly.com',)),
84 ('Typeform', ('embed.typeform.com',)),
85)
86
87WORKABLE_SHORTLINK_RE = re.compile(r'apply\.workable\.com/j/([A-Za-z0-9]+)', re.I)
88
89
90
91RESERVED_ATS_TOKENS = frozenset({
92 'www', 'jobs', 'careers', 'apply', 'app', 'j', 'embed', 'job', 'company',
93 'companies', 'search', 'api', 'static', 'assets', 'images', 'en', 'de',
94 'fr', 'es', 'nl', 'widget', 'help', 'blog', 'support', 'status', 'account',
95 'developers', 'resources', 'docs', 'partners', 'about', 'login', 'signup',
96 'cdn', 'media', 'files', 'img', 'static2', 'go', 'link', 'links',
97})
98
99ATS_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
100 ('greenhouse', re.compile(r'(?:job-)?boards\.greenhouse\.io/(?:embed/job_board\?for=)?([A-Za-z0-9_\-]+)', re.I)),
101 ('greenhouse', re.compile(r'boards-api\.greenhouse\.io/v1/boards/([A-Za-z0-9_\-]+)', re.I)),
102 ('lever', re.compile(r'jobs\.lever\.co/([A-Za-z0-9_\-]+)', re.I)),
103 ('ashby', re.compile(r'jobs\.ashbyhq\.com/([A-Za-z0-9_\-]+)', re.I)),
104 ('workable', re.compile(r'apply\.workable\.com/([A-Za-z0-9_\-]+)/j/', re.I)),
105 ('workable', re.compile(r'\b([A-Za-z0-9_\-]+)\.workable\.com/', re.I)),
106 ('workable', re.compile(r'apply\.workable\.com/([A-Za-z0-9_\-]+)', re.I)),
107 ('personio', re.compile(r'([A-Za-z0-9_\-]+)\.jobs\.personio\.(?:de|com)', re.I)),
108 ('recruitee', re.compile(r'([A-Za-z0-9_\-]+)\.recruitee\.com', re.I)),
109 ('teamtailor', re.compile(r'([A-Za-z0-9_\-]+)\.teamtailor\.com', re.I)),
110 ('smartrecruiters', re.compile(r'careers\.smartrecruiters\.com/([A-Za-z0-9_\-]+)', re.I)),
111 ('bamboohr', re.compile(r'([A-Za-z0-9_\-]+)\.bamboohr\.com/(?:careers|jobs)', re.I)),
112 ('workday', re.compile(r'([A-Za-z0-9_\-]+)\.(?:wd\d+\.)?myworkdayjobs\.com', re.I)),
113 ('breezy', re.compile(r'([A-Za-z0-9_\-]+)\.breezy\.hr', re.I)),
114 ('join', re.compile(r'join\.com/companies/([A-Za-z0-9_\-]+)', re.I)),
115 ('jobvite', re.compile(r'jobs\.jobvite\.com/([A-Za-z0-9_\-]+)', re.I)),
116)
117
118
119
120
121
122CANDIDATE_PATH_GROUPS: tuple[tuple[str, tuple[str, ...]], ...] = (
123 ('contact', ('contact', 'kontakt', 'contacto', 'contatti', 'contact-us')),
124 ('careers', ('careers', 'career', 'jobs', 'join-us', 'work-with-us', 'vacancies',
125 'stellenangebote', 'hiring')),
126 ('imprint', ('impressum', 'imprint', 'mentions-legales', 'legal-notice')),
127 ('about', ('about', 'about-us', 'company', 'who-we-are', 'team')),
128 ('support', ('support', 'help', 'faq')),
129 ('legal', ('legal', 'privacy', 'terms')),
130)
131
132
133def clean_domain(raw: str) -> str | None:
134 """Normalise anything the user pasted into a bare registrable host."""
135 value = (raw or '').strip().lower()
136 if not value:
137 return None
138 if '@' in value and '://' not in value:
139 value = value.rsplit('@', 1)[-1]
140 if '://' not in value:
141 value = 'http://' + value
142 host = urlparse(value).hostname or ''
143 host = host.strip('.')
144 return host or None
145
146
147def is_useful_email(address: str, exclude_generic: bool) -> bool:
148 lowered = address.lower()
149 if lowered.endswith(_ASSET_SUFFIX):
150 return False
151 if any(bad in lowered for bad in _PLACEHOLDER):
152 return False
153 local = lowered.split('@', 1)[0]
154
155 if len(local) > 40 or re.fullmatch(r'[0-9a-f]{16,}', local):
156 return False
157 if exclude_generic and any(bad in lowered for bad in _NOREPLY):
158 return False
159 return True
160
161
162def extract_from_html(
163 html_text: str,
164 page_url: str,
165 wanted: set[str],
166 exclude_generic: bool,
167) -> dict[str, Any]:
168 """Pull everything of interest out of one page."""
169 found: dict[str, Any] = {
170 'emails': set(),
171 'phones': set(),
172 'socials': {},
173 'techStack': set(),
174 'ats': None,
175 'atsShortLinks': [],
176 'meta': {},
177 'links': [],
178 }
179
180 if 'emails' in wanted:
181 for match in EMAIL_RE.findall(html_text):
182 if is_useful_email(match, exclude_generic):
183 found['emails'].add(match.lower())
184
185 if 'socials' in wanted:
186 for name, pattern in SOCIAL_PATTERNS.items():
187 match = pattern.search(html_text)
188 if match:
189 found['socials'][name] = match.group(0).rstrip('/\\"\'.,;:)')
190
191 if 'techStack' in wanted:
192 lowered = html_text.lower()
193 for label, needles in TECH_SIGNALS:
194 if any(needle.lower() in lowered for needle in needles):
195 found['techStack'].add(label)
196
197 if 'atsBoard' in wanted:
198 for provider, pattern in ATS_PATTERNS:
199
200
201
202
203 token = None
204 for match in pattern.finditer(html_text):
205 candidate = match.group(1)
206 if candidate.lower() in RESERVED_ATS_TOKENS:
207 continue
208 token = candidate
209 break
210 if token:
211 found['ats'] = {'provider': provider, 'token': token, 'foundOn': page_url}
212 break
213
214
215
216 for match in WORKABLE_SHORTLINK_RE.finditer(html_text):
217 link = 'https://apply.workable.com/j/' + match.group(1)
218 if link not in found['atsShortLinks']:
219 found['atsShortLinks'].append(link)
220
221 try:
222 soup = BeautifulSoup(html_text, 'lxml')
223 except Exception:
224 soup = BeautifulSoup(html_text, 'html.parser')
225
226
227 for tag in soup(['script', 'style', 'noscript', 'svg']):
228 tag.decompose()
229 text = soup.get_text(' ', strip=True)
230
231 if 'phones' in wanted:
232 for match in PHONE_RE.findall(text):
233 digits = re.sub(r'\D', '', match)
234 if 8 <= len(digits) <= 16:
235 found['phones'].add(re.sub(r'\s{2,}', ' ', match.strip()))
236 for anchor in soup.find_all('a', href=True):
237 if anchor['href'].lower().startswith('tel:'):
238 value = anchor['href'][4:].strip()
239 if 8 <= len(re.sub(r'\D', '', value)) <= 16:
240 found['phones'].add(value)
241
242 if 'meta' in wanted:
243 title = soup.find('title')
244 description = soup.find('meta', attrs={'name': 'description'}) or soup.find(
245 'meta', attrs={'property': 'og:description'}
246 )
247 site_name = soup.find('meta', attrs={'property': 'og:site_name'})
248 generator = soup.find('meta', attrs={'name': 'generator'})
249 html_tag = soup.find('html')
250 found['meta'] = {
251 'title': title.get_text(strip=True)[:300] if title else None,
252 'description': (description.get('content') or '').strip()[:600] if description else None,
253 'siteName': (site_name.get('content') or '').strip()[:200] if site_name else None,
254 'generator': (generator.get('content') or '').strip()[:200] if generator else None,
255 'language': (html_tag.get('lang') or '').strip()[:16] if html_tag else None,
256 }
257
258 base_host = urlparse(page_url).hostname or ''
259 for anchor in soup.find_all('a', href=True):
260 href = anchor['href'].strip()
261 if href.startswith(('mailto:', 'tel:', 'javascript:', '#')):
262 if href.startswith('mailto:') and 'emails' in wanted:
263 address = href[7:].split('?')[0].strip()
264 if EMAIL_RE.fullmatch(address) and is_useful_email(address, exclude_generic):
265 found['emails'].add(address.lower())
266 continue
267 absolute = urljoin(page_url, href)
268 if (urlparse(absolute).hostname or '') == base_host:
269 found['links'].append(absolute.split('#')[0])
270
271 return found
272
273
274def rank_links(links: list[str], base_host: str) -> list[str]:
275 """Order same-host links so the pages most likely to hold company data come first.
276
277 Round-robins across the groups above, so a four-page budget buys a contact
278 page, a careers page, an imprint and an about page rather than four
279 variations on /contact.
280 """
281 buckets: dict[str, list[str]] = {name: [] for name, _ in CANDIDATE_PATH_GROUPS}
282 seen: set[str] = set()
283 for link in links:
284 if link in seen:
285 continue
286 seen.add(link)
287 path = (urlparse(link).path or '/').lower().strip('/')
288 if not path or path.count('/') > 3 or path.endswith(_ASSET_SUFFIX):
289 continue
290 for name, words in CANDIDATE_PATH_GROUPS:
291 if any(word in path for word in words):
292 buckets[name].append(link)
293 break
294
295 for name in buckets:
296
297 buckets[name].sort(key=len)
298
299 ordered: list[str] = []
300 round_index = 0
301 while any(buckets.values()):
302 for name, _ in CANDIDATE_PATH_GROUPS:
303 if len(buckets[name]) > round_index:
304 ordered.append(buckets[name][round_index])
305 round_index += 1
306 if round_index > 4:
307 break
308 return ordered
309
310
311def dedupe_phones(raw: set[str]) -> list[str]:
312 """Collapse +44 20 3872 0620 / +44 203 872 0620 / +442038720620 into one.
313
314 The most readable spelling wins: the one with the most separators, then the
315 longest.
316 """
317 best: dict[str, str] = {}
318 for value in raw:
319 digits = re.sub(r'\D', '', value)
320 if not digits:
321 continue
322 current = best.get(digits)
323 if current is None:
324 best[digits] = value
325 continue
326 score = (sum(c in ' .-()' for c in value), len(value))
327 current_score = (sum(c in ' .-()' for c in current), len(current))
328 if score > current_score:
329 best[digits] = value
330 return sorted(best.values())
331
332
333def same_site(email: str, domain: str) -> bool:
334 """True when the address belongs to the site being crawled."""
335 email_host = email.rsplit('@', 1)[-1].lower()
336 site = domain.lower().split(':')[0]
337 a = email_host.split('.')
338 b = site.split('.')
339 return a[-2:] == b[-2:] if len(a) >= 2 and len(b) >= 2 else email_host == site
