1"""
2Shared utilities for the Jobs Board Scraper boards.
3
4Cost on Apify = memory x wall-clock + residential-proxy GB, so the fetch layer
5is built around four rules:
6
7 * HTTP first. Every board tries a plain httpx request before any browser.
8 A search page over HTTP is ~50-100 KB on the wire and takes ~1 s; the same
9 page in Chromium is 1-3 MB and 10-60 s. Reed, Totaljobs, CWJobs and GOV.UK
10 all serve their results server-side, so a typical UK run never needs it.
11 * Lazy browser. Chromium is launched only when a board is actually blocked
12 over HTTP (bot wall / soft-404 / thin page), and only for that board.
13 * One browser context per board, warmed once, reused for every page: the
14 cookie jar and proxy session persist across pagination, which scores
15 better with Cloudflare Bot Management than a cold context per page and
16 avoids paying 300-500 ms of context setup each time.
17 * No fixed sleeps. We wait for the job-card selector, not "networkidle".
18
19Board-specific anti-bot hooks (page_is_blocked / on_blocked_page) and the
20per-board browser knobs are kept from v0.11 so Reed's soft-404 salvage and the
21GOV.UK WAF handling still work on the rare browser path.
22"""
23
24from __future__ import annotations
25
26import asyncio
27import json
28import random
29import re
30from abc import ABC, abstractmethod
31from dataclasses import dataclass, asdict
32from datetime import date, datetime, timedelta, timezone
33from typing import Awaitable, Callable, Optional
34from urllib.parse import urlparse
35
36import httpx
37from apify import Actor
38from bs4 import BeautifulSoup
39
40
41
42
43
44CHROME_UA = (
45 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
46 "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
47)
48
49STEALTH_ARGS = [
50 "--no-sandbox",
51 "--disable-dev-shm-usage",
52 "--disable-blink-features=AutomationControlled",
53 "--disable-features=IsolateOrigins,site-per-process",
54 "--disable-infobars",
55 "--disable-background-networking",
56 "--disable-default-apps",
57 "--disable-extensions",
58 "--disable-sync",
59 "--disable-translate",
60 "--no-first-run",
61 "--mute-audio",
62 "--window-size=1366,900",
63]
64
65
66
67STEALTH_JS = """
68Object.defineProperty(navigator, 'webdriver', { get: () => false });
69Object.defineProperty(navigator, 'plugins', { get: () => [
70 { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer' },
71 { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai' },
72 { name: 'Native Client', filename: 'internal-nacl-plugin' },
73]});
74Object.defineProperty(navigator, 'languages', { get: () => ['en-GB', 'en-US', 'en'] });
75const _gp = WebGLRenderingContext.prototype.getParameter;
76WebGLRenderingContext.prototype.getParameter = function (p) {
77 if (p === 37445) return 'Intel Inc.';
78 if (p === 37446) return 'Intel Iris OpenGL Engine';
79 return _gp.call(this, p);
80};
81window.chrome = { runtime: {}, loadTimes: function () {}, csi: function () {} };
82const _q = window.navigator.permissions.query;
83window.navigator.permissions.query = (p) => (
84 p.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : _q(p)
85);
86"""
87
88
89
90JS_EXTRACT_JOBS = """
91(() => {
92 const jobs = [];
93 const selectors = [
94 '[data-testid*="job"]', 'article', '[data-at="job-item"]',
95 '[class*="JobCard"]', '[class*="job-card"]',
96 '[class*="SearchResult"]', '[class*="search-result"]',
97 '[class*="job-result"]', 'li[class*="job"]',
98 ];
99 let cards = [];
100 for (const sel of selectors) {
101 const found = document.querySelectorAll(sel);
102 if (found.length > cards.length) cards = found;
103 }
104 if (cards.length < 2) {
105 const allLinks = document.querySelectorAll('a[href*="/job/"]');
106 const parents = new Set();
107 for (const a of allLinks) {
108 if (a.parentElement && a.parentElement.parentElement) parents.add(a.parentElement.parentElement);
109 }
110 if (parents.size > 2) cards = parents;
111 }
112 for (const card of cards) {
113 const allLinks = card.querySelectorAll('a');
114 let titleLink = null;
115 for (const a of allLinks) {
116 const text = (a.innerText || a.textContent || '').trim();
117 if (text.length > 3 && text.length < 200 && a.href &&
118 (a.href.includes('/job/') || a.href.includes('/jobs/') || a.href.includes('viewjob'))) { titleLink = a; break; }
119 }
120 if (!titleLink) {
121 for (const a of allLinks) {
122 const text = (a.innerText || a.textContent || '').trim();
123 if (text.length > 5 && text.length < 200 && a.href) { titleLink = a; break; }
124 }
125 }
126 if (!titleLink) continue;
127 const title = (titleLink.innerText || titleLink.textContent || '').trim();
128 if (!title || title.length < 3) continue;
129 const job = { title: title, url: titleLink.href || '' };
130 const idMatch = job.url.match(/\\/job\\/(\\d+)/) || job.url.match(/-job(\\d+)/) || job.url.match(/\\/(\\d{5,})/);
131 if (idMatch) job.job_id = idMatch[1];
132 job._card_text = (card.innerText || card.textContent || '').substring(0, 2000);
133 const segments = [];
134 const walker = document.createTreeWalker(card, NodeFilter.SHOW_TEXT, null, false);
135 let node;
136 while (node = walker.nextNode()) {
137 const text = node.textContent.trim();
138 if (text.length > 1 && text.length < 300 && text !== title) segments.push(text);
139 }
140 job._segments = segments.slice(0, 30);
141 for (const a of allLinks) {
142 if (a === titleLink) continue;
143 const href = a.href || '';
144 const text = (a.innerText || a.textContent || '').trim();
145 if (text.length > 1 && text.length < 100 &&
146 (href.includes('/company/') || href.includes('/employer/') ||
147 href.includes('/recruiter/') || href.includes('/list-jobs/'))) { job._company_link_text = text; break; }
148 }
149 jobs.push(job);
150 }
151 return jobs;
152})()
153"""
154
155
156
157
158BLOCKED_RESOURCE_TYPES = frozenset({"image", "media", "font", "stylesheet"})
159
160
161
162
163BLOCKED_URL_PATTERNS = (
164 "google-analytics", "googletagmanager", "googlesyndication", "googleadservices",
165 "doubleclick", "adservice.google", "googleoptimize", "google.com/pagead", "imasdk.googleapis",
166 "facebook.net", "connect.facebook", "facebook.com/tr",
167 "bat.bing", "clarity.ms", "snap.licdn", "px.ads.linkedin", "analytics.tiktok", "ads-twitter",
168 "ct.pinterest", "events.redditmedia", "sc-static.net", "tr.snapchat", "mc.yandex",
169 "hotjar", "mixpanel", "segment.io", "segment.com", "amplitude", "heapanalytics", "fullstory",
170 "mouseflow", "crazyegg", "kissmetrics", "quantummetric", "inspectlet", "smartlook",
171 "luckyorange", "chartbeat", "matomo", "plausible.io", "statcounter",
172 "newrelic", "nr-data.net", "sentry.io", "bugsnag", "datadoghq", "rollbar", "speedcurve", "dynatrace",
173 "optimizely", "vwo.com", "visualwebsiteoptimizer", "abtasty", "kameleoon",
174 "adsystem", "adservice", "adnxs", "criteo", "taboola", "outbrain", "scorecardresearch",
175 "quantserve", "quantcount", "pubmatic", "rubiconproject", "openx.net", "casalemedia",
176 "smartadserver", "teads.tv", "yieldlab", "moatads", "adsafeprotected", "doubleverify",
177 "adsrvr.org", "mathtag", "bidswitch", "sharethrough", "gumgum", "33across", "indexww",
178 "sonobi", "adform.net",
179 "id5-sync", "permutive", "liveramp", "krxd.net", "bluekai", "demdex", "everesttech",
180 "agkn.com", "rlcdn.com", "tapad.com", "crwdcntrl", "exelator", "eyeota",
181 "cdn.cookielaw", "onetrust", "cookiebot", "usercentrics", "trustarc", "consensu.org",
182 "sourcepoint", "quantcast",
183 "tiqcdn.com", "ensighten", "adobedtm", "omtrdc.net", "branch.io", "appsflyer", "adjust.com",
184 "intercom.io", "intercomcdn", "livechatinc", "tawk.to", "drift.com", "zdassets", "zopim",
185 "onesignal", "pushwoosh", "liveperson", "salesforceliveagent",
186 "youtube.com/embed", "youtube-nocookie", "player.vimeo", "brightcove", "jwplayer", "jwpcdn",
187 "fonts.googleapis", "fonts.gstatic", "use.typekit", "fast.fonts.net",
188)
189
190CHALLENGE_TITLE_WORDS = (
191 "just a moment", "challenge", "attention required", "access denied",
192 "security check", "pardon our interruption", "blocked", "are you a human",
193 "request unsuccessful", "verify you are", "bot detection", "something went wrong",
194)
195
196
197def is_challenge_page(title: str, status: int | None, body_len: int, keyword: str = "") -> bool:
198 """Heuristic: is this a bot-wall rather than a results page?
199
200 Title words are only trusted on small bodies, and the search keyword is
201 removed first, so a genuine results page like "Blocked Drain Engineer Jobs
202 in London" (800 KB) is never mistaken for Cloudflare.
203 """
204 if status in (401, 403, 406, 429, 503):
205 return True
206 if body_len < 3000:
207 return True
208 t = (title or "").lower()
209 if keyword:
210 t = t.replace(keyword.lower(), "")
211 return body_len < 200_000 and any(w in t for w in CHALLENGE_TITLE_WORDS)
212
213
214def parse_proxy_url(proxy_url: str) -> dict:
215 p = urlparse(proxy_url)
216 d = {"server": f"{p.scheme}://{p.hostname}:{p.port}"}
217 if p.username:
218 d["username"] = p.username
219 if p.password:
220 d["password"] = p.password
221 return d
222
223
224
225
226
227
228class BrowserPool:
229 """Lazily launched Chromium with one persistent context per board.
230
231 Nothing starts until the first `context()` call, so runs whose boards all
232 succeed over HTTP never pay for a browser at all.
233 """
234
235 def __init__(self, proxy_config=None, max_concurrent_pages: int = 3):
236 self.proxy_config = proxy_config
237 self._pw = None
238 self._browser = None
239 self._contexts: dict[str, object] = {}
240 self._launch_lock = asyncio.Lock()
241 self._page_sem = asyncio.Semaphore(max_concurrent_pages)
242 self.launched = False
243
244 async def _ensure_browser(self):
245 if self._browser:
246 return self._browser
247 async with self._launch_lock:
248 if self._browser:
249 return self._browser
250 from playwright.async_api import async_playwright
251
252 Actor.log.info("[browser] Launching Chromium (first board that needs it)")
253 self._pw = await async_playwright().start()
254 launch_kwargs = {"headless": True, "args": STEALTH_ARGS}
255 if self.proxy_config:
256
257
258
259 seed = await self.proxy_config.new_url(session_id=f"seed_{random.randint(1000, 9999)}")
260 launch_kwargs["proxy"] = parse_proxy_url(seed)
261 self._browser = await self._pw.chromium.launch(**launch_kwargs)
262 self.launched = True
263 return self._browser
264
265 @property
266 def version(self) -> str:
267 try:
268 return self._browser.version if self._browser else ""
269 except Exception:
270 return ""
271
272 async def context(self, scraper: "BaseScraper"):
273 key = scraper.source_name
274 ctx = self._contexts.get(key)
275 if ctx:
276 return ctx
277 browser = await self._ensure_browser()
278 ua = CHROME_UA
279 if scraper.ua_from_browser_version:
280 try:
281 major = browser.version.split(".")[0]
282 ua = (f"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
283 f"(KHTML, like Gecko) Chrome/{major}.0.0.0 Safari/537.36")
284 except Exception:
285 pass
286 ctx = await browser.new_context(
287 proxy=parse_proxy_url(scraper.proxy_url) if scraper.proxy_url else None,
288 viewport={"width": 1366, "height": 900},
289 user_agent=ua,
290 locale="en-GB",
291 timezone_id="Europe/London",
292 java_script_enabled=True,
293 bypass_csp=True,
294 )
295 if scraper.use_stealth_js:
296 await ctx.add_init_script(STEALTH_JS)
297 blocked_types = frozenset(scraper.blocked_resource_types)
298
299 async def _route(route):
300 req = route.request
301 if req.resource_type in blocked_types:
302 await route.abort()
303 return
304 url = req.url.lower()
305 for pat in BLOCKED_URL_PATTERNS:
306 if pat in url:
307 await route.abort()
308 return
309 await route.continue_()
310
311 await ctx.route("**/*", _route)
312 self._contexts[key] = ctx
313
314
315
316 if scraper.warmup_url:
317 page = await ctx.new_page()
318 try:
319 Actor.log.info(f"[{key}] warm-up visit: {scraper.warmup_url}")
320 await page.goto(scraper.warmup_url, wait_until="domcontentloaded", timeout=25000)
321 await page.wait_for_timeout(1500 + random.randint(0, 1000))
322 await page.mouse.move(400 + random.randint(0, 400), 300 + random.randint(0, 200))
323 await page.mouse.wheel(0, 500 + random.randint(0, 400))
324 await page.wait_for_timeout(500 + random.randint(0, 500))
325 except Exception as e:
326 Actor.log.debug(f"[{key}] warm-up failed (continuing): {e}")
327 finally:
328 await page.close()
329 return ctx
330
331 async def reset_context(self, key: str):
332 ctx = self._contexts.pop(key, None)
333 if ctx:
334 try:
335 await ctx.close()
336 except Exception:
337 pass
338
339 @property
340 def page_slot(self) -> asyncio.Semaphore:
341 return self._page_sem
342
343 async def close(self):
344 for ctx in list(self._contexts.values()):
345 try:
346 await ctx.close()
347 except Exception:
348 pass
349 self._contexts.clear()
350 if self._browser:
351 try:
352 await self._browser.close()
353 except Exception:
354 pass
355 self._browser = None
356 if self._pw:
357 try:
358 await self._pw.stop()
359 except Exception:
360 pass
361 self._pw = None
362
363
364
365
366
367
368_WS = re.compile(r"\s+")
369
370
371def clean_text(text: str | None) -> str:
372 if not text:
373 return ""
374 return _WS.sub(" ", str(text)).strip()
375
376
377def make_headers(accept_language: str = "en-GB,en;q=0.9") -> dict:
378 """Realistic Chrome request headers for HTML boards.
379
380 A Chrome UA without matching sec-ch-ua client hints gets failover pages
381 from some WAFs (GOV.UK), so the full consistent set is sent. Accept-Encoding
382 is left to httpx, which advertises only the codecs it can decode.
383 """
384 return {
385 "User-Agent": CHROME_UA,
386 "Accept": ("text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,"
387 "image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7"),
388 "Accept-Language": accept_language,
389 "Sec-Ch-Ua": '"Google Chrome";v="131", "Chromium";v="131", "Not_A Brand";v="24"',
390 "Sec-Ch-Ua-Mobile": "?0",
391 "Sec-Ch-Ua-Platform": '"Windows"',
392 "Sec-Fetch-Dest": "document",
393 "Sec-Fetch-Mode": "navigate",
394 "Sec-Fetch-Site": "none",
395 "Sec-Fetch-User": "?1",
396 "Upgrade-Insecure-Requests": "1",
397 "Connection": "keep-alive",
398 }
399
400
401
402
403HTML_TIMEOUT = httpx.Timeout(12.0, connect=10.0)
404
405
406def make_api_headers() -> dict:
407 return {"User-Agent": "Mozilla/5.0 (compatible; jobs-board-scraper/1.0)", "Accept": "application/json"}
408
409
410def make_soup(html: str) -> BeautifulSoup:
411 return BeautifulSoup(html, "lxml")
412
413
414class LazySoup:
415 """BeautifulSoup that is only built on first attribute access.
416
417 Boards that read embedded JSON (StepStone, Indeed) never touch the DOM on
418 the happy path, so they skip the parse entirely.
419 """
420
421 __slots__ = ("_html", "_soup")
422
423 def __init__(self, html: str):
424 self._html = html
425 self._soup = None
426
427 def __getattr__(self, name):
428 if self._soup is None:
429 self._soup = make_soup(self._html)
430 return getattr(self._soup, name)
431
432
433def _as_soup(html_or_soup) -> BeautifulSoup | LazySoup:
434 return html_or_soup if hasattr(html_or_soup, "select") else LazySoup(html_or_soup or "")
435
436
437@dataclass
438class JobListing:
439 """Unified job listing format (kept for import compatibility)."""
440 title: str = ""
441 company: str = ""
442 location: str = ""
443 salary_raw: str = ""
444 salary_min: Optional[float] = None
445 salary_max: Optional[float] = None
446 salary_currency: str = ""
447 salary_period: str = "annum"
448 snippet: str = ""
449 full_description: str = ""
450 employment_type: str = ""
451 date_posted: str = ""
452 valid_through: str = ""
453 url: str = ""
454 job_id: str = ""
455 source: str = ""
456
457 def to_dict(self) -> dict:
458 return asdict(self)
459
460
461CURRENCY_SYMBOL = {"GBP": "£", "USD": "$", "EUR": "€", "AUD": "A$"}
462DOLLAR_CURRENCIES = {"USD", "AUD", "CAD", "NZD", "SGD"}
463
464_SAL_K = re.compile(r"(\d+(?:\.\d+)?)\s*k\b", re.IGNORECASE)
465_SAL_NUM = re.compile(r"\d+(?:\.\d+)?")
466_THOUSANDS_DOT = re.compile(r"(?<=\d)\.(?=\d{3}(?!\d))")
467_THOUSANDS_SPACE = re.compile(r"(?<=\d)[ ](?=\d{3}(?!\d))")
468_PERIOD_ANNUM = re.compile(r"per\s*annum|\bp\.?a\.?\b|per\s*year|\ba\s*year\b|annual|yearly|pro\s*jahr|par\s*an\b|per\s*jaar", re.IGNORECASE)
469_PERIOD_DAY = re.compile(r"(?:\b(?:per|a)\b|/)\s*day\b|\bp/?d\b|\bdaily\b|pro\s*tag|par\s*jour|per\s*dag", re.IGNORECASE)
470_PERIOD_HOUR = re.compile(r"(?:\b(?:per|an)\b|/)\s*hour\b|\bp/?h\b|\bhourly\b|\bph\b|pro\s*stunde|par\s*heure|de\s*l'heure|per\s*uur", re.IGNORECASE)
471_PERIOD_WEEK = re.compile(r"(?:\b(?:per|a)\b|/)\s*week\b|\bweekly\b|\bp/?w\b|pro\s*woche|par\s*semaine|per\s*week", re.IGNORECASE)
472_PERIOD_MONTH = re.compile(r"(?:\b(?:per|a)\b|/)\s*month\b|\bmonthly\b|\bp/?m\b|pro\s*monat|par\s*mois|per\s*maand", re.IGNORECASE)
473
474
475def parse_salary(salary_text: str | None, default_currency: str = "GBP") -> dict:
476 """Parse free-text salary into {raw, min, max, currency, period}."""
477 raw = clean_text(salary_text)
478 result = {"raw": raw, "min": None, "max": None, "currency": default_currency, "period": "annum"}
479 if not raw:
480 return result
481 lower = raw.lower()
482 if any(w in lower for w in ("competitive", "negotiable", "unspecified", "not specified", "undisclosed")):
483 return result
484
485 if "a$" in lower or "au$" in lower or "aud" in lower:
486 result["currency"] = "AUD"
487 elif "$" in raw or "usd" in lower:
488
489 result["currency"] = default_currency if default_currency in DOLLAR_CURRENCIES else "USD"
490 elif "€" in raw or "eur" in lower:
491 result["currency"] = "EUR"
492 elif "£" in raw or "gbp" in lower:
493 result["currency"] = "GBP"
494
495
496 if _PERIOD_ANNUM.search(lower):
497 result["period"] = "annum"
498 elif _PERIOD_DAY.search(lower):
499 result["period"] = "day"
500 elif _PERIOD_HOUR.search(lower):
501 result["period"] = "hour"
502 elif _PERIOD_WEEK.search(lower):
503 result["period"] = "week"
504 elif _PERIOD_MONTH.search(lower):
505 result["period"] = "month"
506
507 cleaned = raw.replace(",", "")
508 cleaned = _THOUSANDS_DOT.sub("", cleaned)
509 cleaned = _THOUSANDS_SPACE.sub("", cleaned)
510 cleaned = _SAL_K.sub(lambda m: str(int(float(m.group(1)) * 1000)), cleaned)
511 numbers = [float(n) for n in _SAL_NUM.findall(cleaned)]
512 numbers = [n for n in numbers if n > 0]
513 if not numbers:
514 return result
515
516 lo = numbers[0]
517 hi = numbers[1] if len(numbers) > 1 else lo
518
519 if hi < lo / 5:
520 hi = lo
521 if hi < lo:
522 lo, hi = hi, lo
523 result["min"], result["max"] = lo, hi
524 return result
525
526
527def annualise(amount: float | None, period: str) -> float | None:
528 if amount is None:
529 return None
530 return {"day": 220, "hour": 1760, "week": 52, "month": 12}.get(period, 1) * amount
531
532
533_MONTHS = {m: i for i, m in enumerate(
534 ["jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"], 1)}
535_REL = re.compile(r"(\d+)\s*\+?\s*(minute|min|hour|hr|day|week|month)s?\s*ago", re.IGNORECASE)
536_REL_EU = re.compile(r"(?:vor|il y a|geleden)?\s*(\d+)\s*\+?\s*(tag|tage|tagen|jour|jours|dag|dagen|woche|wochen|semaine|semaines|week|weken)\b", re.IGNORECASE)
537_DMY = re.compile(r"\b(\d{1,2})[/\-.](\d{1,2})[/\-.](\d{2,4})\b")
538_DM = re.compile(r"\b(\d{1,2})(?:st|nd|rd|th)?\s+([a-z]{3,9})\.?(?:\s+(\d{4}))?", re.IGNORECASE)
539_MD = re.compile(r"\b([a-z]{3,9})\.?\s+(\d{1,2})(?:st|nd|rd|th)?(?:,?\s+(\d{4}))?", re.IGNORECASE)
540
541
542def normalize_date(text, today: date | None = None, future_ok: bool = False) -> str:
543 """Return an ISO date (YYYY-MM-DD) for the many formats boards use, or ''.
544
545 Handles ISO timestamps, "today", "yesterday", "3 days ago", "2 weeks ago",
546 "18 September", "Sep 18, 2026", "18/09/2026" (day-first), epoch seconds or
547 milliseconds, and German/French/Dutch relative phrases. A day-month string
548 with no year is assumed past (a posting date) unless `future_ok` is set
549 (a closing date), in which case it is the next occurrence.
550 """
551 if text is None:
552 return ""
553 if isinstance(text, (int, float)) and not isinstance(text, bool):
554 try:
555 ts = float(text)
556 if ts > 1e11:
557 ts /= 1000
558 return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat()
559 except (ValueError, OverflowError, OSError):
560 return ""
561 s = clean_text(str(text))
562 if not s:
563 return ""
564 today = today or date.today()
565 low = s.lower()
566
567 m = re.match(r"(\d{4})-(\d{2})-(\d{2})", s)
568 if m:
569 try:
570 return date(int(m.group(1)), int(m.group(2)), int(m.group(3))).isoformat()
571 except ValueError:
572 return ""
573
574 if (re.search(r"\b(today|just now|just posted|new|heute|aujourd'hui|vandaag)\b", low)
575 or re.search(r"\d+\s*\+?\s*(minute|min|hour|hr|stunde|stunden|heure|heures|uur)s?\b", low)):
576 return today.isoformat()
577 if re.search(r"\b(yesterday|gestern|hier|gisteren)\b", low):
578 return (today - timedelta(days=1)).isoformat()
579
580 m = _REL.search(low)
581 if m:
582 n = int(m.group(1))
583 unit = m.group(2)
584 days = n if unit == "day" else n * 7 if unit == "week" else n * 30
585 return (today - timedelta(days=days)).isoformat()
586 m = _REL_EU.search(low)
587 if m:
588 n = int(m.group(1))
589 days = n * 7 if m.group(2).lower().startswith(("woch", "semain", "week", "weken")) else n
590 return (today - timedelta(days=days)).isoformat()
591
592 m = _DMY.search(s)
593 if m:
594 d, mo, y = int(m.group(1)), int(m.group(2)), int(m.group(3))
595 if y < 100:
596 y += 2000
597 if mo > 12 and d <= 12:
598 d, mo = mo, d
599 try:
600 return date(y, mo, d).isoformat()
601 except ValueError:
602 return ""
603
604 for pat, d_idx, m_idx in ((_DM, 1, 2), (_MD, 2, 1)):
605 m = pat.search(s)
606 if m:
607 mon = _MONTHS.get(m.group(m_idx)[:3].lower())
608 if not mon:
609 continue
610 d = int(m.group(d_idx))
611 y = int(m.group(3)) if m.group(3) else today.year
612 try:
613 dt = date(y, mon, d)
614 except ValueError:
615 continue
616 if not m.group(3):
617 try:
618 if not future_ok and dt > today + timedelta(days=1):
619 dt = date(y - 1, mon, d)
620 elif future_ok and dt < today - timedelta(days=1):
621 dt = date(y + 1, mon, d)
622 except ValueError:
623 pass
624 return dt.isoformat()
625
626 if re.fullmatch(r"\d{9,13}", s):
627 return normalize_date(int(s), today, future_ok)
628 return ""
629
630
631_WORK_MODE = re.compile(
632 r"\b(remote|work from home|wfh|fully remote|home[\s-]?based)\b|"
633 r"\b(hybrid)\b|"
634 r"\b(on[\s-]?site|office[\s-]?based|in[\s-]?office)\b",
635 re.IGNORECASE,
636)
637
638
639def detect_work_mode(*texts: str | None) -> str:
640 """Return 'Remote', 'Hybrid', 'On-site' or '' from any of the given texts.
641
642 Texts are checked in order (most specific first); within a text, Hybrid
643 beats Remote beats On-site, since "remote ... hybrid" almost always
644 describes a hybrid role.
645 """
646 for t in texts:
647 if not t:
648 continue
649 found = {("Hybrid" if m.group(2) else "Remote" if m.group(1) else "On-site") for m in _WORK_MODE.finditer(str(t))}
650 if found:
651 for mode in ("Hybrid", "Remote", "On-site"):
652 if mode in found:
653 return mode
654 return ""
655
656
657_EMP_TYPE = re.compile(
658 r"\b(permanent|contract|temporary|temp|fixed[\s-]?term|freelance|internship|"
659 r"apprenticeship|graduate)\b(?:\s*,\s*(full[\s-]?time|part[\s-]?time))?|"
660 r"\b(full[\s-]?time|part[\s-]?time)\b",
661 re.IGNORECASE,
662)
663
664
665def detect_employment_type(text: str | None) -> str:
666 if not text:
667 return ""
668 m = _EMP_TYPE.search(str(text))
669 if not m:
670 return ""
671 if m.group(1):
672 out = "Temporary" if m.group(1).lower() == "temp" else m.group(1).title()
673 if m.group(2):
674 out += ", " + m.group(2).lower()
675 return out
676 return m.group(3).title()
677
678
679_SAL_IN_TEXT = re.compile(
680 r"(?:A\$|AU\$|[£$€])\s*[\d,]+(?:\.\d+)?k?"
681 r"(?:\s*(?:-|–|to)\s*(?:A\$|AU\$|[£$€])?\s*[\d,]+(?:\.\d+)?k?)?"
682 r"(?:\s*(?:per|a|an|/)\s*(?:annum|year|day|hour|week|month)|\s*p\.?a\.?|\s*p/?[hdw])?",
683 re.IGNORECASE,
684)
685
686
687def extract_salary_from_text(text: str | None, default_currency: str = "GBP") -> dict | None:
688 if not text:
689 return None
690 m = _SAL_IN_TEXT.search(str(text))
691 if not m:
692 return None
693 sal = parse_salary(m.group(), default_currency)
694 return sal if sal["min"] else None
695
696
697def apply_salary(job: dict, sal: dict | None) -> None:
698 if not sal or not (sal.get("min") or sal.get("max")):
699 return
700 job["salary_raw"] = sal["raw"]
701 job["salary_min"] = sal["min"]
702 job["salary_max"] = sal["max"]
703 job["salary_currency"] = sal["currency"]
704 job["salary_period"] = sal["period"]
705
706
707def strip_tracking(url: str) -> str:
708 """Drop tracking query strings that make the same job look like two URLs."""
709 if not url:
710 return ""
711 p = urlparse(url)
712 if p.query and any(k in p.query for k in ("sid=", "source=", "utm_", "ref=", "cmp=", "src=", "WT.mc_id", "position=")):
713 return p._replace(query="", fragment="").geturl()
714 return p._replace(fragment="").geturl()
715
716
717
718
719
720
721PageCallback = Callable[[str, list[dict]], Awaitable[None]]
722
723
724class BaseScraper(ABC):
725 """Base class with HTTP-first fetching and lazy browser fallback.
726
727 HTML boards implement `_build_url` and `_parse_search(html, soup)`; the
728 resumable pagination loop lives here. API boards override `search`.
729 """
730
731
732 default_currency = "GBP"
733 card_selector: str | None = None
734 page_size = 25
735 max_pages = 40
736 hard_page_cap: int | None = None
737 use_js_fallback = True
738 resumable = True
739 use_stealth_js = True
740 ua_from_browser_version = False
741 warmup_url: str | None = None
742 blocked_resource_types = BLOCKED_RESOURCE_TYPES
743 browser_max_attempts: int | None = None
744 browser_attempt_timeout = 75.0
745 http_retry_rotations = 1
746
747 def __init__(self, client: httpx.AsyncClient, delay: float = 0.8, *,
748 browser_pool: BrowserPool | None = None, proxy_url: str | None = None,
749 proxy_config=None, on_page: PageCallback | None = None, browser=None,
750 client_headers: dict | None = None, direct_client: httpx.AsyncClient | None = None):
751 self.client = client
752 self._client_headers = client_headers
753 self.direct_client = direct_client
754 self.delay = delay
755 self.browser_pool = browser_pool
756 self.proxy_url = proxy_url
757 self.proxy_config = proxy_config
758 self.on_page = on_page
759 self.fetch_mode = "auto"
760 self.stats = {"pages": 0, "http_pages": 0, "browser_pages": 0, "jobs": 0, "mode": "http"}
761 self._last_browser_extracted: list[dict] = []
762 self._last_fetch_salvaged = False
763 self._next_referer: str | None = None
764 self.radius_miles = None
765 self.country_hint = ""
766
767
768 self.job_type_norm = "any"
769 self.remote_only = False
770 self.posted_within_hours = None
771 self.fetch_details = False
772
773
774 self._page = 1
775 self._seen: set[str] = set()
776 self.exhausted = False
777 self._keyword = ""
778 self._location = ""
779
780 @property
781 def browser(self):
782 """True-ish when a browser fallback is available (compat with v0.11 boards)."""
783 return self.browser_pool
784
785
786
787 def page_is_blocked(self, html: str) -> bool:
788 """Board-specific soft-block detection (bot-detection pages served
789 with HTTP 200 and full-size HTML). Default: never blocked."""
790 return False
791
792 async def on_blocked_page(self, page, html: str) -> str | None:
793 """Salvage hook called with the LIVE Playwright page when
794 page_is_blocked() fired. Return replacement content to use it, or
795 None to fail the attempt (proxy rotates, retry)."""
796 return None
797
798 @property
799 @abstractmethod
800 def source_name(self) -> str: ...
801
802
803
804 def _build_url(self, keyword: str, location: str, job_type: str,
805 salary_min: int | None, page: int) -> str:
806 raise NotImplementedError
807
808 def _parse_search(self, html: str, soup) -> tuple[list[dict], bool]:
809 raise NotImplementedError
810
811 def _reset_for(self, keyword: str, location: str) -> None:
812 if keyword != self._keyword or location != self._location:
813 self._keyword, self._location = keyword, location
814 self._page = 1
815 self.exhausted = False
816 self._next_referer = None
817
818 async def search(self, keyword: str, location: str, max_results: int = 50,
819 job_type: str = "all", salary_min: int | None = None) -> list[dict]:
820 all_jobs: list[dict] = []
821 self._reset_for(keyword or "", location or "")
822 if self.exhausted:
823 return all_jobs
824 seen = self._seen
825 page = self._page
826 while len(all_jobs) < max_results and page <= self.max_pages:
827 url = self._build_url(keyword, location, job_type, salary_min, page)
828 html = await self._get_html(url)
829 if not html:
830 self.exhausted = True
831 break
832 soup = LazySoup(html)
833 jobs, has_next = self._parse_search(html, soup)
834 if not jobs and self.use_js_fallback and self._last_browser_extracted:
835 jobs = self._process_js_extracted()
836 if not jobs and self.fetch_mode == "http" and self.browser_pool and page > 1:
837
838
839 Actor.log.info(f"[{self.source_name}] page {page} empty over HTTP, checking with the browser")
840 self.fetch_mode = "browser"
841 self.stats["mode"] = "browser"
842 html = await self._fetch_browser(url)
843 if html:
844 self.stats["browser_pages"] += 1
845 soup = LazySoup(html)
846 jobs, has_next = self._parse_search(html, soup)
847 if not jobs and self.use_js_fallback and self._last_browser_extracted:
848 jobs = self._process_js_extracted()
849 if not jobs:
850 Actor.log.info(f"[{self.source_name}] No jobs on page {page}, stopping")
851 self.exhausted = True
852 break
853 fresh = []
854 for job in jobs:
855 key = job.get("job_id") or job.get("url") or job.get("title")
856 if key in seen:
857 continue
858 seen.add(key)
859 job.setdefault("source", self.source_name)
860 job.setdefault("salary_currency", self.default_currency)
861 fresh.append(job)
862 if not fresh:
863 self.exhausted = True
864 break
865 page += 1
866 self._page = page
867 self._next_referer = url
868
869
870 all_jobs.extend(fresh)
871 self.stats["pages"] += 1
872 self.stats["jobs"] += len(fresh)
873 Actor.log.info(f"[{self.source_name}] page {page - 1}: +{len(fresh)} (run total {self.stats['jobs']}) via {self.stats['mode']}")
874 if self.on_page:
875 await self.on_page(self.source_name, fresh)
876 if not has_next:
877 self.exhausted = True
878 break
879 if len(all_jobs) >= max_results:
880 break
881 await self._polite_delay()
882 return all_jobs
883
884
885
886 async def _get_html(self, url: str) -> str | None:
887 """HTTP first; on a block, retry HTTP on a fresh proxy session (a rate
888 limit or tarpit is per IP); only then switch the board to the browser."""
889 if self.fetch_mode != "browser":
890 html = await self._fetch_html_http(url)
891 if not html and self.proxy_config and self._client_headers is not None:
892 for _ in range(self.http_retry_rotations):
893 await self._rotate_proxy()
894 html = await self._fetch_html_http(url)
895 if html:
896 Actor.log.info(f"[{self.source_name}] fresh proxy session passed over HTTP")
897 break
898 if html:
899 self.fetch_mode = "http"
900 self.stats["mode"] = "http"
901 self.stats["http_pages"] += 1
902 return html
903 if not self.browser_pool:
904 return None
905 Actor.log.info(f"[{self.source_name}] HTTP blocked/thin, switching to browser for this board")
906 self.fetch_mode = "browser"
907 self.stats["mode"] = "browser"
908 html = await self._fetch_browser(url)
909 if html:
910 self.stats["browser_pages"] += 1
911 return html
912
913 async def _fetch_html_http(self, url: str) -> str | None:
914 """Fetch a search page over HTTP and sanity-check it; None if blocked."""
915 try:
916 r = await self.client.get(url)
917 except httpx.HTTPError as e:
918 Actor.log.info(f"[{self.source_name}] HTTP fetch failed: {type(e).__name__}")
919 return None
920 text = r.text
921 m = re.search(r"<title[^>]*>(.*?)</title>", text[:5000], re.IGNORECASE | re.DOTALL)
922 title = clean_text(m.group(1)) if m else ""
923 if is_challenge_page(title, r.status_code, len(text), self._keyword):
924 Actor.log.info(f"[{self.source_name}] HTTP {r.status_code} '{title[:40]}' ({len(text)} B) looks blocked")
925 return None
926 if self.page_is_blocked(text):
927 Actor.log.info(f"[{self.source_name}] HTTP response is the board's soft-block page")
928 return None
929 return text
930
931 async def _fetch(self, url: str) -> str | None:
932 try:
933 r = await self.client.get(url)
934 r.raise_for_status()
935 return r.text
936 except httpx.HTTPError as e:
937 Actor.log.warning(f"[{self.source_name}] Failed to fetch {url}: {type(e).__name__}")
938 return None
939
940 async def _fetch_json(self, url: str, headers: dict | None = None):
941 try:
942 r = await self.client.get(url, headers=headers)
943 r.raise_for_status()
944 return r.json()
945 except (httpx.HTTPError, ValueError) as e:
946 Actor.log.warning(f"[{self.source_name}] Failed to fetch JSON {url[:120]}: {e}")
947 return None
948
949 async def _fetch_detail(self, url: str) -> str | None:
950 """Detail page: httpx first (JSON-LD is in the raw HTML), browser only
951 when the page is thin."""
952 html = await self._fetch(url)
953 if html and ("application/ld+json" in html or len(html) > 10000):
954 return html
955 if self.browser_pool:
956 browser_html = await self._fetch_browser(url)
957 if browser_html and len(browser_html) > len(html or ""):
958 return browser_html
959 return html
960
961 async def _fetch_browser(self, url: str, max_attempts: int | None = None) -> str | None:
962 if not self.browser_pool:
963 return None
964 if max_attempts is None:
965 max_attempts = self.browser_max_attempts
966 if max_attempts is None:
967
968 max_attempts = 2 if self.proxy_config else 1
969 for attempt in range(1, max_attempts + 1):
970 try:
971 self._last_fetch_salvaged = False
972
973
974 html = await asyncio.wait_for(self._fetch_browser_once(url), timeout=self.browser_attempt_timeout)
975 if html:
976 return html
977 except asyncio.TimeoutError:
978 Actor.log.warning(f"[{self.source_name}] browser attempt {attempt} hung past {self.browser_attempt_timeout:.0f}s; discarding context")
979 await self.browser_pool.reset_context(self.source_name)
980 except Exception as e:
981 Actor.log.warning(f"[{self.source_name}] browser attempt {attempt}: {type(e).__name__}: {str(e)[:120]}")
982 if attempt < max_attempts:
983 await self._rotate_proxy()
984 await asyncio.sleep(1.5 + random.random() * 2)
985 Actor.log.warning(f"[{self.source_name}] browser gave up on {url}")
986 return None
987
988 async def _fetch_browser_once(self, url: str) -> str | None:
989 ctx = await self.browser_pool.context(self)
990 async with self.browser_pool.page_slot:
991 page = await ctx.new_page()
992 try:
993 goto_kwargs = {"wait_until": "domcontentloaded", "timeout": 30000}
994 if self._next_referer:
995 goto_kwargs["referer"] = self._next_referer
996 elif self.warmup_url:
997 goto_kwargs["referer"] = self.warmup_url
998 try:
999 resp = await page.goto(url, **goto_kwargs)
1000 except Exception as e:
1001 Actor.log.info(f"[{self.source_name}] navigation failed ({type(e).__name__}), retrying with load event")
1002 resp = await page.goto(url, wait_until="load", timeout=20000)
1003 status = resp.status if resp else None
1004 selector = self.card_selector or 'a[href*="/job"]'
1005 try:
1006 await page.wait_for_selector(selector, timeout=8000)
1007 except Exception:
1008 pass
1009 title = await page.title()
1010 if is_challenge_page(title, status, 10_000, self._keyword):
1011
1012
1013 Actor.log.info(f"[{self.source_name}] challenge page (status={status}, title='{title[:40]}'), waiting")
1014 for _ in range(6):
1015 await page.wait_for_timeout(2000)
1016 t = (await page.title()).lower().replace(self._keyword.lower(), "")
1017 if not any(w in t for w in CHALLENGE_TITLE_WORDS):
1018 break
1019 else:
1020 return None
1021 try:
1022 await page.wait_for_selector(selector, timeout=5000)
1023 except Exception:
1024 pass
1025 html = await page.content()
1026 if self.page_is_blocked(html):
1027 Actor.log.info(f"[{self.source_name}] soft-block page after load, trying salvage")
1028 await page.wait_for_timeout(2500 + random.randint(0, 1000))
1029 salvage = await self.on_blocked_page(page, html)
1030 if salvage:
1031 self._last_fetch_salvaged = True
1032 return salvage
1033 return None
1034 if len(html) < 3000:
1035 return None
1036 self._last_browser_extracted = []
1037 if self.use_js_fallback:
1038 try:
1039 self._last_browser_extracted = await page.evaluate(JS_EXTRACT_JOBS) or []
1040 except Exception:
1041 pass
1042 return html
1043 finally:
1044 try:
1045 await asyncio.wait_for(page.close(), timeout=5)
1046 except Exception:
1047 pass
1048
1049 async def _rotate_proxy(self):
1050 """New proxy session for this board; the warmed browser context is
1051 discarded with it so a burned session is never reused."""
1052 if self.browser_pool:
1053 await self.browser_pool.reset_context(self.source_name)
1054 if not self.proxy_config:
1055 return
1056 try:
1057 safe = re.sub(r"[^a-zA-Z0-9._~]", "_", self.source_name)
1058 self.proxy_url = await self.proxy_config.new_url(session_id=f"{safe}_{random.randint(10000, 99999)}")
1059 if self._client_headers is not None:
1060 old = self.client
1061 self.client = httpx.AsyncClient(headers=self._client_headers, proxy=self.proxy_url,
1062 timeout=HTML_TIMEOUT, follow_redirects=True)
1063 try:
1064 await old.aclose()
1065 except Exception:
1066 pass
1067 Actor.log.info(f"[{self.source_name}] rotated proxy session")
1068 except Exception as e:
1069 Actor.log.warning(f"[{self.source_name}] proxy rotation failed: {e}")
1070
1071 async def _polite_delay(self):
1072 await asyncio.sleep(self.delay + random.uniform(0.1, 0.5))
1073
1074
1075
1076 def _extract_salary_from_text(self, text: str) -> dict | None:
1077 return extract_salary_from_text(text, self.default_currency)
1078
1079 _LOC_PAT = re.compile(
1080 r"\b(?:london|manchester|birmingham|leeds|bristol|liverpool|sheffield|glasgow|edinburgh|cardiff|"
1081 r"newcastle|nottingham|southampton|oxford|cambridge|reading|brighton|bath|york|leicester|coventry|"
1082 r"city of london|west london|east london|central london|north london|south london|canary wharf|"
1083 r"remote|hybrid|on-?site|work from home|wfh|england|scotland|wales|[A-Z]{1,2}\d{1,2}\s*\d[A-Z]{2})\b",
1084 re.IGNORECASE)
1085 _DATE_PAT = re.compile(
1086 r"\b(\d+\s*(?:day|hour|minute|week|month)s?\s*ago|today|yesterday|just\s*(?:now|posted)|"
1087 r"\d{1,2}[/-]\d{1,2}[/-]\d{2,4}|(?:jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)\w*\s+\d{1,2}(?:\s*,?\s*\d{2,4})?)\b",
1088 re.IGNORECASE)
1089
1090 def _process_js_extracted(self) -> list[dict]:
1091 """Turn raw JS-extracted card data into job dicts using text heuristics."""
1092 jobs = []
1093 for raw in self._last_browser_extracted:
1094 title = clean_text(raw.get("title"))
1095 if not title:
1096 continue
1097 job = {"source": self.source_name, "title": title, "url": strip_tracking(raw.get("url", ""))}
1098 if raw.get("job_id"):
1099 job["job_id"] = str(raw["job_id"])
1100 else:
1101 m = re.search(r"/job/(\d+)|-job(\d+)|/(\d{5,})", job["url"])
1102 if m:
1103 job["job_id"] = next(g for g in m.groups() if g)
1104 company_from_link = clean_text(raw.get("_company_link_text"))
1105 if company_from_link:
1106 job["company"] = company_from_link
1107 card_text = raw.get("_card_text", "") or ""
1108 apply_salary(job, extract_salary_from_text(card_text, self.default_currency))
1109 company_candidates, location_candidates, snippet_candidates = [], [], []
1110 for seg in raw.get("_segments", []) or []:
1111 seg_clean = re.sub(r"<[^>]+>", "", seg).strip()
1112 if len(seg_clean) < 2 or seg_clean == title or seg_clean == company_from_link:
1113 continue
1114 if not job.get("date_posted") and len(seg_clean) < 40:
1115 dm = self._DATE_PAT.search(seg_clean)
1116 if dm:
1117 job["date_posted"] = dm.group(1).strip()
1118 continue
1119 if not job.get("employment_type") and len(seg_clean) < 40:
1120 et = detect_employment_type(seg_clean)
1121 if et:
1122 job["employment_type"] = et
1123 if len(seg_clean) < 20:
1124 continue
1125 if re.search(r"[£$€]\s*\d", seg_clean):
1126 continue
1127 if self._LOC_PAT.search(seg_clean) and len(seg_clean) < 80:
1128 if company_from_link and (seg_clean.lower() in company_from_link.lower()
1129 or company_from_link.lower() in seg_clean.lower()):
1130 continue
1131 location_candidates.append(seg_clean)
1132 continue
1133 (company_candidates if len(seg_clean) < 50 else snippet_candidates).append(seg_clean)
1134 if not job.get("employment_type"):
1135 job["employment_type"] = detect_employment_type(card_text)
1136 if not job.get("date_posted"):
1137 dm = self._DATE_PAT.search(card_text)
1138 if dm:
1139 job["date_posted"] = dm.group(1).strip()
1140 if not job.get("company"):
1141 for c in company_candidates:
1142 text = re.sub(r"^(?:Company|Posted by|Employer)\s*:?\s*", "", c, flags=re.IGNORECASE).strip()
1143 if 1 < len(text) < 80:
1144 job["company"] = clean_text(text)
1145 break
1146 if location_candidates:
1147 job["location"] = clean_text(re.sub(r"^Location\s*:?\s*", "", location_candidates[0], flags=re.IGNORECASE))
1148 if snippet_candidates:
1149 job["snippet"] = clean_text(re.sub(r"<[^>]+>", "", snippet_candidates[0]))[:500]
1150 job["work_mode"] = detect_work_mode(job.get("location"), card_text)
1151 jobs.append(job)
1152 return jobs
1153
1154 def _extract_jsonld_jobs(self, html_or_soup) -> list[dict]:
1155 soup = _as_soup(html_or_soup)
1156 jobs = []
1157 for script in soup.select('script[type="application/ld+json"]'):
1158 try:
1159 data = json.loads(script.string or "")
1160 except (json.JSONDecodeError, TypeError):
1161 continue
1162 for item in (data if isinstance(data, list) else [data]):
1163 if not isinstance(item, dict):
1164 continue
1165 t = item.get("@type")
1166 if t == "JobPosting":
1167 jobs.append(self._jsonld_to_job(item))
1168 elif t == "ItemList":
1169 for elem in item.get("itemListElement", []):
1170 posting = elem.get("item", elem) if isinstance(elem, dict) else {}
1171 if isinstance(posting, dict) and posting.get("@type") == "JobPosting":
1172 jobs.append(self._jsonld_to_job(posting))
1173 elif "@graph" in item:
1174 for g in item["@graph"]:
1175 if isinstance(g, dict) and g.get("@type") == "JobPosting":
1176 jobs.append(self._jsonld_to_job(g))
1177 return jobs
1178
1179 def _jsonld_to_job(self, p: dict) -> dict:
1180 job = {
1181 "source": self.source_name,
1182 "title": clean_text(p.get("title", "")),
1183 "url": strip_tracking(p.get("url", "")),
1184 "date_posted": p.get("datePosted", ""),
1185 "valid_through": p.get("validThrough", ""),
1186 "snippet": clean_text(re.sub(r"<[^>]+>", " ", p.get("description", "") or ""))[:500],
1187 "full_description": p.get("description", "") or "",
1188 }
1189 org = p.get("hiringOrganization", {})
1190 job["company"] = clean_text(org.get("name", "") if isinstance(org, dict) else str(org))
1191 loc = p.get("jobLocation", {})
1192 if isinstance(loc, list):
1193 loc = loc[0] if loc else {}
1194 if isinstance(loc, dict):
1195 addr = loc.get("address", {})
1196 if isinstance(addr, dict):
1197 job["location"] = ", ".join(x for x in (addr.get("addressLocality", ""), addr.get("addressRegion", "")) if x)
1198 elif isinstance(addr, str):
1199 job["location"] = addr
1200 sal = p.get("baseSalary", {})
1201 if isinstance(sal, dict):
1202 cur = sal.get("currency") or self.default_currency
1203 val = sal.get("value", {})
1204 if isinstance(val, dict):
1205 lo = val.get("minValue") or val.get("value")
1206 hi = val.get("maxValue") or lo
1207 unit = (val.get("unitText") or "YEAR").upper()
1208 period = {"YEAR": "annum", "MONTH": "month", "WEEK": "week", "DAY": "day", "HOUR": "hour"}.get(unit, "annum")
1209 try:
1210 lo = float(lo or hi or 0)
1211 hi = float(hi or lo or 0)
1212 except (TypeError, ValueError):
1213 lo = hi = 0.0
1214 if lo or hi:
1215 sym = CURRENCY_SYMBOL.get(cur, cur + " ")
1216 label = {"annum": "per annum", "month": "per month", "week": "per week", "day": "per day", "hour": "per hour"}[period]
1217 raw = f"{sym}{lo:,.0f} {label}" if lo == hi else f"{sym}{lo:,.0f} - {sym}{hi:,.0f} {label}"
1218 apply_salary(job, {"raw": raw, "min": lo, "max": hi, "currency": cur, "period": period})
1219 emp = p.get("employmentType", "")
1220 if isinstance(emp, list):
1221 emp = ", ".join(emp)
1222 job["employment_type"] = clean_text(str(emp)).replace("_", "-").title()
1223 if p.get("jobLocationType") == "TELECOMMUTE":
1224 job["work_mode"] = "Remote"
1225 ident = p.get("identifier", {})
1226 if isinstance(ident, dict) and ident.get("value"):
1227 job["job_id"] = str(ident["value"])
1228 elif isinstance(ident, str) and ident:
1229 job["job_id"] = ident
1230 elif job["url"]:
1231 m = re.search(r"/(\d{4,})", job["url"])
1232 if m:
1233 job["job_id"] = m.group(1)
1234 return job
1235
1236 @staticmethod
1237 def _extract_next_data(html_or_soup) -> dict | None:
1238 soup = _as_soup(html_or_soup)
1239 script = soup.select_one("script#__NEXT_DATA__")
1240 if script and script.string:
1241 try:
1242 return json.loads(script.string)
1243 except json.JSONDecodeError:
1244 return None
1245 return None
![🚀 Indeed jobs scraper [PPR] avatar](https://images.apifyusercontent.com/PEDOk9MmTf1yHtczA1A4wQEoGyAKCe5Aft1HzkhwROk/rs:fill:76:76/cb:1/aHR0cHM6Ly9hcGlmeS1pbWFnZS11cGxvYWRzLXByb2QuczMudXMtZWFzdC0xLmFtYXpvbmF3cy5jb20vTVhMcG5nbVZwRThXVEVTUXIvQjFrZ3BRS1p5dzJvVEpJUmctTG9nby1IZXJvLVN0YXRpYy1lemdpZi5jb20td2VicC10by1qcGctY29udmVydGVyLmpwZw.webp){kind=avatar}
