1"""OpenRice Hong Kong restaurant scraper — the second data source.
2
3Same shape as `hk_jobs.py`: a polite, dependency-light client over a public
4JSON endpoint, returning dataclasses that serialise straight to a dataset.
5
6Every behaviour below was verified against the live API. Each one breaks a
7naive scraper while still returning HTTP 200 or a plausible-looking dataset.
8
91. A browser-like ``User-Agent`` is mandatory. Requests without one get a
10 blanket 403 — the endpoint itself is open, the UA filter is the only gate.
112. The ``page`` parameter is silently ignored: every value returns the same
12 first rows. Real pagination is ``startAt`` (a row offset).
133. ``startAt + rows`` may not exceed 10,000; past that the API answers
14 HTTP 500. The city has ~29,000 open venues, so a query larger than the cap
15 is split into one query per district (the biggest district holds < 2,000).
164. Search results include moved and renovating venues (~13%). A moved venue
17 is listed again at its new address, so they are excluded by default. The
18 ``status`` filter runs on a search index that lags the records: about 2 in
19 10,000 rows still come back moved, renovating or unlabelled, so each row's
20 own status is checked again before it is kept.
215. ``reviewCount`` is the real review total. Smile + cry leaves out the
22 neutral ("OK") reviews and undercounts every venue.
236. The ``/r{poiId}`` restaurant URL is a 404, and HTML pages sit behind a bot
24 challenge that trips within a dozen requests. Only the JSON API is called;
25 links use the API's own ``shortenUrl``.
267. The JSON API has the same protection, only far more tolerant: after a
27 whole-city run on top of a day of probing, one IP was served a BytePlus
28 challenge page (HTTP 200, HTML) for 12-16 minutes, then JSON again. Retries
29 back off for over 20 minutes to outlast such a block, then raise
30 ``SourceUnavailable`` so the caller can stop cleanly with what it has.
31"""
32from __future__ import annotations
33
34import gzip
35import http.client
36import json
37import logging
38import time
39import urllib.parse
40import urllib.request
41from dataclasses import asdict, dataclass, field
42from datetime import datetime, timezone
43from typing import Iterator
44
45log = logging.getLogger(__name__)
46
47API = "https://www.openrice.com/api/v2/search"
48UA = (
49 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
50 "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
51)
52PAGE_ROWS = 200
53DEEP_LIMIT = 10_000
54
55
56
57RETRY_WAITS = (2, 4, 8, 16, 60, 120, 240, 300, 300, 300)
58MAX_RETRIES = len(RETRY_WAITS) + 1
59
60STATUS_OPEN = 10
61STATUS_LABELS = {10: "open", 4: "moved", 3: "under_renovation"}
62
63
64
65WEEKDAYS = {2: "mon", 3: "tue", 4: "wed", 5: "thu", 6: "fri", 7: "sat", 1: "sun"}
66HOURS_ORDER = [*WEEKDAYS.values(), "public_holiday", "public_holiday_eve"]
67
68
69PRICE_BANDS = {
70 1: ("Below $50", 0, 50),
71 2: ("$51-100", 51, 100),
72 3: ("$101-200", 101, 200),
73 4: ("$201-400", 201, 400),
74 5: ("$401-800", 401, 800),
75 6: ("Above $801", 801, None),
76}
77
78
79@dataclass
80class Restaurant:
81 poi_id: int
82 name: str
83 name_other_lang: str
84 address: str
85 address_other_lang: str
86 district: str | None
87 district_id: int | None
88 latitude: float | None
89 longitude: float | None
90 price_band: str | None
91 price_min_hkd: int | None
92 price_max_hkd: int | None
93 score_overall: float | None
94 smiles: int | None
95 cries: int | None
96 review_count: int | None
97 bookmark_count: int | None
98 cuisines: list[str] = field(default_factory=list)
99 dish_types: list[str] = field(default_factory=list)
100 phones: list[str] = field(default_factory=list)
101 opening_hours: dict[str, list[str]] | None = None
102 open_since: str | None = None
103 status: str | None = None
104 moved_to_poi_id: int | None = None
105 url: str | None = None
106 scraped_at: str = ""
107
108 def to_dict(self) -> dict:
109 return asdict(self)
110
111
112class RateLimiter:
113 """Polite fixed-delay limiter — one request per `delay` seconds."""
114
115 def __init__(self, delay: float = 1.0):
116 self.delay = delay
117 self._last = 0.0
118
119 def wait(self) -> None:
120 gap = time.monotonic() - self._last
121 if gap < self.delay:
122 time.sleep(self.delay - gap)
123 self._last = time.monotonic()
124
125
126class SourceUnavailable(RuntimeError):
127 """OpenRice kept refusing after every retry: bot protection or an outage."""
128
129
130def _request(params: dict, limiter: RateLimiter | None = None) -> dict:
131 """GET the search endpoint with retry/backoff, returning parsed JSON."""
132 url = f"{API}?{urllib.parse.urlencode(params)}"
133 headers = {
134 "User-Agent": UA,
135 "Accept": "application/json, text/plain, */*",
136 "Accept-Encoding": "gzip",
137 "Accept-Language": "en-HK,en;q=0.9",
138 "Referer": "https://www.openrice.com/en/hongkong",
139 }
140 last_err: Exception | None = None
141 for attempt in range(MAX_RETRIES):
142 if limiter:
143 limiter.wait()
144 try:
145 req = urllib.request.Request(url, headers=headers)
146 with urllib.request.urlopen(req, timeout=60) as resp:
147 raw = resp.read()
148 if resp.headers.get("Content-Encoding") == "gzip":
149 raw = gzip.decompress(raw)
150 return json.loads(raw)
151
152
153 except (OSError, ValueError, http.client.HTTPException) as exc:
154 last_err = exc
155 if attempt < len(RETRY_WAITS):
156 wait = RETRY_WAITS[attempt]
157 if wait >= 60:
158
159 reason = "non-JSON answer" if isinstance(exc, ValueError) else f"{type(exc).__name__}: {exc}"
160 log.warning(
161 f"OpenRice request failed ({reason}), possibly bot protection. "
162 f"Waiting {wait}s before attempt {attempt + 2} of {MAX_RETRIES}."
163 )
164 time.sleep(wait)
165 raise SourceUnavailable(f"OpenRice request failed after {MAX_RETRIES} attempts: {last_err}")
166
167
168def _categories(raw: dict, type_id: int) -> list[str]:
169 """Category names of one type — 1 is cuisine, 3 is dish/venue type."""
170 return [
171 c.get("name")
172 for c in (raw.get("categories") or [])
173 if c.get("categoryTypeId") == type_id and c.get("name")
174 ]
175
176
177def _opening_hours(raw: dict) -> dict[str, list[str]] | None:
178 """Weekly hours as ``{"mon": ["11:30-15:00", "18:00-22:30"], ...}``.
179
180 ``[]`` means closed that day. ``isClose`` must win over the period fields:
181 closed days still carry the regular hours in ``period1Start/End``, so
182 reading the periods first reports a closed Sunday as open. Date-specific
183 exceptions (``day``/``lunarDay``/``weekOfMonth``) are not a weekly pattern
184 and are skipped. Days the source does not state are left out, never
185 guessed.
186 """
187 found: dict[str, list[str]] = {}
188 for h in raw.get("poiHours") or []:
189 if h.get("day") or h.get("lunarDay") or h.get("weekOfMonth"):
190 continue
191 dow = h.get("dayOfWeek")
192 if dow in WEEKDAYS:
193 key = WEEKDAYS[dow]
194 elif dow == 0 and h.get("isHoliday"):
195 key = "public_holiday"
196 elif dow == 0 and h.get("isHolidayEve"):
197 key = "public_holiday_eve"
198 else:
199 continue
200 if key in found:
201 continue
202 if h.get("isClose"):
203 found[key] = []
204 elif h.get("is24hr"):
205 found[key] = ["00:00-24:00"]
206 else:
207 spans = []
208 n = 1
209 while f"period{n}Start" in h or f"period{n}End" in h:
210 start, end = h.get(f"period{n}Start"), h.get(f"period{n}End")
211 if start and end:
212 spans.append(f"{start[:5]}-{end[:5]}")
213 n += 1
214 if spans:
215 found[key] = spans
216 return {k: found[k] for k in HOURS_ORDER if k in found} or None
217
218
219def _parse(raw: dict, now: str) -> Restaurant:
220 district = raw.get("district") or {}
221 price_id = raw.get("priceRangeId")
222 band = PRICE_BANDS.get(price_id) if isinstance(price_id, int) else None
223 status_code = raw.get("status")
224 moved_to = raw.get("moveToId")
225
226 def _float(key: str) -> float | None:
227 try:
228 return float(raw[key])
229 except (KeyError, TypeError, ValueError):
230 return None
231
232 def _int(key: str) -> int | None:
233 try:
234 return int(raw[key])
235 except (KeyError, TypeError, ValueError):
236 return None
237
238 poi_id = raw.get("poiId")
239 score = _float("scoreOverall")
240 return Restaurant(
241 poi_id=int(poi_id) if poi_id is not None else 0,
242 name=raw.get("name") or "",
243 name_other_lang=raw.get("nameOtherLang") or "",
244 address=raw.get("address") or "",
245 address_other_lang=raw.get("addressOtherLang") or "",
246 district=district.get("name"),
247 district_id=district.get("districtId"),
248 latitude=_float("mapLatitude"),
249 longitude=_float("mapLongitude"),
250 price_band=band[0] if band else None,
251 price_min_hkd=band[1] if band else None,
252 price_max_hkd=band[2] if band else None,
253
254 score_overall=round(score, 2) if score else None,
255 smiles=_int("scoreSmile"),
256 cries=_int("scoreCry"),
257 review_count=_int("reviewCount"),
258 bookmark_count=_int("bookmarkedUserCount"),
259 cuisines=_categories(raw, 1),
260 dish_types=_categories(raw, 3),
261 phones=[p for p in (raw.get("phones") or []) if p],
262 opening_hours=_opening_hours(raw),
263 open_since=raw.get("openSince") or None,
264 status=STATUS_LABELS.get(status_code) if isinstance(status_code, int) else None,
265 moved_to_poi_id=moved_to if isinstance(moved_to, int) and moved_to > 0 else None,
266 url=raw.get("shortenUrl")
267 or (f"https://www.openrice.com/en/hongkong/r-restaurant-r{poi_id}" if poi_id else None),
268 scraped_at=now,
269 )
270
271
272def _params(
273 keywords: str | None,
274 district_id: str | None,
275 cuisine_id: str | None,
276 include_inactive: bool,
277) -> dict:
278 """Base query parameters shared by counting and paging."""
279 params: dict = {"uiLang": "en", "uiCity": "hongkong"}
280 if keywords:
281 params["whatwhere"] = keywords
282 if district_id:
283 params["districtId"] = district_id
284 if cuisine_id:
285 params["cuisineId"] = cuisine_id
286 if not include_inactive:
287 params["status"] = STATUS_OPEN
288 return params
289
290
291def _probe(base: dict, limiter: RateLimiter | None = None) -> tuple[int, list[dict]]:
292 """Match count plus the per-district breakdown — one cheap request.
293
294 The ``districts`` facet in a response is already narrowed by the query's
295 own filters, so its counts describe exactly the rows a split would fetch.
296 """
297 data = _request({**base, "rows": 1, "startAt": 0}, limiter)
298 count = int((data.get("paginationResult") or {}).get("count") or 0)
299 facets = (data.get("refineSearchFilter") or {}).get("districts") or []
300 return count, facets
301
302
303def _leaf_districts(facets: list[dict], exclude: str | None = None) -> list[str]:
304 """District ids that partition a query without overlap.
305
306 Ids ending in 999 are whole regions and negative ids are named sub-areas
307 (Soho, Lan Kwai Fong, ...) that overlap the real districts; neither may be
308 used as a partition or venues would be fetched twice.
309 """
310 out = []
311 for facet in facets:
312 fid = facet.get("id")
313 if not isinstance(fid, int) or fid <= 0 or fid % 1000 == 999:
314 continue
315 if not facet.get("count") or str(fid) == str(exclude):
316 continue
317 out.append(str(fid))
318 return out
319
320
321def _paginate(
322 base: dict,
323 limiter: RateLimiter,
324 seen: set[int],
325 max_items: int,
326 now: str,
327) -> Iterator[Restaurant]:
328 """Walk one query by `startAt` offset, never past the 10,000-row cap.
329
330 Stops when the API runs out of rows or stops producing anything new, so a
331 silently capped result set ends the walk instead of spinning forever.
332 """
333 want_status = base.get("status")
334 dropped: set[int] = set()
335 start_at = 0
336 while len(seen) < max_items and start_at < DEEP_LIMIT:
337
338 rows_wanted = min(PAGE_ROWS, DEEP_LIMIT - start_at, max_items - len(seen))
339 data = _request({**base, "rows": rows_wanted, "startAt": start_at}, limiter)
340 rows = (data.get("paginationResult") or {}).get("results") or []
341 if not rows:
342 return
343
344 fresh = 0
345 for raw in rows:
346 poi_id = raw.get("poiId")
347 if poi_id is None or poi_id in seen or poi_id in dropped:
348 continue
349 fresh += 1
350 if want_status is not None and raw.get("status") != want_status:
351 dropped.add(poi_id)
352 continue
353 seen.add(poi_id)
354 yield _parse(raw, now)
355 if len(seen) >= max_items:
356 return
357
358 if fresh == 0:
359 return
360
361
362 start_at += len(rows)
363
364
365def total_count(
366 keywords: str | None = None,
367 district_id: str | None = None,
368 cuisine_id: str | None = None,
369 include_inactive: bool = False,
370) -> int:
371 """How many restaurants match — one cheap request."""
372 count, _ = _probe(_params(keywords, district_id, cuisine_id, include_inactive))
373 return count
374
375
376def scrape_restaurants(
377 keywords: str | None = None,
378 district_id: str | None = None,
379 cuisine_id: str | None = None,
380 max_items: int = 200,
381 delay: float = 1.0,
382 include_inactive: bool = False,
383) -> Iterator[Restaurant]:
384 """Yield restaurants, de-duplicated by poi_id, until `max_items`.
385
386 Up to 10,000 results come from one query in the source's own order
387 (best-rated first). Beyond that the query is split into one query per
388 district, which reaches every venue that has a district (~99.6%).
389 """
390 limiter = RateLimiter(delay)
391 now = datetime.now(timezone.utc).isoformat(timespec="seconds")
392 seen: set[int] = set()
393 base = _params(keywords, district_id, cuisine_id, include_inactive)
394
395 if max_items <= DEEP_LIMIT:
396 yield from _paginate(base, limiter, seen, max_items, now)
397 return
398
399 count, facets = _probe(base, limiter)
400 leaves = _leaf_districts(facets, exclude=district_id)
401 if count <= DEEP_LIMIT or not leaves:
402
403
404 yield from _paginate(base, limiter, seen, max_items, now)
405 return
406
407 for did in leaves:
408 if len(seen) >= max_items:
409 return
410 yield from _paginate({**base, "districtId": did}, limiter, seen, max_items, now)