1"""Core scraping + normalization for AU grocery chains (Coles, Woolworths, ALDI, IGA).
2
3No Apify imports here so it can be tested locally. Each fetcher returns
4(rows, errors) where rows are normalized product dicts and errors are strings.
5"""
6from __future__ import annotations
7
8import json
9import random
10import re
11import time
12from datetime import datetime, timezone
13from urllib.parse import quote
14
15from curl_cffi import requests as creq
16
17AL = {"Accept-Language": "en-AU,en;q=0.9"}
18UA_IMP = "chrome"
19
20
21
22def proxy_url(mode: str | None, password: str | None) -> str | None:
23 """Build an Apify proxy URL for a mode: None | 'datacenter' | 'residential-au'.
24
25 `password` may be either the raw Apify proxy password (legacy) or a dict of
26 pre-built proxy URLs supplied by the Actor wrapper via
27 Actor.create_proxy_configuration(), which is the reliable path on-platform.
28 """
29 if mode is None or not password:
30 return None
31 if isinstance(password, dict):
32 return password.get(mode)
33 sess = f"session-{random.randint(100000, 999999)}"
34 if mode == "datacenter":
35 user = f"groups-BUYPROXIES94952,{sess}"
36 elif mode == "residential-au":
37 user = f"groups-RESIDENTIAL,country-AU,{sess}"
38 else:
39 return None
40 return f"http://{user}:{password}@proxy.apify.com:8000"
41
42
43def new_session(mode: str | None, password: str | None) -> creq.Session:
44 px = proxy_url(mode, password)
45 kwargs = {"impersonate": UA_IMP}
46 if px:
47 kwargs["proxies"] = {"http": px, "https": px}
48 return creq.Session(**kwargs)
49
50
51
52
53_STOP = {"the", "and", "with", "of", "per", "pack", "pk", "x", "each", "ea",
54 "value", "multipack", "&"}
55
56def _norm_space(s):
57 return re.sub(r"\s+", " ", (s or "").strip())
58
59
60def parse_size(text: str | None):
61 """Parse a size string to {'value', 'unit' ('g'|'ml'|'ea'), 'approx'}."""
62 if not text:
63 return None
64 t = text.lower().replace("litres", "l").replace("litre", "l").replace("liter", "l")
65 approx = "approx" in t
66 m = re.search(r"(\d+(?:\.\d+)?)\s*(kg|ml|g|l)\b\s*x\s*(\d+)", t)
67 if m:
68 return _mk_size(float(m[1]) * int(m[3]), m[2], approx)
69 m = re.search(r"(\d+)\s*x\s*(\d+(?:\.\d+)?)\s*(kg|ml|g|l)\b", t)
70 if m:
71 return _mk_size(float(m[2]) * int(m[1]), m[3], approx)
72 m = re.search(r"(\d+(?:\.\d+)?)\s*(kg|ml|g|l)\b", t)
73 if m:
74 return _mk_size(float(m[1]), m[2], approx)
75 m = re.search(r"(\d+)\s*(?:pack|pk|pce|pieces?|sheets?|each|ea)\b", t)
76 if m:
77 return {"value": float(m[1]), "unit": "ea", "approx": approx}
78 return None
79
80
81def _mk_size(v, u, approx):
82 if u == "kg":
83 v, u = v * 1000, "g"
84 elif u == "l":
85 v, u = v * 1000, "ml"
86 return {"value": round(v, 2), "unit": u, "approx": approx}
87
88
89def size_display(sn):
90 if not sn:
91 return None
92 v = sn["value"]
93 v = int(v) if v == int(v) else v
94 return f"{v}{sn['unit']}"
95
96
97HOUSE_BRANDS = {
98 "coles": {"coles", "coles simply", "coles finest", "coles organic", "coles kitchen"},
99 "woolworths": {"woolworths", "essentials", "macro", "woolworths cook", "woolworths float"},
100 "aldi": {"farmdale", "lodge farms", "westacre", "westacre dairy", "brooklea",
101 "yoguri", "premiere", "inner goodness", "food envy", "lyttos",
102 "snackers market", "sunnyvale", "pure valley", "remano",
103 "emporium selection", "world kitchen", "specially selected",
104 "goldenvale", "damora", "bakers life", "beautifully butterfully",
105 "sprinters", "confidence", "mamia", "cowbelle", "colway",
106 "bramwells", "hillcrest", "imperial grain", "corale", "just organic"},
107 "iga": {"community co", "black & gold", "black and gold", "iga"},
108}
109
110
111
112VARIANT_MARKERS = [
113
114 {"original", "classic"}, {"dark"}, {"white"}, {"double", "doublecoat"},
115 {"caramel"}, {"chewy"}, {"mint"}, {"latte", "cafe"}, {"vovo"}, {"jatz"},
116 {"triple", "decadent"}, {"berry", "wildberry", "wildberries"},
117 {"honey"}, {"cranberry"}, {"coconut"}, {"vanilla"}, {"strawberry"},
118 {"chocolate"}, {"salted"}, {"raspberry"}, {"lemon"}, {"apple"},
119
120 {"lactose"}, {"gluten"}, {"organic"}, {"free-range", "freerange"},
121 {"skim", "lite", "light"}, {"uht", "longlife"}, {"a2"},
122 {"cholesterol"}, {"protein"}, {"multigrain", "grain", "grains"},
123
124 {"2ply", "2-ply"}, {"3ply", "3-ply"}, {"4ply", "4-ply"},
125 {"kingsize", "king"}, {"scented"}, {"unscented"}, {"recycled"},
126 {"wipes", "wipe"}, {"flushable"}, {"cage"}, {"caged"},
127 {"jumbo"}, {"extra"}, {"large"}, {"small"},
128]
129
130
131def norm_brand(b):
132 return re.sub(r"[^a-z0-9 ]", "", (b or "").lower()).strip()
133
134
135def is_house(chain, brand):
136 return norm_brand(brand) in HOUSE_BRANDS.get(chain, set())
137
138
139def name_tokens(name, brand):
140 t = re.sub(r"[^a-z0-9. ]", " ", (name or "").lower())
141 t = re.sub(r"(\d)\s*ply", r"\1ply", t)
142 t = t.replace("king size", "kingsize").replace("free range", "freerange")
143 toks = set(t.split())
144 toks -= set(norm_brand(brand).split())
145 toks -= _STOP
146 toks = {x for x in toks if not re.fullmatch(r"[\d.]+(kg|g|ml|l)?", x)}
147 return toks
148
149
150def variant_signature(tokens):
151 """Which variant markers this product asserts, as a frozenset of group indices."""
152 sig = set()
153 for i, group in enumerate(VARIANT_MARKERS):
154 if tokens & group:
155 sig.add(i)
156 return frozenset(sig)
157
158
159def _now():
160 return datetime.now(timezone.utc).isoformat(timespec="seconds")
161
162
163def _money(s):
164 if s is None:
165 return None
166 m = re.search(r"\$?\s*([\d,]+\.?\d*)", str(s))
167 return float(m[1].replace(",", "")) if m else None
168
169
170
171
172class Woolworths:
173 chain = "woolworths"
174 chain_name = "Woolworths"
175 PAGE = 36
176
177 def __init__(self, proxy_password=None):
178 self.pw = proxy_password
179 self.modes = [None, "datacenter", "residential-au"]
180 self.mode_i = 0
181 self.sess = None
182
183 def _fresh(self, bump=False):
184 if bump and self.mode_i < len(self.modes) - 1:
185 self.mode_i += 1
186 self.sess = new_session(self.modes[self.mode_i], self.pw)
187 try:
188 self.sess.get("https://www.woolworths.com.au/", headers=AL, timeout=30)
189 except Exception:
190 pass
191
192 def _get(self, url):
193 for attempt in range(4):
194 if self.sess is None:
195 self._fresh()
196 try:
197 r = self.sess.get(url, headers={**AL, "Accept": "application/json",
198 "Referer": "https://www.woolworths.com.au/"},
199 timeout=30)
200 if r.status_code == 200 and r.headers.get("content-type", "").startswith("application/json"):
201 return r.json()
202 except Exception:
203 pass
204 self._fresh(bump=attempt >= 1)
205 raise RuntimeError(f"woolworths: gave up on {url[:120]}")
206
207 def search(self, keyword, max_n):
208 rows, errors, page = [], [], 1
209 while len(rows) < max_n:
210 url = ("https://www.woolworths.com.au/apis/ui/Search/products"
211 f"?searchTerm={quote(keyword)}&pageSize={self.PAGE}&pageNumber={page}")
212 try:
213 d = self._get(url)
214 except RuntimeError as e:
215 errors.append(str(e))
216 break
217 groups = d.get("Products") or []
218 prods = [p for g in groups for p in (g.get("Products") or [])]
219 if not prods:
220 break
221 for p in prods:
222 rows.append(self._norm(p, keyword))
223 if len(rows) >= max_n:
224 break
225 total = d.get("SearchResultsCount") or 0
226 if page * self.PAGE >= total:
227 break
228 page += 1
229 return rows, errors
230
231 def _norm(self, p, keyword):
232 ct = p.get("CentreTag") or {}
233 ht = p.get("HeaderTag") or {}
234 price, was = p.get("Price"), p.get("WasPrice")
235 mb = ct.get("MultibuyData")
236 multibuy = None
237 if mb:
238 multibuy = {"quantity": mb.get("Quantity"), "totalPrice": mb.get("Price"),
239 "unitPriceText": mb.get("CupTag")}
240 member = ct.get("MemberPriceData") or None
241 on_special = bool(p.get("IsOnSpecial") or p.get("IsHalfPrice")
242 or multibuy or (was and price and was > price))
243 special_type = None
244 if p.get("IsHalfPrice"):
245 special_type = "half_price"
246 elif multibuy:
247 special_type = "multi_buy"
248 elif p.get("IsOnSpecial"):
249 special_type = "special"
250 elif (ht or {}).get("Promotion") in ("LowPrice", "LowerShelfPrice"):
251 special_type = str(ht.get("Promotion"))
252 attrs = p.get("AdditionalAttributes") or {}
253 cat = attrs.get("sapcategoryname")
254 if not cat:
255 try:
256 cat = (json.loads(attrs.get("piesdepartmentnamesjson") or "[]") or [None])[0]
257 except Exception:
258 cat = None
259 sn = parse_size(p.get("PackageSize"))
260 per_kg = (p.get("Unit") == "KG")
261 return {
262 "type": "product", "chain": self.chain, "chainName": self.chain_name,
263 "keyword": keyword, "productId": str(p.get("Stockcode")),
264 "barcode": p.get("Barcode"),
265 "name": _norm_space(p.get("DisplayName") or p.get("Name")),
266 "brand": p.get("Brand"), "size": p.get("PackageSize"),
267 "sizeNormalised": size_display(sn), "_sn": sn,
268 "price": price, "wasPrice": was if (was and price and was > price) else None,
269 "priceUnit": "per kg" if per_kg else "each",
270 "isOnSpecial": on_special, "specialType": special_type,
271 "saveAmount": p.get("SavingsAmount") or None, "savePercent": None,
272 "offerDescription": ct.get("TagContentText") if multibuy else None,
273 "multiBuy": multibuy, "memberPrice": member, "specialEndsAt": None,
274 "unitPrice": p.get("CupPrice"), "unitPriceText": p.get("CupString"),
275 "available": bool(p.get("IsAvailable")), "category": cat,
276 "url": f"https://www.woolworths.com.au/shop/productdetails/{p.get('Stockcode')}/{p.get('UrlFriendlyName') or ''}",
277 "imageUrl": p.get("MediumImageFile"), "scrapedAt": _now(),
278 }
279
280
281
282
283class Coles:
284 chain = "coles"
285 chain_name = "Coles"
286
287 def __init__(self, proxy_password=None, cached_bid=None):
288 self.pw = proxy_password
289 self.modes = [None, "datacenter", "residential-au"]
290 self.mode_i = 0
291 self.sess = None
292 self.bid = cached_bid
293 self.bid_refreshed = False
294
295 def _fresh(self, bump=False):
296 if bump and self.mode_i < len(self.modes) - 1:
297 self.mode_i += 1
298 self.sess = new_session(self.modes[self.mode_i], self.pw)
299
300 def _fetch_bid(self):
301 for attempt in range(5):
302 if self.sess is None:
303 self._fresh()
304 try:
305 r = self.sess.get("https://www.coles.com.au/", headers=AL, timeout=30)
306 m = re.search(r'<script id="__NEXT_DATA__"[^>]*>(.*?)</script>', r.text, re.S)
307 print(f"[coles] bid fetch attempt {attempt} mode={self.modes[self.mode_i]} "
308 f"status={r.status_code} len={len(r.text)} nd={bool(m)}", flush=True)
309 if m:
310 self.bid = json.loads(m[1])["buildId"]
311 self.bid_refreshed = True
312 return
313 except Exception as e:
314 print(f"[coles] bid fetch attempt {attempt} mode={self.modes[self.mode_i]} exc={e}", flush=True)
315 self._fresh(bump=True)
316 time.sleep(0.8)
317 raise RuntimeError("coles: could not obtain buildId (blocked on all proxy modes)")
318
319 def _data(self, path):
320 """GET a Next.js data URL, escalating sessions/proxies; refresh bid on 404."""
321 bid_retried = False
322 for attempt in range(6):
323 if self.bid is None:
324 self._fetch_bid()
325 if self.sess is None:
326 self._fresh()
327 url = f"https://www.coles.com.au/_next/data/{self.bid}/en/{path}"
328 try:
329 r = self.sess.get(url, headers={**AL, "x-nextjs-data": "1"}, timeout=30)
330 ct = r.headers.get("content-type", "")
331 print(f"[coles] data attempt {attempt} mode={self.modes[self.mode_i]} "
332 f"status={r.status_code} ct={ct[:24]}", flush=True)
333 if r.status_code == 200 and ct.startswith("application/json"):
334 return r.json()
335 if r.status_code == 404 and not bid_retried:
336 bid_retried = True
337 self.bid = None
338 continue
339 except Exception as e:
340 print(f"[coles] data attempt {attempt} mode={self.modes[self.mode_i]} exc={e}", flush=True)
341 self._fresh(bump=attempt >= 1)
342 time.sleep(0.5 + random.random())
343 raise RuntimeError(f"coles: gave up on {path[:100]}")
344
345 def search(self, keyword, max_n):
346 rows, errors, page = [], [], 1
347 while len(rows) < max_n:
348 try:
349 d = self._data(f"search/products.json?q={quote(keyword)}&page={page}")
350 except RuntimeError as e:
351 errors.append(str(e))
352 break
353 sr = (d.get("pageProps") or {}).get("searchResults") or {}
354 prods = [p for p in (sr.get("results") or []) if p.get("_type") == "PRODUCT"]
355 if not prods:
356 break
357 for p in prods:
358 rows.append(self._norm(p, keyword))
359 if len(rows) >= max_n:
360 break
361 if (sr.get("start", 0) + sr.get("pageSize", 48)) >= (sr.get("noOfResults") or 0):
362 break
363 page += 1
364 time.sleep(0.6 + random.random() * 0.6)
365 return rows, errors
366
367 def _norm(self, p, keyword):
368 pr = p.get("pricing") or {}
369 was = pr.get("was") or None
370 promo = pr.get("promotionType")
371 on_special = promo == "SPECIAL"
372 mb = pr.get("multiBuyPromotion")
373 multibuy = None
374 if mb:
375 multibuy = {"quantity": mb.get("minQuantity") or mb.get("quantity"),
376 "totalPrice": mb.get("reward") or mb.get("price"),
377 "unitPriceText": None}
378 special_type = None
379 if on_special:
380 st = pr.get("specialType")
381 desc = (pr.get("priceDescription") or "")
382 if st == "PERCENT_OFF" and (pr.get("savePercent") == 50 or "1/2" in desc):
383 special_type = "half_price"
384 elif st == "MULTI_SAVE" or multibuy:
385 special_type = "multi_buy"
386 else:
387 special_type = (st or "special").lower()
388 unit = pr.get("unit") or {}
389 heirs = (p.get("onlineHeirs") or [{}])[0]
390 sn = parse_size(p.get("size"))
391 return {
392 "type": "product", "chain": self.chain, "chainName": self.chain_name,
393 "keyword": keyword, "productId": str(p.get("id")), "barcode": None,
394 "name": _norm_space(f'{p.get("brand") or ""} {p.get("name") or ""}'),
395 "brand": p.get("brand"), "size": p.get("size"),
396 "sizeNormalised": size_display(sn), "_sn": sn,
397 "price": pr.get("now"), "wasPrice": was,
398 "priceUnit": "each",
399 "isOnSpecial": on_special, "specialType": special_type,
400 "saveAmount": pr.get("saveAmount"), "savePercent": pr.get("savePercent"),
401 "offerDescription": pr.get("offerDescription") or pr.get("priceDescription"),
402 "multiBuy": multibuy, "memberPrice": None, "specialEndsAt": None,
403 "unitPrice": unit.get("price"), "unitPriceText": pr.get("comparable"),
404 "available": bool(p.get("availability")),
405 "category": heirs.get("aisle") or heirs.get("category"),
406 "url": f"https://www.coles.com.au/product/{p.get('id')}",
407 "imageUrl": ("https://productimages.coles.com.au/productimages"
408 + (p.get("imageUris") or [{}])[0].get("uri", "")) if p.get("imageUris") else None,
409 "scrapedAt": _now(),
410 }
411
412
413
414
415class Aldi:
416 chain = "aldi"
417 chain_name = "ALDI"
418 LIMITS = [60, 48, 32, 30, 24, 16, 12]
419
420 def __init__(self, proxy_password=None):
421 self.pw = proxy_password
422 self.sess = new_session(None, None)
423 self.mode_i = 0
424
425 def _get(self, url):
426 for attempt in range(3):
427 try:
428 r = self.sess.get(url, headers={**AL, "Accept": "application/json",
429 "Origin": "https://www.aldi.com.au"}, timeout=30)
430 if r.status_code == 200:
431 return r.json()
432 except Exception:
433 pass
434 self.mode_i = min(self.mode_i + 1, 1)
435 self.sess = new_session("datacenter" if self.mode_i else None, self.pw)
436 raise RuntimeError(f"aldi: gave up on {url[:120]}")
437
438 def search(self, keyword, max_n):
439 rows, errors, offset = [], [], 0
440 while len(rows) < max_n:
441 remaining = max_n - len(rows)
442 limit = next((l for l in self.LIMITS if l <= max(remaining, 12)), 12)
443 url = ("https://api.aldi.com.au/v3/product-search?currency=AUD"
444 f"&serviceType=walk-in&q={quote(keyword)}&limit={limit}&offset={offset}")
445 try:
446 d = self._get(url)
447 except RuntimeError as e:
448 errors.append(str(e))
449 break
450 data = d.get("data") or []
451 if not data:
452 break
453 for p in data:
454 rows.append(self._norm(p, keyword))
455 if len(rows) >= max_n:
456 break
457 total = ((d.get("meta") or {}).get("pagination") or {}).get("totalCount") or 0
458 offset += limit
459 if offset >= total:
460 break
461 return rows, errors
462
463 def _norm(self, p, keyword):
464 pr = p.get("price") or {}
465 price = (pr.get("amountRelevant") or pr.get("amount") or 0) / 100 or None
466 was = _money(pr.get("wasPriceDisplay"))
467 unit_price = (pr.get("comparison") / 100) if pr.get("comparison") else None
468 sn = parse_size(p.get("sellingSize"))
469 cats = p.get("categories") or []
470 return {
471 "type": "product", "chain": self.chain, "chainName": self.chain_name,
472 "keyword": keyword, "productId": p.get("sku"), "barcode": None,
473 "name": _norm_space(f'{p.get("brandName") or ""} {p.get("name") or ""}'),
474 "brand": p.get("brandName"), "size": p.get("sellingSize"),
475 "sizeNormalised": size_display(sn), "_sn": sn,
476 "price": price, "wasPrice": was, "priceUnit": "each",
477 "isOnSpecial": bool(was), "specialType": "special" if was else None,
478 "saveAmount": _money(pr.get("savingsDisplay")), "savePercent": None,
479 "offerDescription": None, "multiBuy": None, "memberPrice": None,
480 "specialEndsAt": None,
481 "unitPrice": unit_price, "unitPriceText": pr.get("comparisonDisplay"),
482 "available": not p.get("discontinued", False),
483 "category": " / ".join(c.get("name", "") for c in cats[:2]) or None,
484 "url": f"https://www.aldi.com.au/product/{p.get('urlSlugText')}",
485 "imageUrl": ((p.get("assets") or [{}])[0].get("url") or "").replace("{width}", "600").replace("{slug}", p.get("urlSlugText") or "") or None,
486 "scrapedAt": _now(),
487 }
488
489
490
491
492class Iga:
493 chain = "iga"
494 chain_name = "IGA"
495
496 def __init__(self, proxy_password=None, store_id="32600"):
497 self.pw = proxy_password
498 self.store = store_id
499 self.sess = new_session(None, None)
500 self.mode_i = 0
501
502 def _get(self, url):
503 for attempt in range(3):
504 try:
505 r = self.sess.get(url, headers={**AL, "Accept": "application/json"}, timeout=30)
506 if r.status_code == 200:
507 return r.json()
508 except Exception:
509 pass
510 self.mode_i = min(self.mode_i + 1, 1)
511 self.sess = new_session("datacenter" if self.mode_i else None, self.pw)
512 raise RuntimeError(f"iga: gave up on {url[:120]}")
513
514 def search(self, keyword, max_n):
515 rows, errors, skip = [], [], 0
516 while len(rows) < max_n:
517 take = min(100, max_n - len(rows))
518 url = (f"https://www.igashop.com.au/api/storefront/stores/{self.store}/search"
519 f"?q={quote(keyword)}&take={take}&skip={skip}")
520 try:
521 d = self._get(url)
522 except RuntimeError as e:
523 errors.append(str(e))
524 break
525 items = d.get("items") or []
526 if not items:
527 break
528 for p in items:
529 rows.append(self._norm(p, keyword))
530 if len(rows) >= max_n:
531 break
532 skip += take
533 if skip >= (d.get("total") or 0):
534 break
535 return rows, errors
536
537 def _norm(self, p, keyword):
538 tpr = (p.get("tprPrice") or [{}])
539 tpr0 = tpr[0] if tpr else {}
540 on_special = p.get("priceSource") == "tpr"
541 label = (p.get("priceLabel") or "").lower()
542 special_type = None
543 if on_special:
544 special_type = ("half_price" if ("half" in label or "1/2" in label)
545 else (label or "special"))
546 uos = p.get("unitOfSize") or {}
547 size_txt = f'{uos.get("size")}{uos.get("abbreviation") or ""}' if uos.get("size") else None
548 sn = parse_size(size_txt) if size_txt else None
549 if sn is None and uos.get("type") == "each" and uos.get("size"):
550 sn = {"value": float(uos["size"]), "unit": "ea", "approx": False}
551 cat = ((p.get("defaultCategory") or [{}])[0].get("categoryBreadcrumb"))
552 return {
553 "type": "product", "chain": self.chain, "chainName": self.chain_name,
554 "keyword": keyword, "productId": str(p.get("sku")), "barcode": None,
555 "name": _norm_space(p.get("name")), "brand": p.get("brand") or None,
556 "size": size_txt, "sizeNormalised": size_display(sn), "_sn": sn,
557 "price": p.get("priceNumeric"), "wasPrice": p.get("wasWholePrice"),
558 "priceUnit": "per kg" if p.get("sellBy") == "EachUnit" else "each",
559 "isOnSpecial": on_special, "specialType": special_type,
560 "saveAmount": (round(p["wasWholePrice"] - p["priceNumeric"], 2)
561 if on_special and p.get("wasWholePrice") and p.get("priceNumeric") else None),
562 "savePercent": None,
563 "offerDescription": p.get("priceLabel") or None,
564 "multiBuy": None, "memberPrice": None,
565 "specialEndsAt": tpr0.get("effectiveUntil"),
566 "unitPrice": _money(p.get("pricePerUnit")),
567 "unitPriceText": p.get("pricePerUnit"),
568 "available": bool(p.get("available")), "category": cat,
569 "url": f"https://www.igashop.com.au/product/{p.get('sku')}",
570 "imageUrl": (p.get("image") or {}).get("default"),
571 "scrapedAt": _now(),
572 }
573
574
575FETCHERS = {"woolworths": Woolworths, "coles": Coles, "aldi": Aldi, "iga": Iga}
576
577
578
579
580def _brand_key(chain, brand):
581 if is_house(chain, brand):
582 return "HOUSE"
583 return norm_brand(brand)
584
585
586def _compatible(a, b):
587 ba, bb = _brand_key(a["chain"], a["brand"]), _brand_key(b["chain"], b["brand"])
588 if ba == "HOUSE" and bb == "HOUSE":
589 pass
590 elif ba and bb and (ba == bb or ba in bb or bb in ba):
591 pass
592 else:
593 return False
594 ta, tb = a["_tokens"], b["_tokens"]
595 if not ta or not tb:
596 return False
597
598
599 if a["_vsig"] != b["_vsig"]:
600 return False
601 inter = len(ta & tb)
602 jac = inter / len(ta | tb)
603 return jac >= 0.4 and (inter >= 2 or ta == tb)
604
605
606def build_comparisons(products):
607 """Group same product across chains. Conservative: same normalized size,
608 compatible brand (house brands are mutually comparable), name-token overlap."""
609 for p in products:
610 p["_tokens"] = name_tokens(p["name"], p["brand"])
611 p["_vsig"] = variant_signature(p["_tokens"])
612 buckets = {}
613 for p in products:
614 sn = p.get("_sn")
615 if not sn:
616 continue
617 buckets.setdefault((sn["unit"], round(sn["value"])), []).append(p)
618 comparisons = []
619 for key, plist in buckets.items():
620 used = set()
621 for i, a in enumerate(plist):
622 if i in used:
623 continue
624 group = [a]
625 gchains = {a["chain"]}
626 for j in range(i + 1, len(plist)):
627 b = plist[j]
628 if j in used or b["chain"] in gchains:
629 continue
630 if _compatible(a, b):
631 group.append(b)
632 gchains.add(b["chain"])
633 used.add(j)
634 if len(group) >= 2:
635 used.add(i)
636 priced = [g for g in group if g.get("price")]
637 cheapest = min(priced, key=lambda g: g["price"]) if priced else None
638 comparisons.append({
639 "type": "comparison",
640 "name": max(group, key=lambda g: len(g["name"]))["name"],
641 "sizeNormalised": size_display(a["_sn"]),
642 "chains": sorted(gchains),
643 "cheapestChain": cheapest and cheapest["chain"],
644 "cheapestPrice": cheapest and cheapest["price"],
645 "priceSpread": (round(max(g["price"] for g in priced)
646 - min(g["price"] for g in priced), 2)
647 if len(priced) >= 2 else None),
648 "products": {g["chain"]: {k: v for k, v in g.items()
649 if not k.startswith("_") and k != "type"}
650 for g in group},
651 "scrapedAt": _now(),
652 })
653 return comparisons
654
655
656def strip_private(p):
657 return {k: v for k, v in p.items() if not k.startswith("_")}
