1"""Airbnb scraper: market search with recursive map-tile subdivision, listing detail,
2availability calendar and dated pricing. Talks to Airbnb's internal GraphQL (the same
3persisted queries the website uses) over an Apify residential proxy."""
4
5from __future__ import annotations
6
7import asyncio
8import base64
9import json
10import re
11from typing import Any
12
13from apify import Actor
14
15from .http_client import RequestClient
16
17API_KEY = "d306zoyjsyarp7ifhu67rjxn52tv0t20"
18
19
20
21DEFAULT_HASHES = {
22 "StaysSearch": "aa52154ae19d9c581fa59773a72c719f4844df441b9253fdb46f5f3da5b836ed",
23 "PdpAvailabilityCalendar": "8f08e03c7bd16fcad3c92a3592c19a8b559a0d0855a84028d1163d4733ed9ade",
24 "StaysPdpBookItQuery": "dbb612ce6e09072ae8f2e9364a2b07d63d18f43b01128cc4d62d0c1e3a916fc8",
25}
26
27TREATMENT_FLAGS = [
28 "feed_map_decouple_m11_treatment", "recommended_amenities_2024_treatment_b",
29 "filter_redesign_2024_treatment", "filter_reordering_2024_roomtype_treatment",
30 "p2_category_bar_removal_treatment", "selected_filters_2024_treatment",
31 "recommended_filters_2024_treatment_b", "m13_search_input_phase2_treatment",
32 "m13_search_input_services_enabled", "m13_2025_experiences_p2_treatment",
33 "homes_p25_refresh_2025_treatment",
34]
35
36BASE_HEADERS = {
37 "x-airbnb-api-key": API_KEY,
38 "content-type": "application/json",
39 "accept": "*/*",
40 "accept-language": "en-US,en;q=0.9",
41 "x-airbnb-graphql-platform": "web",
42 "x-airbnb-graphql-platform-client": "minimalist-niobe",
43 "x-csrf-without-token": "1",
44}
45
46_ROOM_ID_RE = re.compile(r"/rooms/(?:plus/)?(\d+)")
47_HASH_RE_TMPL = r"name:'{op}',type:'query',operationId:'([a-f0-9]{{64}})'"
48_RATING_RE = re.compile(r"([0-9]+[.,][0-9]+)\D+([0-9][0-9,]*)")
49
50
51def _decode_listing_id(encoded: str | None) -> str | None:
52 if not encoded:
53 return None
54 try:
55 decoded = base64.b64decode(encoded).decode()
56 return decoded.split(":")[-1]
57 except Exception:
58 return None
59
60
61def listing_id_from_url(url: str) -> str | None:
62 m = _ROOM_ID_RE.search(url)
63 return m.group(1) if m else None
64
65
66class AirbnbScraper:
67 def __init__(self, client: RequestClient, currency: str = "USD", locale: str = "en",
68 hash_cache: dict | None = None) -> None:
69 self.client = client
70 self.currency = currency
71 self.locale = locale
72 self.hashes = dict(DEFAULT_HASHES)
73 if hash_cache:
74 self.hashes.update({k: v for k, v in hash_cache.items() if v})
75 self._hash_lock = asyncio.Lock()
76
77
78 async def _resolve_hash(self, operation: str) -> str | None:
79 """Fetch the current operationId for a GraphQL operation from Airbnb's JS bundles."""
80 async with self._hash_lock:
81 pages = {
82 "StaysSearch": "https://www.airbnb.com/s/homes",
83 }.get(operation, "https://www.airbnb.com/rooms/48145872")
84 res = await self.client.request("GET", pages, headers={
85 "accept-language": "en-US,en;q=0.9",
86 "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
87 "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
88 })
89 if not res.ok:
90 return None
91 bundles = re.findall(r'https://a0\.muscache\.com/airbnb/static/packages/web/[^"\s]+\.js', res.text)
92 pat = re.compile(_HASH_RE_TMPL.format(op=operation))
93 for url in dict.fromkeys(bundles):
94 jb = await self.client.request("GET", url, headers={"accept-language": "en-US"})
95 if not jb.ok:
96 continue
97 m = pat.search(jb.text)
98 if m:
99 self.hashes[operation] = m.group(1)
100 Actor.log.info(f"[airbnb] refreshed hash for {operation}")
101 return m.group(1)
102 return None
103
104 async def _gql(self, operation: str, variables: dict, method: str = "POST",
105 ok_key: str | None = "data") -> Any | None:
106 """Call a persisted GraphQL query, refreshing the hash once if it is rejected."""
107 for refresh in (False, True):
108 if refresh:
109 if not await self._resolve_hash(operation):
110 return None
111 h = self.hashes.get(operation)
112 ext = json.dumps({"persistedQuery": {"version": 1, "sha256Hash": h}})
113 url = f"https://www.airbnb.com/api/v3/{operation}/{h}"
114 base_params = {"operationName": operation, "locale": self.locale, "currency": self.currency}
115 if method == "GET":
116 params = {**base_params, "variables": json.dumps(variables, separators=(",", ":")), "extensions": ext}
117 res = await self.client.request("GET", url, headers=BASE_HEADERS, params=params)
118 else:
119 body = {"operationName": operation, "variables": variables,
120 "extensions": {"persistedQuery": {"version": 1, "sha256Hash": h}}}
121 res = await self.client.request("POST", url, headers=BASE_HEADERS, params=base_params, json_body=body)
122 if not res.ok:
123 continue
124 try:
125 payload = res.json()
126 except Exception:
127 continue
128 errors = payload.get("errors")
129 if errors:
130 msg = json.dumps(errors)
131 if "PersistedQueryNotFound" in msg or "persisted" in msg.lower():
132 continue
133 Actor.log.debug(f"[airbnb] {operation} errors: {msg[:200]}")
134 return None
135 if ok_key and ok_key not in payload:
136 continue
137 return payload
138 return None
139
140
141 def _raw_params(self, extra: dict) -> list[dict]:
142 params = {
143 "cdnCacheSafe": "false", "itemsPerGrid": "50", "refinementPaths": ["/homes"],
144 "screenSize": "large", "tabId": "home_tab", "version": "1.8.8",
145 }
146 params.update(extra)
147 out = []
148 for k, v in params.items():
149 out.append({"filterName": k, "filterValues": v if isinstance(v, list) else [str(v)]})
150 return out
151
152 async def _search_page(self, extra: dict, cursor: str | None) -> dict | None:
153 raw = self._raw_params(extra)
154 ssr = {"maxMapItems": 9999, "metadataOnly": False, "rawParams": raw,
155 "requestedPageType": "STAYS_SEARCH", "treatmentFlags": TREATMENT_FLAGS,
156 "searchType": "user_map_move" if "ne_lat" in extra else "filter_change"}
157 if cursor:
158 ssr["cursor"] = cursor
159 map_raw = [p for p in raw if p["filterName"] != "itemsPerGrid"]
160 variables = {
161 "aiSearchEnabled": False, "isLeanTreatment": False, "staysSearchRequest": ssr,
162 "staysMapSearchRequestV2": {"metadataOnly": False, "rawParams": map_raw,
163 "requestedPageType": "STAYS_SEARCH", "treatmentFlags": TREATMENT_FLAGS,
164 **({"cursor": cursor} if cursor else {})},
165 "includeMapResults": True, "skipExtendedSearchParams": False,
166 }
167 payload = await self._gql("StaysSearch", variables, method="POST")
168 if not payload:
169 return None
170 try:
171 return payload["data"]["presentation"]["staysSearch"]
172 except (KeyError, TypeError):
173 return None
174
175 async def _collect_tile(self, extra: dict, remaining: int, max_pages: int) -> tuple[dict, dict | None, bool]:
176 """Return (listings_by_id, map_bounds_hint, saturated) for a single tile/query."""
177 found: dict[str, dict] = {}
178 stays = await self._search_page(extra, None)
179 if not stays:
180 return found, None, False
181 results = stays.get("results") or {}
182 hint = None
183 try:
184 hint = stays["mapResults"]["mapMetadata"].get("mapBoundsHint")
185 except (KeyError, TypeError):
186 hint = None
187 cursors = (results.get("paginationInfo") or {}).get("pageCursors") or []
188 for item in results.get("searchResults") or []:
189 row = parse_search_result(item)
190 if row:
191 found[row["id"]] = row
192 saturated = len(cursors) >= 15
193 pages = min(len(cursors), max_pages)
194 for idx in range(1, pages):
195 if len(found) >= remaining:
196 break
197 stays = await self._search_page(extra, cursors[idx])
198 if not stays:
199 break
200 for item in (stays.get("results") or {}).get("searchResults") or []:
201 row = parse_search_result(item)
202 if row:
203 found[row["id"]] = row
204 return found, hint, saturated
205
206 async def search(self, *, query: str | None = None, bounds: dict | None = None,
207 extra_filters: dict | None = None, max_listings: int = 500,
208 max_tiles: int = 40, max_pages: int = 15, subdivide: bool = True):
209 """Yield unique listing rows for a market. bounds = {ne_lat,ne_lng,sw_lat,sw_lng}."""
210 extra_filters = extra_filters or {}
211 seen: set[str] = set()
212
213 def take(rows: dict):
214 """Yield only the not-yet-seen rows, stopping exactly at max_listings."""
215 for lid, row in rows.items():
216 if len(seen) >= max_listings:
217 return
218 if lid not in seen:
219 seen.add(lid)
220 yield row
221
222
223 seed_bounds = None
224 if bounds:
225 seed_bounds = bounds
226 elif query:
227 rows, hint, _ = await self._collect_tile({"query": query, **extra_filters}, max_listings, max_pages)
228 for row in take(rows):
229 yield row
230 if hint:
231 seed_bounds = {
232 "ne_lat": hint["northeast"]["latitude"], "ne_lng": hint["northeast"]["longitude"],
233 "sw_lat": hint["southwest"]["latitude"], "sw_lng": hint["southwest"]["longitude"],
234 }
235 if len(seen) >= max_listings or seed_bounds is None:
236 return
237
238 if seed_bounds is None:
239 return
240
241 tiles = [(seed_bounds, 0)]
242 processed = 0
243 max_depth = 6 if subdivide else 0
244 while tiles and len(seen) < max_listings and processed < max_tiles:
245 tile, depth = tiles.pop(0)
246 processed += 1
247 extra = {
248 "ne_lat": tile["ne_lat"], "ne_lng": tile["ne_lng"],
249 "sw_lat": tile["sw_lat"], "sw_lng": tile["sw_lng"],
250 "search_by_map": "true", "zoom_level": str(min(19, 10 + depth * 2)),
251 **({"query": query} if query else {}), **extra_filters,
252 }
253 rows, _, saturated = await self._collect_tile(extra, max_listings - len(seen), max_pages)
254 for row in take(rows):
255 yield row
256 if saturated and depth < max_depth and len(seen) < max_listings:
257 tiles.extend((q, depth + 1) for q in _quadrants(tile))
258
259
260 async def detail(self, listing_id: str) -> dict | None:
261 url = f"https://www.airbnb.com/rooms/{listing_id}"
262 res = await self.client.request("GET", url, headers={
263 "accept-language": "en-US,en;q=0.9",
264 "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
265 "(KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36",
266 }, ok_predicate=lambda s, t: s == 200 and "data-deferred-state" in t)
267 if not res.ok:
268 return None
269 m = re.search(r'<script id="data-deferred-state-0"[^>]*>(.*?)</script>', res.text, re.S)
270 if not m:
271 return None
272 try:
273 state = json.loads(m.group(1))
274 node = state["niobeClientData"][0][1]
275 except Exception:
276 return None
277 return parse_pdp(node, listing_id)
278
279 async def calendar(self, listing_id: str, months: int = 12) -> list[dict] | None:
280 import datetime
281
282 today = datetime.date.today()
283 variables = {"request": {"count": months, "listingId": str(listing_id),
284 "month": today.month, "year": today.year}}
285 payload = await self._gql("PdpAvailabilityCalendar", variables, method="GET")
286 if not payload:
287 return None
288 try:
289 cal = payload["data"]["merlin"]["pdpAvailabilityCalendar"]["calendarMonths"]
290 except (KeyError, TypeError):
291 return None
292 days = []
293 for month in cal:
294 for d in month.get("days") or []:
295 days.append({
296 "date": d.get("calendarDate"),
297 "available": bool(d.get("available")),
298 "minNights": d.get("minNights"),
299 "maxNights": d.get("maxNights"),
300 "bookable": d.get("bookable"),
301 })
302 return days
303
304 async def price_for_dates(self, listing_id: str, checkin: str, checkout: str, adults: int = 1) -> dict | None:
305 enc = base64.b64encode(f"DemandStayListing:{listing_id}".encode()).decode()
306 variables = {
307 "id": enc, "dateRange": {"startDate": checkin, "endDate": checkout},
308 "guestCounts": {"numberOfAdults": adults},
309 "includePdpMigrationBookItCalendarSheetFragment": True,
310 "includePdpMigrationBookItFloatingFooterFragment": True,
311 "includePdpMigrationBookItNavFragment": True,
312 "includePdpMigrationBookItSidebarFragment": True,
313 "includeOverviewMerchandisingTipsFragment": False,
314 "includeStaysPdpPriceHeatmapFragment": False,
315 "priceHeatmapDateRange": {"startDate": checkin, "endDate": checkout},
316 "p3ImpressionId": "p3_1_A", "selectedCancellationPolicyId": None,
317 "causeId": None, "selectedGuestOptionId": None,
318 }
319 payload = await self._gql("StaysPdpBookItQuery", variables, method="GET")
320 if not payload:
321 return None
322 try:
323 book = payload["data"]["node"]["pdpPresentation"]["bookIt"]
324 except (KeyError, TypeError):
325 return None
326 return parse_bookit(book, checkin, checkout)
327
328
329def _quadrants(b: dict) -> list[dict]:
330 mid_lat = (b["ne_lat"] + b["sw_lat"]) / 2
331 mid_lng = (b["ne_lng"] + b["sw_lng"]) / 2
332 return [
333 {"ne_lat": b["ne_lat"], "ne_lng": mid_lng, "sw_lat": mid_lat, "sw_lng": b["sw_lng"]},
334 {"ne_lat": b["ne_lat"], "ne_lng": b["ne_lng"], "sw_lat": mid_lat, "sw_lng": mid_lng},
335 {"ne_lat": mid_lat, "ne_lng": mid_lng, "sw_lat": b["sw_lat"], "sw_lng": b["sw_lng"]},
336 {"ne_lat": mid_lat, "ne_lng": b["ne_lng"], "sw_lat": b["sw_lat"], "sw_lng": mid_lng},
337 ]
338
339
340def _rating_parts(label: str | None):
341 if not label:
342 return None, None
343 m = _RATING_RE.search(label.replace("\xa0", " "))
344 if not m:
345 return None, None
346 try:
347 rating = float(m.group(1).replace(",", "."))
348 except ValueError:
349 rating = None
350 try:
351 count = int(m.group(2).replace(",", ""))
352 except ValueError:
353 count = None
354 return rating, count
355
356
357def _structured_lines(structured: dict, keys: set[str]) -> list[str]:
358 out = []
359 for line in structured.get("primaryLine") or []:
360 if line.get("type") in keys and line.get("body"):
361 out.append(line["body"])
362 return out
363
364
365def parse_search_result(item: dict) -> dict | None:
366 dsl = item.get("demandStayListing") or {}
367 lid = _decode_listing_id(dsl.get("id"))
368 if not lid:
369 return None
370 coord = (dsl.get("location") or {}).get("coordinate") or {}
371 rating, reviews = _rating_parts(item.get("avgRatingA11yLabel") or item.get("avgRatingLocalized"))
372 structured = item.get("structuredContent") or {}
373 price = item.get("structuredDisplayPrice") or {}
374 primary = price.get("primaryLine") or {}
375 badges = [b.get("text") for b in item.get("badges") or [] if b.get("text")]
376 images = [p.get("picture") for p in item.get("contextualPictures") or [] if p.get("picture")]
377 name = item.get("name") or item.get("title")
378 if isinstance(item.get("nameLocalized"), dict):
379 name = item["nameLocalized"].get("localizedStringWithTranslationPreference") or name
380 return {
381 "platform": "airbnb",
382 "id": lid,
383 "url": f"https://www.airbnb.com/rooms/{lid}",
384 "name": name,
385 "propertyType": item.get("title"),
386 "coordinates": {"lat": coord.get("latitude"), "lng": coord.get("longitude")} if coord else None,
387 "rating": rating,
388 "reviewsCount": reviews,
389 "roomInfo": _structured_lines(structured, {"BEDINFO", "BATHROOMINFO"}),
390 "priceLabel": primary.get("discountedPrice") or primary.get("price"),
391 "originalPriceLabel": primary.get("originalPrice"),
392 "priceQualifier": primary.get("qualifier"),
393 "badges": badges,
394 "isGuestFavorite": any("GUEST_FAVORITE" in json.dumps(b) for b in item.get("badges") or []),
395 "images": images[:12],
396 "thumbnail": images[0] if images else None,
397 }
398
399
400def parse_pdp(node: dict, listing_id: str) -> dict | None:
401 data = node.get("data") or {}
402 n = data.get("node") or {}
403 pdp = n.get("pdpPresentation") or {}
404 metadata = {}
405 try:
406 metadata = data["presentation"]["stayProductDetailPage"]["sections"]["metadata"]
407 except (KeyError, TypeError):
408 metadata = {}
409 logging_ctx = ((metadata.get("loggingContext") or {}).get("eventDataLogging") or {})
410 sharing = metadata.get("sharingConfig") or {}
411 seo = metadata.get("seoFeatures") or {}
412
413 loc = pdp.get("location") or {}
414 coord = (n.get("location") or {}).get("coordinate") or {}
415 host = (pdp.get("hostInfo") or {})
416 passport = host.get("passportData") or {}
417 overview = pdp.get("overview") or {}
418 quality = pdp.get("quality") or {}
419 rating_stats = (quality.get("listingRatingStats") or {}).get("overallRatingStats") or {}
420
421 amenities = []
422 for group in (pdp.get("amenities") or {}).get("seeAllAmenitiesGroups") or []:
423 for a in group.get("amenities") or []:
424 if a.get("available") and a.get("title"):
425 amenities.append(a["title"])
426
427 photos = []
428 for stop in (pdp.get("mediaTour") or {}).get("stops") or []:
429 for it in stop.get("items") or []:
430 img = it.get("image") or {}
431 if img.get("uri"):
432 photos.append(img["uri"])
433
434 house_rules = []
435 for group in (pdp.get("rules") or {}).get("groupItems") or []:
436 for it in group.get("items") or []:
437 if it.get("title"):
438 house_rules.append(it["title"])
439
440 description = None
441 desc = pdp.get("descriptions") or {}
442 if isinstance(desc.get("longDescriptionHtml"), dict):
443 description = desc["longDescriptionHtml"].get("localizedStringWithTranslationPreference")
444
445 category_ratings = {}
446 for cat in quality.get("categoryRatings") or []:
447 if cat.get("categoryType") and cat.get("localizedRating"):
448 category_ratings[cat["categoryType"].lower()] = _to_float(cat["localizedRating"])
449
450 return {
451 "platform": "airbnb",
452 "id": listing_id,
453 "url": f"https://www.airbnb.com/rooms/{listing_id}",
454 "name": _ugc(pdp.get("title")) or (n.get("description") or {}).get("name", {}).get("localizedStringWithTranslationPreference"),
455 "propertyType": sharing.get("propertyType") or n.get("propertyType"),
456 "roomType": logging_ctx.get("roomType"),
457 "spaceType": n.get("spaceType"),
458 "personCapacity": n.get("personCapacity") or sharing.get("personCapacity"),
459 "overviewItems": overview.get("items"),
460 "coordinates": {"lat": coord.get("latitude") or loc.get("latitude"),
461 "lng": coord.get("longitude") or loc.get("longitude")},
462 "isExactLocation": loc.get("isExactLocation"),
463 "locationSubtitle": loc.get("subtitle"),
464 "rating": rating_stats.get("ratingAverage") or logging_ctx.get("guestSatisfactionOverall"),
465 "reviewsCount": _to_int(rating_stats.get("ratingCount")) or _to_int(logging_ctx.get("visibleReviewCount")),
466 "categoryRatings": category_ratings or None,
467 "isSuperhost": passport.get("isSuperhost"),
468 "isGuestFavorite": quality.get("isGuestFavorite"),
469 "host": {
470 "name": passport.get("name"),
471 "isSuperhost": passport.get("isSuperhost"),
472 "isVerified": passport.get("isVerified"),
473 "ratingCount": passport.get("ratingCount"),
474 "ratingAverage": passport.get("ratingAverage"),
475 "yearsHosting": (passport.get("timeAsHost") or {}).get("years"),
476 "responseRate": host.get("responseRateText"),
477 "responseTime": host.get("responseTimeText"),
478 "profileUrl": f"https://www.airbnb.com/users/show/{_decode_user_id(passport.get('userId'))}"
479 if passport.get("userId") else None,
480 },
481 "amenities": amenities,
482 "amenitiesCount": len(amenities),
483 "houseRules": house_rules,
484 "description": description,
485 "images": photos[:40],
486 "thumbnail": (photos[0] if photos else sharing.get("imageUrl")),
487 "seoTitle": (seo.get("title") if isinstance(seo.get("title"), str) else None),
488 "metaDescription": seo.get("metaDescription") if isinstance(seo.get("metaDescription"), str) else None,
489 }
490
491
492def parse_bookit(book: dict, checkin: str, checkout: str) -> dict:
493 sdp = book.get("structuredDisplayPrice") or {}
494 primary = sdp.get("primaryLine") or {}
495 availability = book.get("availability") or {}
496 line_items = []
497 total = None
498 for group in ((sdp.get("explanationData") or {}).get("priceDetails") or []):
499 for it in group.get("items") or []:
500 if it.get("description") and it.get("priceString"):
501 line_items.append({"label": it["description"], "amount": it["priceString"]})
502 if it.get("__typename") == "HighlightExplanationLineItem":
503 total = it.get("priceString")
504 return {
505 "checkIn": checkin,
506 "checkOut": checkout,
507 "isAvailable": availability.get("isAvailable"),
508 "canInstantBook": availability.get("canInstantBook"),
509 "displayPrice": primary.get("discountedPrice") or primary.get("price"),
510 "originalPrice": primary.get("originalPrice"),
511 "totalBeforeTaxes": total,
512 "priceItems": line_items,
513 }
514
515
516def _ugc(v):
517 if isinstance(v, dict):
518 content = v.get("content") or v
519 return content.get("localizedStringWithTranslationPreference") or content.get("localizedString")
520 return v
521
522
523def _decode_user_id(encoded):
524 return _decode_listing_id(encoded)
525
526
527def _to_float(v):
528 try:
529 return float(str(v).replace(",", "."))
530 except (TypeError, ValueError):
531 return None
532
533
534def _to_int(v):
535 try:
536 return int(str(v).replace(",", "").split(".")[0])
537 except (TypeError, ValueError):
538 return None
![VRBO Scraper — Properties + Calendar + Full Reviews [$2.5💰] avatar](https://images.apifyusercontent.com/GjYEFnGAoN4h98SQVX5_-16p4UJ22WTlYPlZvYf0T8c/rs:fill:76:76/cb:1/aHR0cHM6Ly9hcGlmeS1pbWFnZS11cGxvYWRzLXByb2QuczMudXMtZWFzdC0xLmFtYXpvbmF3cy5jb20vbmduOHZSRVNUa3NleVdFNnAtYWN0b3ItMk5jcGRpZ2VTOU5mOE91U28tamRMQkNsTlV6Vy1WUkJPX2FwcC53ZWJw.webp){kind=avatar}
