1"""Hacker News Search & Monitor — Apify Actor.
2
3Uses the free, no-auth Algolia Hacker News Search API:
4 https://hn.algolia.com/api/v1/search (sorted by relevance: points & comments)
5 https://hn.algolia.com/api/v1/search_by_date (sorted by date: most recent first)
6
7API contract verified against a live response. Relevant fields per hit:
8 objectID, title, url, author, points, num_comments, created_at,
9 created_at_i, story_id, story_title, story_url, comment_text, _tags
10Top-level: hits, nbHits, nbPages, page, hitsPerPage
11
12Paging: Algolia numbers pages by hitsPerPage, so every request asks for a full page
13and the last page is cut locally (asking for a smaller last page, as earlier versions
14did, returned rows that were already delivered). Rows are de-duplicated by objectID.
15Algolia returns at most 1,000 hits per search; sorted by date the actor goes past
16that by moving a created_at_i window back in time, up to 10,000 items.
17
18Billing: pay per event, one 'apify-default-dataset-item' per item pushed (charged by
19the platform). The run stops cleanly at the user's maximum charge, and after a
20restart (the platform moving the run to another server) items already in the
21dataset are not pushed or charged again.
22"""
23
24from __future__ import annotations
25
26import asyncio
27import re
28from datetime import datetime, timezone
29
30import httpx
31from apify import Actor, Event
32
33BASE_RELEVANCE = "https://hn.algolia.com/api/v1/search"
34BASE_DATE = "https://hn.algolia.com/api/v1/search_by_date"
35HN_ITEM_URL = "https://news.ycombinator.com/item?id={}"
36HN_USER_URL = "https://news.ycombinator.com/user?id={}"
37PAGE_SIZE = 100
38ALGOLIA_MAX_HITS = 1000
39MAX_ITEMS = 10000
40DATASET_EVENT = "apify-default-dataset-item"
41FETCH_ATTEMPTS = 5
42
43
44CONTENT_TYPE_TAGS = {
45 "story": "story",
46 "comment": "comment",
47 "ask_hn": "ask_hn",
48 "show_hn": "show_hn",
49 "poll": "poll",
50 "job": "job",
51 "all": "(story,comment)",
52}
53
54_ISO_DATE_RE = re.compile(r"(\d{4})[-/](\d{1,2})[-/](\d{1,2})")
55_DMY_DATE_RE = re.compile(r"(\d{1,2})[-/.](\d{1,2})[-/.](\d{4})")
56
57
58
59
60
61
62def as_text(value) -> str:
63 if value is None or isinstance(value, bool):
64 return ""
65 if isinstance(value, (list, tuple)):
66 return " ".join(p for p in (as_text(v) for v in value) if p)
67 return str(value).strip()
68
69
70def as_int(value) -> int | None:
71 if value is None or isinstance(value, bool):
72 return None
73 try:
74 return int(float(str(value).strip()))
75 except (TypeError, ValueError):
76 return None
77
78
79def _date_to_timestamp(date_str) -> int | None:
80 """YYYY-MM-DD (also an ISO date-time, YYYY/MM/DD or DD-MM-YYYY) -> unix timestamp (UTC)."""
81 text = as_text(date_str)
82 if not text:
83 return None
84 match = _ISO_DATE_RE.search(text)
85 if match:
86 year, month, day = (int(g) for g in match.groups())
87 else:
88 match = _DMY_DATE_RE.search(text)
89 if not match:
90 return None
91 day, month, year = (int(g) for g in match.groups())
92 try:
93 return int(datetime(year, month, day, tzinfo=timezone.utc).timestamp())
94 except ValueError:
95 return None
96
97
98def build_tags(content_type: str, author: str | None) -> str:
99 """Combine content-type tag with an optional author filter (ANDed)."""
100 base = CONTENT_TYPE_TAGS.get(content_type, "story")
101 if author:
102
103 return f"{base},author_{author}"
104 return base
105
106
107def build_numeric_filters(
108 min_points: int | None,
109 min_comments: int | None,
110 created_after_ts: int | None,
111 created_before_ts: int | None,
112 window_end_ts: int | None = None,
113) -> str:
114 """Build the Algolia numericFilters string from the active constraints.
115 window_end_ts (inclusive) is the moving end of the date window used past 1,000 hits."""
116 filters: list[str] = []
117 if min_points is not None:
118 filters.append(f"points>={min_points}")
119 if min_comments is not None:
120 filters.append(f"num_comments>={min_comments}")
121 if created_after_ts is not None:
122 filters.append(f"created_at_i>={created_after_ts}")
123 if created_before_ts is not None:
124 filters.append(f"created_at_i<{created_before_ts}")
125 if window_end_ts is not None:
126 filters.append(f"created_at_i<={window_end_ts}")
127 return ",".join(filters)
128
129
130def transform_hit(hit: dict, include_text: bool) -> dict:
131 """Convert a raw Algolia hit into a clean, stable output record.
132
133 Pure function — no I/O — so it can be unit-tested against captured fixtures.
134 """
135 tags = hit.get("_tags", []) or []
136
137 if "comment" in tags:
138 item_type = "comment"
139 elif "poll" in tags:
140 item_type = "poll"
141 elif "job" in tags:
142 item_type = "job"
143 elif "show_hn" in tags:
144 item_type = "show_hn"
145 elif "ask_hn" in tags:
146 item_type = "ask_hn"
147 else:
148 item_type = "story"
149
150 object_id = hit.get("objectID")
151
152 record = {
153 "id": object_id,
154 "type": item_type,
155 "title": hit.get("title") or hit.get("story_title"),
156 "url": hit.get("url") or hit.get("story_url"),
157 "author": hit.get("author"),
158 "points": hit.get("points"),
159 "numComments": hit.get("num_comments"),
160 "createdAt": hit.get("created_at"),
161 "createdAtTimestamp": hit.get("created_at_i"),
162 "storyId": hit.get("story_id"),
163 "hnUrl": HN_ITEM_URL.format(object_id) if object_id else None,
164 "authorUrl": HN_USER_URL.format(hit["author"]) if hit.get("author") else None,
165 }
166
167 if include_text:
168
169 record["text"] = hit.get("comment_text") or hit.get("story_text")
170
171 return record
172
173
174
175
176
177
178async def fetch_page(client: httpx.AsyncClient, url: str, params: dict) -> dict | None:
179 """One Algolia page, with backoff on rate limits and transient errors. None if it keeps failing."""
180 for attempt in range(1, FETCH_ATTEMPTS + 1):
181 try:
182 resp = await client.get(url, params=params)
183 if resp.status_code == 429 or resp.status_code >= 500:
184 Actor.log.warning(f"Algolia answered HTTP {resp.status_code} on attempt {attempt}, backing off.")
185 elif resp.status_code >= 400:
186 Actor.log.error(f"Algolia rejected the search (HTTP {resp.status_code}): {resp.text[:300]}")
187 return None
188 else:
189 return resp.json()
190 except (httpx.HTTPError, ValueError) as exc:
191 Actor.log.warning(f"Request attempt {attempt} failed: {exc!r}")
192 if attempt < FETCH_ATTEMPTS:
193 await asyncio.sleep(min(3 * attempt, 15))
194 return None
195
196
197def budget_left(pushed_this_run: int) -> int | None:
198 """How many more items the user's maximum charge allows; None = no limit. The
199 platform charges the dataset-item event itself and the SDK doesn't count those
200 charges while the run is going, so the items pushed in this run are subtracted."""
201 try:
202 charging = Actor.get_charging_manager()
203 if not charging.get_pricing_info().is_pay_per_event:
204 return None
205 n = charging.calculate_max_event_charge_count_within_limit(DATASET_EVENT)
206 except Exception:
207 return None
208 return None if n is None else max(0, n - pushed_this_run)
209
210
211async def read_delivered() -> tuple[set[str], int | None]:
212 """IDs already in this run's dataset (only after a restart), and the oldest
213 created_at_i among them (where a date-sorted run continues from)."""
214 ids: set[str] = set()
215 oldest: int | None = None
216 dataset = await Actor.open_dataset()
217 offset = 0
218 while True:
219 page = await dataset.get_data(offset=offset, limit=1000)
220 items = page.items or []
221 if not items:
222 break
223 for item in items:
224 if item.get("id"):
225 ids.add(str(item["id"]))
226 ts = item.get("createdAtTimestamp")
227 if isinstance(ts, int) and (oldest is None or ts < oldest):
228 oldest = ts
229 offset += len(items)
230 return ids, oldest
231
232
233
234
235
236
237async def main() -> None:
238 async with Actor:
239 actor_input = await Actor.get_input() or {}
240
241 query = as_text(actor_input.get("query"))
242 content_type = as_text(actor_input.get("contentType")) or "story"
243 if content_type not in CONTENT_TYPE_TAGS:
244 Actor.log.warning(f"Unknown contentType '{content_type}', searching stories.")
245 content_type = "story"
246 sort_by = "date" if as_text(actor_input.get("sortBy")) == "date" else "relevance"
247 max_items = as_int(actor_input.get("maxItems"))
248 if max_items is None or max_items < 1:
249 max_items = 100
250 max_items = min(max_items, MAX_ITEMS)
251 min_points = as_int(actor_input.get("minPoints"))
252 min_comments = as_int(actor_input.get("minComments"))
253 author = as_text(actor_input.get("author")).lstrip("@") or None
254 include_text = actor_input.get("includeCommentText") is not False
255
256 created_after_ts = _date_to_timestamp(actor_input.get("createdAfter"))
257 created_before_ts = _date_to_timestamp(actor_input.get("createdBefore"))
258 for field in ("createdAfter", "createdBefore"):
259 if as_text(actor_input.get(field)) and _date_to_timestamp(actor_input.get(field)) is None:
260 Actor.log.warning(f"Could not read the date in '{field}' ({actor_input.get(field)!r}); use YYYY-MM-DD.")
261
262 base_url = BASE_DATE if sort_by == "date" else BASE_RELEVANCE
263 tags = build_tags(content_type, author)
264 if sort_by == "relevance" and max_items > ALGOLIA_MAX_HITS:
265 Actor.log.warning(
266 f"Hacker News search returns at most {ALGOLIA_MAX_HITS:,} results per search sorted by "
267 f"relevance. Sort by date to get up to {MAX_ITEMS:,}."
268 )
269 Actor.log.info(
270 f"Searching HN | query={query!r} type={content_type} sort={sort_by} tags={tags} "
271 f"numericFilters={build_numeric_filters(min_points, min_comments, created_after_ts, created_before_ts) or '(none)'} "
272 f"maxItems={max_items}"
273 )
274
275 seen, oldest_delivered = await read_delivered()
276 if seen:
277 Actor.log.info(
278 f"Continuing this run after a restart: {len(seen):,} items are already in the dataset "
279 f"and will not be pushed or charged again."
280 )
281 delivered = len(seen)
282 pushed = 0
283 stop = None
284 matched: int | None = None
285 progress = {"aborting": False}
286
287 async def on_aborting(_event_data=None):
288 progress["aborting"] = True
289
290 Actor.on(Event.ABORTING, on_aborting)
291
292
293
294
295 window_end = oldest_delivered if sort_by == "date" else None
296
297 async with httpx.AsyncClient(timeout=30.0, headers={"User-Agent": "apify-hn-search/0.2"}) as client:
298 while delivered < max_items and stop is None:
299 numeric_filters = build_numeric_filters(
300 min_points, min_comments, created_after_ts, created_before_ts, window_end)
301 window_new = 0
302 window_oldest: int | None = None
303 window_full = False
304 pages_per_window = ALGOLIA_MAX_HITS // PAGE_SIZE
305 for page in range(pages_per_window):
306 if delivered >= max_items:
307 break
308 if progress["aborting"]:
309 stop = "aborted"
310 break
311 params = {"tags": tags, "hitsPerPage": PAGE_SIZE, "page": page}
312 if query:
313 params["query"] = query
314 if numeric_filters:
315 params["numericFilters"] = numeric_filters
316 data = await fetch_page(client, base_url, params)
317 if data is None:
318 stop = "unreachable"
319 break
320 if matched is None:
321 matched = as_int(data.get("nbHits")) or 0
322 Actor.log.info(f"Matched {matched:,} items.")
323 hits = data.get("hits") or []
324 rows = []
325 for hit in hits:
326 ts = hit.get("created_at_i")
327 if isinstance(ts, int) and (window_oldest is None or ts < window_oldest):
328 window_oldest = ts
329 object_id = str(hit.get("objectID") or "")
330 if not object_id or object_id in seen:
331 continue
332 seen.add(object_id)
333 rows.append(transform_hit(hit, include_text))
334 rows = rows[: max_items - delivered]
335 left = budget_left(pushed)
336 if left is not None and len(rows) > left:
337 rows = rows[:left]
338 stop = "budget"
339 if rows:
340 await Actor.push_data(rows)
341 pushed += len(rows)
342 delivered += len(rows)
343 window_new += len(rows)
344 Actor.log.info(f"Pushed {delivered}/{max_items} items.")
345 await Actor.set_status_message(f"{delivered:,} of {max_items:,} items so far.")
346 if stop or len(hits) < PAGE_SIZE:
347 break
348 if page + 1 >= pages_per_window:
349 window_full = True
350 break
351 if page + 1 >= (as_int(data.get("nbPages")) or 0):
352 break
353 if (window_full and not stop and sort_by == "date" and window_oldest is not None
354 and window_new > 0 and delivered < max_items):
355 window_end = window_oldest
356 Actor.log.info(f"Read {ALGOLIA_MAX_HITS:,} hits; continuing with older items.")
357 continue
358 break
359
360 if stop == "budget":
361 message = (f"Stopped at the maximum charge you set for this run, after {delivered:,} items. "
362 f"Raise the maximum charge to get more.")
363 elif stop == "aborted":
364 message = f"Run aborted after {delivered:,} items."
365 elif stop == "unreachable":
366 message = f"The Hacker News search API kept failing, so the run stopped after {delivered:,} items."
367 elif matched == 0 and delivered == 0:
368 message = "Done: no Hacker News items match this search."
369 else:
370 message = f"Done: {delivered:,} items."
371 if (sort_by == "relevance" and delivered >= ALGOLIA_MAX_HITS and max_items > delivered
372 and (matched or 0) > delivered):
373 message += (f" Sorted by relevance the search stops at {ALGOLIA_MAX_HITS:,}; "
374 f"sort by date to get up to {MAX_ITEMS:,}.")
375 Actor.log.info(message)
376 await Actor.set_status_message(message)