1import "dotenv/config";
2import { Actor, log } from "apify";
3import { withAnalytics } from "@apifyhub/analytics";
4import { withSentinel } from "@apifyhub/sentinel";
5
6interface Input {
7 query?: string;
8 tag?: string;
9 author?: string;
10 sortBy?: string;
11 dateAfter?: string;
12 dateBefore?: string;
13 minPoints?: number;
14 minComments?: number;
15 maxItems?: number;
16 pageId?: string;
17}
18
19interface AlgoliaHit {
20 objectID: string;
21 created_at?: string;
22 created_at_i?: number;
23 author?: string;
24 title?: string | null;
25 url?: string | null;
26 story_text?: string | null;
27 comment_text?: string | null;
28 story_id?: number | null;
29 story_title?: string | null;
30 story_url?: string | null;
31 parent_id?: number | null;
32 points?: number | null;
33 num_comments?: number | null;
34 _tags?: string[];
35}
36
37interface AlgoliaResponse {
38 hits: AlgoliaHit[];
39 nbHits?: number;
40 page?: number;
41 nbPages?: number;
42 hitsPerPage?: number;
43}
44
45interface ItemRecord {
46 objectID: string;
47 type: string | null;
48 title: string | null;
49 url: string | null;
50 author: string | null;
51 points: number | null;
52 numComments: number | null;
53 storyText: string | null;
54 commentText: string | null;
55 storyId: number | null;
56 storyTitle: string | null;
57 storyUrl: string | null;
58 parentId: number | null;
59 createdAt: string | null;
60 createdAtUnix: number | null;
61 hnUrl: string;
62 tags: string[];
63}
64
65const NEXT_PAGE_ID_KEY = "NEXT_PAGE_ID";
66const VALID_TAGS = new Set(["any", "story", "comment", "show_hn", "ask_hn", "poll", "job", "front_page"]);
67const VALID_SORT = new Set(["relevance", "date"]);
68const TYPE_TAGS = ["story", "comment", "show_hn", "ask_hn", "poll", "pollopt", "job"];
69const PAGE_SIZE = 100;
70const MAX_PAGES = 1000;
71
72function deriveType(tags: string[]): string | null {
73 for (const t of tags) {
74 if (TYPE_TAGS.includes(t)) return t;
75 }
76 return null;
77}
78
79function parseDate(raw: string, endOfDay: boolean): number | null {
80 const s = raw.trim();
81 if (!s) return null;
82 const m = /^(\d{4})-(\d{2})-(\d{2})$/.exec(s);
83 if (!m) return null;
84 const y = parseInt(m[1], 10);
85 const mo = parseInt(m[2], 10) - 1;
86 const d = parseInt(m[3], 10);
87 const ts = endOfDay
88 ? Date.UTC(y, mo, d, 23, 59, 59)
89 : Date.UTC(y, mo, d, 0, 0, 0);
90 if (Number.isNaN(ts)) return null;
91 return Math.floor(ts / 1000);
92}
93
94function buildTagsFilter(tag: string, author: string): string | null {
95 const parts: string[] = [];
96 if (tag !== "any") parts.push(tag);
97 if (author) parts.push(`author_${author}`);
98 return parts.length > 0 ? parts.join(",") : null;
99}
100
101function buildNumericFilters(
102 afterUnix: number | null,
103 beforeUnix: number | null,
104 minPoints: number,
105 minComments: number,
106): string | null {
107 const parts: string[] = [];
108 if (afterUnix !== null) parts.push(`created_at_i>=${afterUnix}`);
109 if (beforeUnix !== null) parts.push(`created_at_i<=${beforeUnix}`);
110 if (minPoints > 0) parts.push(`points>=${minPoints}`);
111 if (minComments > 0) parts.push(`num_comments>=${minComments}`);
112 return parts.length > 0 ? parts.join(",") : null;
113}
114
115function mapHit(h: AlgoliaHit): ItemRecord {
116 const tags = h._tags ?? [];
117 return {
118 objectID: h.objectID,
119 type: deriveType(tags),
120 title: h.title ?? null,
121 url: h.url ?? null,
122 author: h.author ?? null,
123 points: h.points ?? null,
124 numComments: h.num_comments ?? null,
125 storyText: h.story_text ?? null,
126 commentText: h.comment_text ?? null,
127 storyId: h.story_id ?? null,
128 storyTitle: h.story_title ?? null,
129 storyUrl: h.story_url ?? null,
130 parentId: h.parent_id ?? null,
131 createdAt: h.created_at ?? null,
132 createdAtUnix: h.created_at_i ?? null,
133 hnUrl: `https://news.ycombinator.com/item?id=${h.objectID}`,
134 tags,
135 };
136}
137
138async function fetchPage(
139 endpoint: "search" | "search_by_date",
140 query: string,
141 tagsFilter: string | null,
142 numericFilters: string | null,
143 page: number,
144): Promise<AlgoliaResponse> {
145 const url = new URL(`https://hn.algolia.com/api/v1/${endpoint}`);
146 if (query) url.searchParams.set("query", query);
147 if (tagsFilter) url.searchParams.set("tags", tagsFilter);
148 if (numericFilters) url.searchParams.set("numericFilters", numericFilters);
149 url.searchParams.set("hitsPerPage", String(PAGE_SIZE));
150 url.searchParams.set("page", String(page));
151
152 const response = await fetch(url, {
153 headers: { accept: "application/json" },
154 signal: AbortSignal.timeout(30_000),
155 });
156
157 if (!response.ok) {
158 throw new Error(`Algolia ${response.status}: ${await response.text().catch(() => "")}`);
159 }
160
161 return (await response.json()) as AlgoliaResponse;
162}
163
164await Actor.init();
165
166const input = (await Actor.getInput<Input>()) ?? {};
167
168const query = (input.query ?? "").trim();
169const tag = (input.tag ?? "story").toLowerCase();
170if (!VALID_TAGS.has(tag)) {
171 await Actor.fail(`Input 'tag' must be one of: ${[...VALID_TAGS].join(", ")}.`);
172 process.exit(1);
173}
174
175const sortBy = (input.sortBy ?? "relevance").toLowerCase();
176if (!VALID_SORT.has(sortBy)) {
177 await Actor.fail(`Input 'sortBy' must be 'relevance' or 'date'.`);
178 process.exit(1);
179}
180
181const author = (input.author ?? "").trim();
182const dateAfterRaw = (input.dateAfter ?? "").trim();
183const dateBeforeRaw = (input.dateBefore ?? "").trim();
184const afterUnix = dateAfterRaw ? parseDate(dateAfterRaw, false) : null;
185const beforeUnix = dateBeforeRaw ? parseDate(dateBeforeRaw, true) : null;
186
187if (dateAfterRaw && afterUnix === null) {
188 await Actor.fail(`Input 'dateAfter' must be in YYYY-MM-DD format (got '${dateAfterRaw}').`);
189 process.exit(1);
190}
191if (dateBeforeRaw && beforeUnix === null) {
192 await Actor.fail(`Input 'dateBefore' must be in YYYY-MM-DD format (got '${dateBeforeRaw}').`);
193 process.exit(1);
194}
195
196const minPoints = Math.max(0, input.minPoints ?? 0);
197const minComments = Math.max(0, input.minComments ?? 0);
198const maxItems = Math.max(0, input.maxItems ?? 100);
199const startPage = input.pageId ? parseInt(input.pageId.trim(), 10) : 0;
200if (Number.isNaN(startPage) || startPage < 0) {
201 await Actor.fail(`Input 'pageId' must be a non-negative integer (got '${input.pageId}').`);
202 process.exit(1);
203}
204
205await withSentinel({ apifyHubKey: process.env.APIFYHUB_KEY ?? "" }, () =>
206withAnalytics({ apifyHubKey: process.env.APIFYHUB_KEY ?? "" }, async () => {
207 const endpoint = sortBy === "date" ? "search_by_date" : "search";
208 const tagsFilter = buildTagsFilter(tag, author);
209 const numericFilters = buildNumericFilters(afterUnix, beforeUnix, minPoints, minComments);
210
211 const dataset = await Actor.openDataset<ItemRecord>();
212 const timeoutAt = Actor.getEnv().timeoutAt;
213 const deadlineMs = timeoutAt ? timeoutAt.getTime() - 60_000 : null;
214
215 let pushedTotal = 0;
216 let page = startPage;
217 let lastGoodPage: number | null = startPage;
218 let pages = 0;
219
220 log.info(
221 `[hn] query='${query}' tag=${tag}${author ? ` author=${author}` : ""} sort=${sortBy} startPage=${startPage} max=${maxItems || "unlimited"}`,
222 );
223
224 try {
225 while (true) {
226 if (maxItems > 0 && pushedTotal >= maxItems) break;
227 if (pages >= MAX_PAGES) {
228 log.warning(`[hn] hit MAX_PAGES safety cap.`);
229 break;
230 }
231 if (deadlineMs && Date.now() > deadlineMs) {
232 log.warning(`[hn] approaching actor timeout. Stopping early; resume with NEXT_PAGE_ID.`);
233 break;
234 }
235
236 const data = await fetchPage(endpoint, query, tagsFilter, numericFilters, page);
237 const hits = data.hits ?? [];
238 pages++;
239
240 let pushedThisPage = 0;
241 for (const h of hits) {
242 if (maxItems > 0 && pushedTotal >= maxItems) break;
243 await dataset.pushData(mapHit(h));
244 pushedTotal++;
245 pushedThisPage++;
246 }
247
248 log.info(
249 `[hn] page ${page} (${pages} fetched): +${pushedThisPage} (total ${pushedTotal}/${data.nbHits ?? "?"})`,
250 );
251
252 const nbPages = data.nbPages ?? 0;
253 if (hits.length === 0 || page + 1 >= nbPages) {
254 lastGoodPage = null;
255 break;
256 }
257
258 page++;
259 lastGoodPage = page;
260 }
261 } catch (err) {
262 log.error(`[hn] failed: ${(err as Error).message}. Pushed ${pushedTotal} so far.`);
263 }
264
265 await Actor.setValue(NEXT_PAGE_ID_KEY, lastGoodPage === null ? null : String(lastGoodPage));
266
267 log.info(`[hn] done. items=${pushedTotal} ${NEXT_PAGE_ID_KEY}=${lastGoodPage ?? "null"}`);
268
269}));
270
271await Actor.exit();
