1import assert from 'node:assert/strict';
2import { readFileSync } from 'node:fs';
3import { describe, it } from 'node:test';
4import { fetchJson, HttpError, mapLimit, reviewsUrl, reviewsUrlAlt } from '../src/fetch.js';
5import { addAppFields, normaliseLookup, normaliseReviewsFeed } from '../src/normalise.js';
6import { parseAppId, parseBundleId, parseCountry, resolveInput, shouldEmitAppRows } from '../src/parse.js';
7import { fetchReviews, fetchReviewsResult, matchesFilters, newestState, nextState } from '../src/scrape.js';
8
9const fixture = (name: string) => JSON.parse(readFileSync(new URL(`./fixtures/${name}`, import.meta.url), 'utf8'));
10const NOW = '2026-10-06T00:00:00.000Z';
11
12describe('parseAppId', () => {
13 it('accepts numeric ids, id prefix and URLs', () => {
14 assert.equal(parseAppId('389801252'), '389801252');
15 assert.equal(parseAppId('id389801252'), '389801252');
16 assert.equal(parseAppId(' ID389801252 '), '389801252');
17 assert.equal(parseAppId('https://apps.apple.com/us/app/duolingo/id570060128'), '570060128');
18 assert.equal(parseAppId('https://apps.apple.com/us/app/duolingo/id570060128?mt=8'), '570060128');
19 assert.equal(parseAppId('https://itunes.apple.com/app?id=123'), '123');
20 });
21 it('rejects junk', () => {
22 assert.equal(parseAppId('hello'), null);
23 assert.equal(parseAppId(''), null);
24 assert.equal(parseAppId(null), null);
25 });
26});
27
28describe('parseCountry', () => {
29 it('validates 2-letter codes', () => {
30 assert.equal(parseCountry('US'), 'us');
31 assert.equal(parseCountry('usa'), null);
32 assert.equal(parseCountry('u1'), null);
33 });
34});
35
36describe('resolveInput', () => {
37 it('applies defaults and de-duplicates', () => {
38 const r = resolveInput({ apps: ['123', 'id123', 'bad'] });
39 assert.deepEqual(r.appIds, ['123']);
40 assert.deepEqual(r.countries, ['us']);
41 assert.equal(r.maxReviewsPerApp, 100);
42 assert.equal(r.includeMetadata, false);
43 assert.equal(r.warnings.length, 1);
44 });
45 it('clamps maxReviewsPerApp to 500', () => {
46 assert.equal(resolveInput({ apps: ['1'], maxReviewsPerApp: 9999 }).maxReviewsPerApp, 500);
47 });
48 it('throws on no valid app, bad date', () => {
49 assert.throws(() => resolveInput({ apps: ['x'] }));
50 assert.throws(() => resolveInput({ apps: ['1'], sinceDate: 'nope' }));
51 assert.throws(() => resolveInput({ apps: ['1'], countries: ['usa'] }));
52 });
53});
54
55describe('normaliseReviewsFeed', () => {
56 it('maps review fields from a real feed page', () => {
57 const items = normaliseReviewsFeed(fixture('reviews-page1.json'), '389801252', 'us', NOW);
58 assert.equal(items.length, 4);
59 const r = items[0];
60 assert.equal(r.type, 'review');
61 assert.equal(r.appId, '389801252');
62 assert.equal(typeof r.rating, 'number');
63 assert.ok(r.reviewId.length > 0);
64 assert.ok(!Number.isNaN(Date.parse(r.date)));
65 assert.equal(r.scrapedAt, NOW);
66 });
67 it('handles a single entry object', () => {
68 const items = normaliseReviewsFeed(fixture('reviews-single-entry.json'), '1', 'us', NOW);
69 assert.equal(items.length, 1);
70 });
71 it('handles an empty feed and garbage', () => {
72 assert.deepEqual(normaliseReviewsFeed(fixture('reviews-empty.json'), '1', 'us', NOW), []);
73 assert.deepEqual(normaliseReviewsFeed(null, '1', 'us', NOW), []);
74 });
75 it('skips the app info entry (no rating)', () => {
76 const feed = fixture('reviews-page1.json');
77 feed.feed.entry.unshift({ 'im:name': { label: 'App' }, id: { label: 'https://apps.apple.com/app' } });
78 assert.equal(normaliseReviewsFeed(feed, '1', 'us', NOW).length, 4);
79 });
80});
81
82describe('normaliseLookup', () => {
83 it('maps app metadata', () => {
84 const app = normaliseLookup(fixture('lookup.json'), '389801252', 'us', NOW);
85 assert.ok(app);
86 assert.equal(app.type, 'app');
87 assert.equal(app.appId, '389801252');
88 assert.ok(app.name.length > 0);
89 assert.ok(Array.isArray(app.genres));
90 });
91 it('returns null when not found', () => {
92 assert.equal(normaliseLookup({ resultCount: 0, results: [] }, '1', 'us', NOW), null);
93 });
94});
95
96const res = (status: number, body = '') => new Response(body, { status });
97
98describe('fetchJson', () => {
99 const sleep = async () => {};
100 it('retries 5xx then succeeds', async () => {
101 let calls = 0;
102 const fetchImpl = (async () => (++calls < 3 ? res(503) : res(200, '{"ok":1}'))) as typeof fetch;
103 assert.deepEqual(await fetchJson('u', { fetchImpl, sleep }), { ok: 1 });
104 assert.equal(calls, 3);
105 });
106 it('retries network errors, then throws after 3 tries', async () => {
107 let calls = 0;
108 const fetchImpl = (async () => {
109 calls++;
110 throw new TypeError('fetch failed');
111 }) as typeof fetch;
112 await assert.rejects(fetchJson('u', { fetchImpl, sleep }), /fetch failed/);
113 assert.equal(calls, 3);
114 });
115 it('does not retry 404', async () => {
116 let calls = 0;
117 const fetchImpl = (async () => (calls++, res(404))) as typeof fetch;
118 await assert.rejects(fetchJson('u', { fetchImpl, sleep }), (e: unknown) => e instanceof HttpError && e.status === 404);
119 assert.equal(calls, 1);
120 });
121 it('backs off exponentially', async () => {
122 const delays: number[] = [];
123 const fetchImpl = (async () => res(429)) as typeof fetch;
124 await assert.rejects(
125 fetchJson('u', { fetchImpl, baseDelayMs: 100, sleep: async (ms) => void delays.push(ms) }),
126 );
127 assert.deepEqual(delays, [100, 200]);
128 });
129 it('returns null for an empty body', async () => {
130 const fetchImpl = (async () => res(200, '')) as typeof fetch;
131 assert.equal(await fetchJson('u', { fetchImpl, sleep }), null);
132 });
133});
134
135describe('mapLimit', () => {
136 it('settles every item and respects the limit', async () => {
137 let active = 0;
138 let peak = 0;
139 const out = await mapLimit([1, 2, 3, 4, 5], 2, async (n) => {
140 active++;
141 peak = Math.max(peak, active);
142 await new Promise((r) => setTimeout(r, 5));
143 active--;
144 if (n === 3) throw new Error('boom');
145 return n;
146 });
147 assert.equal(peak, 2);
148 assert.equal(out.filter((o) => o.status === 'rejected').length, 1);
149 assert.equal(out.filter((o) => o.status === 'fulfilled').length, 4);
150 });
151});
152
153
154function fakePage(start: number, n: number) {
155 const entry = Array.from({ length: n }, (_, i) => {
156 const k = start + i;
157 return {
158 id: { label: `r${k}` },
159 'im:rating': { label: '5' },
160 title: { label: `t${k}` },
161 content: { label: 'c' },
162 updated: { label: new Date(Date.UTC(2026, 0, 1) - k * 3600_000).toISOString() },
163 author: { name: { label: 'a' } },
164 };
165 });
166 return { feed: { entry } };
167}
168
169describe('fetchReviews', () => {
170 const pages = (map: Record<number, unknown>) =>
171 (async (url: string) => {
172 const page = Number(/page=(\d+)/.exec(String(url))![1]);
173 if (!(page in map)) return res(400);
174 return res(200, JSON.stringify(map[page]));
175 }) as unknown as typeof fetch;
176 const base = { appId: '1', country: 'us', sinceDate: null, state: null };
177
178 it('pages until maxReviews and de-duplicates ids', async () => {
179 const fetchImpl = pages({ 1: fakePage(0, 50), 2: fakePage(40, 50), 3: fakePage(90, 50) });
180 const out = await fetchReviews({ ...base, maxReviews: 120, fetchOptions: { fetchImpl } });
181 assert.equal(out.length, 120);
182 assert.equal(new Set(out.map((r) => r.reviewId)).size, 120);
183 });
184 it('stops on a 400 past the last page', async () => {
185 const fetchImpl = pages({ 1: fakePage(0, 50), 2: fakePage(50, 10) });
186 const out = await fetchReviews({ ...base, maxReviews: 500, fetchOptions: { fetchImpl } });
187 assert.equal(out.length, 60);
188 });
189 it('returns nothing for an empty feed', async () => {
190 const fetchImpl = pages({ 1: { feed: {} } });
191 assert.deepEqual(await fetchReviews({ ...base, maxReviews: 50, fetchOptions: { fetchImpl } }), []);
192 });
193 it('throws on a 403 at page 1', async () => {
194 const fetchImpl = (async () => res(403)) as typeof fetch;
195 await assert.rejects(fetchReviews({ ...base, maxReviews: 50, fetchOptions: { fetchImpl, sleep: async () => {} } }));
196 });
197 it('filters by sinceDate', async () => {
198 const fetchImpl = pages({ 1: fakePage(0, 50) });
199 const since = new Date(Date.UTC(2026, 0, 1) - 9.5 * 3600_000);
200 const out = await fetchReviews({ ...base, maxReviews: 50, sinceDate: since, fetchOptions: { fetchImpl } });
201 assert.equal(out.length, 10);
202 });
203 it('returns only reviews newer than state', async () => {
204 const fetchImpl = pages({ 1: fakePage(0, 50) });
205 const all = await fetchReviews({ ...base, maxReviews: 50, fetchOptions: { fetchImpl } });
206 const state = newestState(all.slice(5));
207 assert.ok(state);
208 const out = await fetchReviews({ ...base, maxReviews: 50, state, fetchOptions: { fetchImpl } });
209 assert.deepEqual(out.map((r) => r.reviewId), ['r0', 'r1', 'r2', 'r3', 'r4']);
210 });
211});
212
213describe('newestState', () => {
214 it('returns null for empty input', () => assert.equal(newestState([]), null));
215});
216
217describe('fetchReviewsResult and state safety', () => {
218 const pages = (map: Record<number, unknown>, failFrom = 99) =>
219 (async (url: string) => {
220 const page = Number(/page=(\d+)/.exec(String(url))![1]);
221 if (page >= failFrom) return res(500);
222 if (!(page in map)) return res(400);
223 return res(200, JSON.stringify(map[page]));
224 }) as unknown as typeof fetch;
225 const base = { appId: '1', country: 'us', sinceDate: null, state: null };
226 const fo = (fetchImpl: typeof fetch) => ({ fetchImpl, retries: 1, sleep: async () => {} });
227
228 it('marks the result incomplete when maxReviews cuts the run', async () => {
229 const r = await fetchReviewsResult({ ...base, maxReviews: 20, fetchOptions: fo(pages({ 1: fakePage(0, 50) })) });
230 assert.equal(r.reviews.length, 20);
231 assert.equal(r.complete, false);
232 });
233 it('marks the result complete when the feed ends', async () => {
234 const r = await fetchReviewsResult({ ...base, maxReviews: 500, fetchOptions: fo(pages({ 1: fakePage(0, 10) })) });
235 assert.equal(r.complete, true);
236 });
237 it('keeps earlier pages when a later page fails', async () => {
238 const r = await fetchReviewsResult({ ...base, maxReviews: 500, fetchOptions: fo(pages({ 1: fakePage(0, 50), 2: fakePage(50, 50) }, 2)) });
239 assert.equal(r.reviews.length, 50);
240 assert.equal(r.complete, false);
241 });
242 it('returns an empty list on a 400 at page 1', async () => {
243 const r = await fetchReviewsResult({ ...base, maxReviews: 50, fetchOptions: fo(pages({})) });
244 assert.deepEqual(r, { reviews: [], complete: true, newestSeen: null });
245 });
246 it('maps uk to gb with a warning', () => {
247 const r = resolveInput({ apps: ['1'], countries: ['uk'] });
248 assert.deepEqual(r.countries, ['gb']);
249 assert.equal(r.warnings.length, 1);
250 });
251});
252
253describe('sort option', () => {
254 it('puts sortby in both feed URLs', () => {
255 assert.match(reviewsUrl('1', 'us', 2), /sortby=mostrecent/);
256 assert.match(reviewsUrl('1', 'us', 2, 'mosthelpful'), /sortby=mosthelpful/);
257 assert.match(reviewsUrlAlt('1', 'us', 2, 'mosthelpful'), /sortby=mosthelpful.*cc=us/);
258 });
259 it('defaults to mostrecent and ignores onlyNewReviews with mosthelpful', () => {
260 assert.equal(resolveInput({ apps: ['1'] }).sort, 'mostrecent');
261 const r = resolveInput({ apps: ['1'], sort: 'mosthelpful', onlyNewReviews: true });
262 assert.equal(r.onlyNewReviews, false);
263 assert.equal(r.warnings.length, 1);
264 });
265});
266
267const pagesOf = (map: Record<number, unknown>, urls: string[] = []) =>
268 (async (url: string) => {
269 urls.push(String(url));
270 const page = Number(/page=(\d+)/.exec(String(url))![1]);
271 if (!(page in map)) return res(400);
272 return res(200, JSON.stringify(map[page]));
273 }) as unknown as typeof fetch;
274const baseQ = { appId: '1', country: 'us', sinceDate: null, state: null };
275
276describe('mosthelpful and the cutoff', () => {
277 it('does not stop at an old review', async () => {
278 const page = fakePage(0, 10);
279
280 page.feed.entry[2].updated.label = '2020-01-01T00:00:00.000Z';
281 const urls: string[] = [];
282 const fetchImpl = pagesOf({ 1: page, 2: fakePage(50, 5) }, urls);
283 const since = new Date('2025-01-01T00:00:00Z');
284 const helpful = await fetchReviews({ ...baseQ, maxReviews: 100, sinceDate: since, sort: 'mosthelpful', fetchOptions: { fetchImpl } });
285 assert.equal(helpful.length, 14);
286 assert.ok(urls.every((u) => u.includes('sortby=mosthelpful')));
287 const recent = await fetchReviews({ ...baseQ, maxReviews: 100, sinceDate: since, fetchOptions: { fetchImpl } });
288 assert.equal(recent.length, 9);
289 });
290});
291
292describe('rating and keyword filters', () => {
293 const feedWith = (rows: Array<[string, number, string]>) => ({
294 feed: {
295 entry: rows.map(([id, rating, text], i) => ({
296 id: { label: id },
297 'im:rating': { label: String(rating) },
298 title: { label: 'T' },
299 content: { label: text },
300 updated: { label: new Date(Date.UTC(2026, 0, 1) - i * 1000).toISOString() },
301 author: { name: { label: 'a' } },
302 })),
303 },
304 });
305 it('validates ratings and keywords', () => {
306 assert.throws(() => resolveInput({ apps: ['1'], minRating: 4, maxRating: 2 }), /minRating/);
307 assert.throws(() => resolveInput({ apps: ['1'], minRating: 0 }));
308 assert.throws(() => resolveInput({ apps: ['1'], maxRating: 6 }));
309 const r = resolveInput({ apps: ['1'], minRating: 2, keywords: [' Crash ', '', 'crash'] });
310 assert.deepEqual(r.keywords, ['crash']);
311 assert.equal(r.minRating, 2);
312 assert.equal(r.maxRating, null);
313 });
314 it('matchesFilters checks stars and any keyword, case-insensitive', () => {
315 const r = normaliseReviewsFeed(feedWith([['a', 3, 'App CRASHES daily']]), '1', 'us', NOW)[0];
316 assert.equal(matchesFilters(r, { minRating: 3, maxRating: 3, keywords: ['crash'] }), true);
317 assert.equal(matchesFilters(r, { minRating: 4 }), false);
318 assert.equal(matchesFilters(r, { maxRating: 2 }), false);
319 assert.equal(matchesFilters(r, { keywords: ['login', 'daily'] }), true);
320 assert.equal(matchesFilters(r, { keywords: ['login'] }), false);
321 });
322 it('does not count filtered reviews toward max and keeps reading pages', async () => {
323 const p1 = feedWith(Array.from({ length: 5 }, (_, i) => [`a${i}`, 1, 'x'] as [string, number, string]));
324 const p2 = feedWith(Array.from({ length: 5 }, (_, i) => [`b${i}`, 5, 'x'] as [string, number, string]));
325 p2.feed.entry.forEach((e) => (e.updated.label = '2025-12-01T00:00:00.000Z'));
326 const r = await fetchReviewsResult({ ...baseQ, maxReviews: 3, minRating: 5, fetchOptions: { fetchImpl: pagesOf({ 1: p1, 2: p2 }) } });
327 assert.deepEqual(r.reviews.map((x) => x.reviewId), ['b0', 'b1', 'b2']);
328 assert.equal(r.complete, false);
329 });
330 it('state follows the newest review seen, matching or not', async () => {
331 const p1 = feedWith([['n1', 1, 'x'], ['n2', 5, 'x'], ['n3', 1, 'x']]);
332 const done = await fetchReviewsResult({ ...baseQ, maxReviews: 5, minRating: 5, fetchOptions: { fetchImpl: pagesOf({ 1: p1 }) } });
333 assert.deepEqual(done.reviews.map((x) => x.reviewId), ['n2']);
334 assert.equal(done.complete, true);
335 assert.equal(done.newestSeen?.reviewId, 'n1');
336 });
337 it('is incomplete when max fills and the rest of the page is new but non-matching', async () => {
338 const p1 = feedWith([['m1', 5, 'x'], ['x1', 2, 'x'], ['x2', 2, 'x']]);
339 const p2 = feedWith([['m4', 5, 'x']]);
340 p2.feed.entry.forEach((e) => (e.updated.label = '2025-12-01T00:00:00.000Z'));
341 const r = await fetchReviewsResult({ ...baseQ, maxReviews: 1, minRating: 5, fetchOptions: { fetchImpl: pagesOf({ 1: p1, 2: p2 }) } });
342 assert.deepEqual(r.reviews.map((x) => x.reviewId), ['m1']);
343 assert.equal(r.complete, false);
344 });
345 it('is complete when all MAX_PAGES are read, because Apple serves no page 11', async () => {
346 const pages: Record<number, ReturnType<typeof feedWith>> = {};
347 for (let i = 1; i <= 10; i++) {
348 pages[i] = feedWith([[`p${i}`, 1, 'x']]);
349 pages[i].feed.entry.forEach((e) => (e.updated.label = new Date(Date.UTC(2026, 0, 1) - i * 1000).toISOString()));
350 }
351 const r = await fetchReviewsResult({ ...baseQ, maxReviews: 5, minRating: 5, fetchOptions: { fetchImpl: pagesOf(pages) } });
352 assert.deepEqual(r.reviews, []);
353 assert.equal(r.complete, true);
354 });
355 it('onlyNewReviews: the next run skips filtered reviews', async () => {
356 const p1 = feedWith([['n1', 1, 'x'], ['n2', 5, 'x'], ['n3', 1, 'x']]);
357 const first = await fetchReviewsResult({ ...baseQ, maxReviews: 5, minRating: 5, fetchOptions: { fetchImpl: pagesOf({ 1: p1 }) } });
358 const saved = nextState(null, first.newestSeen);
359 const second = await fetchReviewsResult({ ...baseQ, maxReviews: 5, minRating: 5, state: saved, fetchOptions: { fetchImpl: pagesOf({ 1: p1 }) } });
360 assert.deepEqual(second.reviews, []);
361 assert.equal(second.complete, true);
362 assert.equal(nextState(saved, second.newestSeen), null);
363 });
364});
365
366describe('403 retry', () => {
367 it('retries 403 then succeeds', async () => {
368 let calls = 0;
369 const fetchImpl = (async () => (++calls < 3 ? res(403) : res(200, '{"ok":1}'))) as typeof fetch;
370 assert.deepEqual(await fetchJson('u', { fetchImpl, sleep: async () => {} }), { ok: 1 });
371 assert.equal(calls, 3);
372 });
373});
374
375describe('bundle ids', () => {
376 it('parses reverse-DNS bundle ids', () => {
377 assert.equal(parseBundleId('com.duolingo.DuolingoMobile'), 'com.duolingo.DuolingoMobile');
378 assert.equal(parseBundleId('hello'), null);
379 assert.equal(parseBundleId('https://apps.apple.com/us/app/x'), null);
380 });
381 it('keeps bundle ids apart from numeric ids and de-duplicates them', () => {
382 const r = resolveInput({ apps: ['123', 'com.a.b', 'com.a.b', 'junk'] });
383 assert.deepEqual(r.appIds, ['123']);
384 assert.deepEqual(r.bundleIds, ['com.a.b']);
385 assert.equal(r.warnings.length, 1);
386 assert.deepEqual(resolveInput({ apps: ['com.a.b'] }).appIds, []);
387 });
388});
389
390describe('same-second state', () => {
391 it('skips every id saved at the state date and keeps a new id at the same date', async () => {
392 const entry = (id: string, date: string) => ({
393 id: { label: id },
394 'im:rating': { label: '5' },
395 title: { label: 't' },
396 content: { label: 'c' },
397 updated: { label: date },
398 author: { name: { label: 'a' } },
399 });
400 const same = '2026-01-01T00:00:00.000Z';
401 const feed = { feed: { entry: [entry('x3', same), entry('x2', same), entry('x1', same), entry('x0', '2025-12-31T00:00:00.000Z')] } };
402 const state = { reviewId: 'x1', date: same, idsAtDate: ['x1', 'x2'] };
403 const r = await fetchReviewsResult({ ...baseQ, maxReviews: 10, state, fetchOptions: { fetchImpl: pagesOf({ 1: feed }) } });
404 assert.deepEqual(r.reviews.map((x) => x.reviewId), ['x3']);
405 const next = nextState(state, r.newestSeen);
406 assert.deepEqual(next?.idsAtDate?.sort(), ['x1', 'x2', 'x3']);
407
408 const old = await fetchReviewsResult({ ...baseQ, maxReviews: 10, state: { reviewId: 'x1', date: same }, fetchOptions: { fetchImpl: pagesOf({ 1: feed }) } });
409 assert.deepEqual(old.reviews.map((x) => x.reviewId), ['x3', 'x2']);
410 });
411 it('newestState fills idsAtDate', () => {
412 const mk = (id: string, date: string) => ({ reviewId: id, date }) as never;
413 const s = newestState([mk('a', '2026-01-01T00:00:00.000Z'), mk('b', '2026-01-02T00:00:00.000Z'), mk('c', '2026-01-02T00:00:00.000Z')]);
414 assert.deepEqual(s?.idsAtDate, ['b', 'c']);
415 });
416});
417
418describe('app fields and app rows', () => {
419 it('emits app rows only for metadata-only runs', () => {
420 assert.equal(shouldEmitAppRows(true, 0), true);
421 assert.equal(shouldEmitAppRows(true, 100), false);
422 assert.equal(shouldEmitAppRows(false, 0), false);
423 });
424 it('copies app fields onto a review, null when no lookup', () => {
425 const review = normaliseReviewsFeed(fixture('reviews-single-entry.json'), '1', 'us', NOW)[0];
426 const app = normaliseLookup(fixture('lookup.json'), '389801252', 'us', NOW);
427 addAppFields(review, app, 'mosthelpful');
428 assert.equal(review.appName, app?.name);
429 assert.equal(review.appDeveloper, app?.developer);
430 assert.equal(review.appUrl, app?.url);
431 assert.equal(review.sort, 'mosthelpful');
432 addAppFields(review, null, 'mostrecent');
433 assert.equal(review.appDeveloper, null);
434 assert.equal(review.appAverageRating, null);
435 });
436});
437
438describe('app url', () => {
439 it('drops the ?uo= tracking parameter', () => {
440 const app = normaliseLookup(fixture('lookup.json'), '389801252', 'us', 'now');
441 assert.equal(app?.url, 'https://apps.apple.com/us/app/instagram/id389801252');
442 });
443});
