1
2
3
4
5
6process.env.REBROWSER_PATCHES_RUNTIME_FIX_MODE ||= 'addBinding';
7process.env.REBROWSER_PATCHES_SOURCE_URL ||= 'app.js';
8process.env.REBROWSER_PATCHES_UTILITY_WORLD_NAME ||= 'util';
9
10import { Actor, log } from 'apify';
11import { PlaywrightCrawler, createPlaywrightRouter } from 'crawlee';
12import { chromium } from 'rebrowser-playwright';
13import {
14 InputSchema,
15 ValidatedInput,
16 EtsyProduct,
17 ExtractionResult,
18 RunOutput,
19 RunOutcome,
20 InterruptionReason,
21 INTERRUPTION_CAUSE_TEXT,
22 ResumeState,
23 RatingStats,
24 CardMeta,
25} from './types.js';
26import { HumanBehavior } from './human-behavior.js';
27import { DataDomeSolver } from './datadome-solver.js';
28import { extractProductsInPage } from './extract.js';
29import { Target, InputError, classifyEtsyUrl, gridPageUrl } from './target.js';
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51const MAX_PAGES = 12;
52
53
54
55
56
57const PLAUSIBLE_PAGE_CARDS = 40;
58
59
60const HANDLER_BUDGET_MS = 240_000;
61
62
63const OUTPUT_KEY = 'OUTPUT';
64const RESUME_KEY = 'RESUME_STATE';
65
66
67const CHECKPOINT_INTERVAL_MS = 30_000;
68
69
70
71
72
73const ALWAYS_CHECKPOINT_WRITES = 3;
74
75const SHUTDOWN_WRITE_TIMEOUT_MS = 5_000;
76const SHUTDOWN_SETTLE_TIMEOUT_MS = 3_000;
77
78
79
80
81
82
83
84const FINAL_WRITE_QUEUE_WAIT_MS = 2_000;
85
86const SHUTDOWN_TOTAL_TIMEOUT_MS = 20_000;
87
88type StopReason =
89 | 'reached-max-items'
90 | 'end-of-results'
91 | 'max-pages'
92 | 'blocked'
93 | 'nav-timeout'
94 | 'time-budget'
95 | 'charge-limit';
96
97
98
99
100
101
102const STRANDING_STOP_REASONS: ReadonlySet<StopReason> = new Set<StopReason>([
103 'max-pages',
104 'blocked',
105 'nav-timeout',
106 'time-budget',
107 'charge-limit',
108]);
109
110
111const FAILING_OUTCOMES: ReadonlySet<RunOutcome> = new Set<RunOutcome>([
112 'blocked',
113 'extraction-failed',
114 'filters-unusable',
115 'charge-limit-reached',
116]);
117
118
119async function withTimeout<T>(p: Promise<T>, ms: number): Promise<T | null> {
120 return Promise.race([
121 p.catch(() => null),
122 new Promise<null>((resolve) => setTimeout(() => resolve(null), ms).unref?.()),
123 ]);
124}
125
126class EtsyScraper {
127 private input: ValidatedInput;
128 private target: Target;
129 private dataDomeSolver: DataDomeSolver;
130
131 private itemCount = 0;
132 private seen = new Set<string>();
133 private pagesVisited = 0;
134 private listingsSeen = 0;
135 private uniqueExtracted = 0;
136 private droppedByFilter = 0;
137 private droppedUnfilterable = 0;
138
139
140
141
142
143
144
145 private unfilterableRatingAbsent = 0;
146 private unfilterableRatingUnresolved = 0;
147 private unfilterableReviewsAbsent = 0;
148 private unfilterableReviewsUnresolved = 0;
149 private unfilterablePrice = 0;
150 private ratingsMissing = 0;
151 private reviewCountsMissing = 0;
152
153
154
155
156 private ratingsAbsent = 0;
157 private ratingsUnresolved = 0;
158 private reviewCountsAbsent = 0;
159 private reviewCountsUnresolved = 0;
160 private ratingSources: Record<string, number> = {};
161 private reviewCountSources: Record<string, number> = {};
162 private pushChargeFailures = 0;
163 private pushDropped = 0;
164
165
166
167 private chargeLimitReached = false;
168 private chargeLimitDropped = 0;
169 private stopReason: StopReason = 'end-of-results';
170 private lastFailure: string | null = null;
171
172 private gridSettleMissedPages = 0;
173 private noResultsSignal = false;
174
175
176
177
178 private interruption: { reason: InterruptionReason; at: string } | null = null;
179 private stopRequested = false;
180 private finalWritten = false;
181
182
183
184
185
186 private finalWriteStarted = false;
187
188 private writeChain: Promise<unknown> = Promise.resolve();
189 private dirty = true;
190 private lastCheckpointAt = 0;
191 private checkpointWrites = 0;
192 private shutdownPromise: Promise<void> | null = null;
193
194 private nextPage: number;
195
196 private pageInProgress: number | null = null;
197
198 private pagesOpened = new Set<number>();
199
200
201
202
203
204
205 private pagesDelivered = new Set<number>();
206
207 private pageCutShort: number | null = null;
208
209 private resumedItems = 0;
210
211 private deliveredIds: string[] = [];
212
213 private pushing = false;
214
215
216
217
218
219 private attempt = 1;
220
221 constructor(input: ValidatedInput, target: Target) {
222 this.input = input;
223 this.target = target;
224 this.nextPage = target.startPage;
225 const apiKey = process.env.CAPSOLVER_API_KEY || '';
226 this.dataDomeSolver = new DataDomeSolver(apiKey);
227 }
228
229
230
231
232
233 async execute(): Promise<void> {
234 try {
235 await this.run();
236 } catch (err: any) {
237 if (!this.interruption) this.interruption = { reason: 'fatal-error', at: new Date().toISOString() };
238 this.stopRequested = true;
239 log.exception(err, 'The run failed before it could finish');
240 await this.writeOutput(true).catch(() => { });
241 throw err;
242 }
243 }
244
245 async run(): Promise<void> {
246
247
248 await this.resumePreviousAttempt();
249
250
251 await this.writeOutput(false);
252
253 const proxyConfiguration = await Actor.createProxyConfiguration({
254 groups: ['RESIDENTIAL'],
255 countryCode: 'US',
256 });
257 log.info('Using US residential proxies');
258
259 const router = createPlaywrightRouter();
260 router.addHandler('SEARCH', async ({ page, session }) => {
261 log.info(`Target: ${this.target.label}`);
262 const deadline = Date.now() + HANDLER_BUDGET_MS;
263
264
265
266 if (this.itemCount >= this.input.maxItems) {
267 log.info('Everything requested was already delivered by a previous attempt — nothing left to scrape.');
268 this.stopReason = 'reached-max-items';
269 return;
270 }
271 if (this.stopRequested) return;
272
273
274 try {
275 await page.waitForLoadState('domcontentloaded', { timeout: 30000 });
276 } catch { }
277 await this.dumpState(page, 'homepage');
278 if (!(await this.handleBlock(page, session, 'homepage'))) {
279 throw new Error('Homepage blocked (DataDome) — rotating session');
280 }
281
282 const human = new HumanBehavior(page);
283 await human.initialize();
284 await this.naturalDelay(1500, 2500);
285 await human.naturalScroll(1);
286 await human.randomMouseMovements(2);
287
288
289
290
291 const firstPage = this.nextPage;
292 if (firstPage > 1) {
293 log.info(`Starting at results page ${firstPage} (earlier pages are already done).`);
294 await page.goto(this.pageUrl(firstPage), { waitUntil: 'domcontentloaded', timeout: 60000 });
295 } else if (this.target.kind === 'query') {
296
297 const searchInput = await this.findSearchInput(page);
298 if (!searchInput) throw new Error('Search input not found on homepage');
299 await searchInput.click();
300 await this.naturalDelay(250, 500);
301 for (const ch of this.target.query) {
302 await page.keyboard.type(ch, { delay: 45 + Math.random() * 90 });
303 }
304 await this.naturalDelay(400, 800);
305 await page.keyboard.press('Enter');
306 } else {
307
308 await page.goto(this.target.url, { waitUntil: 'domcontentloaded', timeout: 60000 });
309 }
310 await page.waitForLoadState('domcontentloaded', { timeout: 30000 }).catch(() => {});
311
312
313
314 await this.dumpState(page, 'search-results');
315 if (!(await this.handleBlock(page, session, 'search-results'))) {
316 throw new Error('Search results blocked (DataDome) — rotating session');
317 }
318 if (!(await this.settleGrid(page, human))) {
319
320
321 await this.dumpState(page, 'search-results-no-grid', true);
322 if (!(await this.handleBlock(page, session, 'search-results-late'))) {
323 throw new Error('Search results blocked (DataDome, late) — rotating session');
324 }
325 }
326
327
328 const first = await this.scrapeCurrentPage(page, firstPage);
329 if (first.cardCount === 0) {
330 if (this.noResultsSignal) {
331
332 this.stopReason = 'end-of-results';
333 return;
334 }
335
336
337
338
339
340
341
342 await this.dumpState(page, `no-grid-page-${firstPage}`, true);
343 throw new Error(
344 'Search results page rendered no listing grid and no "no results" message '
345 + '— retrying on a fresh session',
346 );
347 }
348
349
350
351 const lastPage = firstPage + MAX_PAGES - 1;
352 for (let pageNum = firstPage + 1; pageNum <= lastPage; pageNum++) {
353 if (this.stopRequested) return;
354 if (this.itemCount >= this.input.maxItems) {
355 this.stopReason = 'reached-max-items';
356 return;
357 }
358
359
360
361 if (this.chargeLimitReached) {
362 log.warning('Charge budget (maxTotalChargeUsd) is exhausted — stopping.');
363 this.stopReason = 'charge-limit';
364 return;
365 }
366
367
368 if (Date.now() > deadline) {
369 log.warning('Reached the per-session time budget — stopping with results so far.');
370 this.stopReason = 'time-budget';
371 return;
372 }
373 const nextUrl = this.pageUrl(pageNum);
374
375
376 this.pageInProgress = pageNum;
377 log.info(`Page ${pageNum}...`);
378 try {
379 await page.goto(nextUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
380 } catch {
381 log.warning(`Pagination navigation to page ${pageNum} timed out — stopping with what we have.`);
382 this.stopReason = 'nav-timeout';
383 return;
384 }
385
386
387 const info = await this.getBlockInfo(page);
388 if (info.blocked) {
389 await this.dumpState(page, `block-page-${pageNum}`, true);
390 log.warning(`Page ${pageNum} blocked by DataDome — stopping with results so far.`);
391 this.stopReason = 'blocked';
392 return;
393 }
394 await this.settleGrid(page, human);
395 const res = await this.scrapeCurrentPage(page, pageNum);
396
397
398
399 if (res.cardCount === 0 || res.newUnique === 0) {
400 log.info('No further new listings — reached the end of the results.');
401 this.stopReason = 'end-of-results';
402 return;
403 }
404 this.stopReason = 'max-pages';
405 }
406 });
407
408 const crawler = new PlaywrightCrawler({
409 proxyConfiguration,
410 requestHandlerTimeoutSecs: 300,
411 maxRequestRetries: 5,
412 useSessionPool: true,
413 persistCookiesPerSession: true,
414 requestHandler: router,
415 failedRequestHandler: async ({ request }) => {
416 this.lastFailure = request.errorMessages?.slice(-1)[0]
417 ?? 'request failed with no error message';
418 log.error(`All ${request.retryCount + 1} attempts failed: ${this.lastFailure}`);
419 },
420 maxConcurrency: 1,
421 browserPoolOptions: {
422
423 useFingerprints: false,
424 },
425 sessionPoolOptions: {
426 blockedStatusCodes: [],
427 maxPoolSize: 10,
428 },
429 launchContext: {
430 launcher: chromium,
431 launchOptions: {
432 headless: false,
433
434
435
436 args: [
437 '--blink-settings=imagesEnabled=false',
438 '--disable-remote-fonts',
439 ],
440 },
441 },
442 preNavigationHooks: [
443 async ({ page }, gotoOptions) => {
444 await page.setViewportSize({ width: 1920, height: 1080 });
445 gotoOptions.waitUntil = 'domcontentloaded';
446 gotoOptions.timeout = 60000;
447 },
448 ],
449 });
450
451 log.info('Starting scraper');
452
453
454
455
456
457
458
459
460
461
462
463
464
465 await crawler.run([{
466 url: 'https://www.etsy.com',
467 label: 'SEARCH',
468 uniqueKey: `SEARCH:${this.targetKey()}:attempt-${this.attempt}`,
469 }]);
470
471 await this.reportOutcome();
472 }
473
474
475
476
477
478
479
480 private async settleGrid(page: any, human: HumanBehavior): Promise<boolean> {
481 const grid = await page
482 .waitForSelector('[data-palette-listing-id]', { timeout: 15000 })
483 .catch(() => null);
484 if (!grid) {
485 log.warning('Listing grid did not appear within 15s — the page is blocked, empty or restructured.');
486 return false;
487 }
488
489 await human.naturalScroll(1);
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510 const settled = await page.waitForFunction(
511 (minCards: number) => {
512 const w = window as any;
513 const n = document.querySelectorAll('[data-palette-listing-id]').length;
514 if (n === 0) return false;
515 if (w.__etsyCardsPrev === n) {
516 w.__etsyCardsStable = (w.__etsyCardsStable || 0) + 1;
517 } else {
518 w.__etsyCardsPrev = n;
519 w.__etsyCardsStable = 0;
520 }
521
522 return n >= minCards ? w.__etsyCardsStable >= 2 : w.__etsyCardsStable >= 12;
523 },
524 PLAUSIBLE_PAGE_CARDS,
525 { timeout: 15000, polling: 400 },
526 ).catch(() => null);
527 if (!settled) this.gridSettleMissedPages++;
528 return true;
529 }
530
531
532 private async scrapeCurrentPage(
533 page: any,
534 pageNum: number,
535 ): Promise<{ cardCount: number; newUnique: number; saved: number }> {
536 this.pageInProgress = pageNum;
537 let res: ExtractionResult;
538 try {
539 res = await page.evaluate(extractProductsInPage);
540 } catch (e: any) {
541 log.warning(`Extraction on page ${pageNum} failed: ${e?.message}`);
542 return { cardCount: 0, newUnique: 0, saved: 0 };
543 }
544
545 this.pagesVisited++;
546 this.pagesOpened.add(pageNum);
547 this.listingsSeen += res.cardCount;
548 if (res.noResultsSignal) this.noResultsSignal = true;
549
550
551 const srcTally = Object.entries(
552 Object.values(res.cardMeta).reduce((acc: Record<string, number>, m) => {
553 if (m.ratingSource) acc[m.ratingSource] = (acc[m.ratingSource] || 0) + 1;
554 return acc;
555 }, {}),
556 ).map(([k, v]) => `${k}=${v}`).join(' ');
557 log.info(
558 `Page ${pageNum}: ${res.cardCount} listing cards, ${res.items.length} parsed, `
559 + `${res.ratingElCount} with a rating, ${res.reviewCountElCount} with a review count`
560 + (srcTally ? ` [rating via ${srcTally}]` : ''),
561 );
562
563
564
565
566 const unresolvedHere = Object.values(res.cardMeta).filter((m) => m.rating === 'unresolved').length;
567 if (unresolvedHere > 0) {
568 log.warning(
569 `Page ${pageNum}: ${unresolvedHere} of ${res.cardCount} listing card(s) published rating `
570 + `markup this actor could NOT read — those ratings are null because extraction failed, `
571 + `not because the listing is unrated. Etsy's card markup has probably changed. `
572 + `Saving the rating region of one to the key-value store as `
573 + `rating-unresolved-sample-page-${pageNum}.html.`
574 + (res.unresolvedFingerprints.length
575 ? ` Markers seen: ${res.unresolvedFingerprints.join(' ;; ')}`
576 : ''),
577 );
578 if (res.sampleCardHtml) {
579 await Actor.setValue(`rating-unresolved-sample-page-${pageNum}.html`, res.sampleCardHtml, {
580 contentType: 'text/plain; charset=utf-8',
581 }).catch(() => { });
582 }
583 }
584
585 let newUnique = 0;
586 let saved = 0;
587 let cutShort = false;
588 for (const product of res.items) {
589 if (this.itemCount >= this.input.maxItems) break;
590
591
592 if (this.stopRequested) { cutShort = true; break; }
593
594 if (this.chargeLimitReached) { cutShort = true; break; }
595 if (this.seen.has(product.productId)) continue;
596 this.seen.add(product.productId);
597 newUnique++;
598 this.uniqueExtracted++;
599
600
601 const meta = res.cardMeta[product.productId];
602 if (product.rating === null) this.ratingsMissing++;
603 if (product.reviewCount === null) this.reviewCountsMissing++;
604 if (meta) {
605 if (meta.rating === 'absent') this.ratingsAbsent++;
606 else if (meta.rating === 'unresolved') this.ratingsUnresolved++;
607 else if (meta.ratingSource) {
608 this.ratingSources[meta.ratingSource] = (this.ratingSources[meta.ratingSource] || 0) + 1;
609 }
610 if (meta.reviewCount === 'absent') this.reviewCountsAbsent++;
611 else if (meta.reviewCount === 'unresolved') this.reviewCountsUnresolved++;
612 else if (meta.reviewCountSource) {
613 this.reviewCountSources[meta.reviewCountSource] =
614 (this.reviewCountSources[meta.reviewCountSource] || 0) + 1;
615 }
616 } else if (product.rating === null) {
617
618
619
620 this.ratingsUnresolved++;
621 this.reviewCountsUnresolved++;
622 }
623
624 const verdict = this.applyFilters(product);
625 if (verdict === 'dropped') { this.droppedByFilter++; continue; }
626 if (verdict !== 'keep') { this.countUnfilterable(verdict, meta); continue; }
627
628
629
630
631 if (await this.pushProduct(product)) {
632 saved++;
633 this.pagesDelivered.add(pageNum);
634 this.dirty = true;
635
636
637
638 if (saved % 20 === 0) await this.maybeCheckpoint();
639 }
640 }
641 if (cutShort) this.pageCutShort = pageNum;
642 if (this.chargeLimitReached) this.stopReason = 'charge-limit';
643
644
645
646
647 if (!cutShort && res.cardCount > 0) this.nextPage = pageNum + 1;
648 this.pageInProgress = null;
649 this.dirty = true;
650 log.info(`Page ${pageNum}: saved ${saved} (total ${this.itemCount}/${this.input.maxItems})`);
651 await this.maybeCheckpoint();
652 return { cardCount: res.cardCount, newUnique, saved };
653 }
654
655
656 private pageUrl(n: number): string {
657 if (this.target.kind === 'query') {
658 const base = `https://www.etsy.com/search?q=${encodeURIComponent(this.target.query)}`;
659 return n > 1 ? `${base}&page=${n}` : base;
660 }
661 return gridPageUrl(this.target.url, n);
662 }
663
664 private async findSearchInput(page: any): Promise<any> {
665 const selectors = [
666 'input#global-enhancements-search-query',
667 'input[name="search_query"]',
668 'input[type="search"]',
669 'input[placeholder*="Search"]',
670 ];
671 for (const sel of selectors) {
672 const el = await page.$(sel);
673 if (el && await el.isVisible().catch(() => false)) return el;
674 }
675 return null;
676 }
677
678
679
680
681
682
683
684 private async dumpState(page: any, label: string, force = false): Promise<void> {
685 try {
686 const info = await page.evaluate(() => ({
687 url: location.href,
688 title: document.title,
689 listings: document.querySelectorAll('[data-palette-listing-id]').length,
690 rated: document.querySelectorAll(
691 '[aria-label*="star rating"], [aria-label*="out of 5 stars"]',
692 ).length,
693 }));
694 log.info(`[${label}] "${info.title}" | listings=${info.listings} | rated=${info.rated}`);
695 } catch (e: any) {
696 log.warning(`[${label}] page probe failed (${e?.message}) — saving artifacts anyway`);
697 }
698
699 if (!force && !this.input.debug) return;
700 try {
701 const png = await page.screenshot({ fullPage: false, timeout: 10000 }).catch(() => null);
702 if (png) await Actor.setValue(`debug-${label}.png`, png, { contentType: 'image/png' });
703 const html = await page.content().catch(() => '');
704 if (html) await Actor.setValue(`debug-${label}.html`, html, { contentType: 'text/html' });
705 } catch (e: any) {
706 log.warning(`dumpState[${label}] persist failed: ${e?.message}`);
707 }
708 }
709
710 private async getBlockInfo(page: any): Promise<{ blocked: boolean; type: 'interstitial' | 'captcha' | 'none'; raw: string | null }> {
711 try {
712 return await page.evaluate(() => {
713 const html = document.documentElement.outerHTML;
714 const ddMatch = html.match(/var dd=\{.*?\}/s);
715 const raw = ddMatch ? ddMatch[0] : null;
716 const hasIframe = !!document.querySelector('iframe[src*="captcha-delivery.com"]');
717 const blocked = hasIframe || html.includes('geo.captcha-delivery.com') || html.includes('dd={');
718 if (!blocked) return { blocked: false, type: 'none' as const, raw };
719 const isCaptcha = /\/captcha\//.test(html) || /'rt'\s*:\s*'c'/.test(raw || '');
720 return { blocked: true, type: isCaptcha ? ('captcha' as const) : ('interstitial' as const), raw };
721 });
722 } catch {
723 return { blocked: false, type: 'none', raw: null };
724 }
725 }
726
727
728 private async handleBlock(page: any, session: any, label: string): Promise<boolean> {
729 let info = await this.getBlockInfo(page);
730 if (!info.blocked) return true;
731
732 log.warning(`[${label}] DataDome block — type=${info.type}`);
733 await this.dumpState(page, `block-${label}`, true);
734
735 if (info.type === 'interstitial') {
736 for (let i = 0; i < 3; i++) {
737 await page.waitForTimeout(2500);
738 info = await this.getBlockInfo(page);
739 if (!info.blocked) {
740 log.info('Interstitial cleared on its own');
741 return true;
742 }
743 }
744 log.warning('Interstitial persisted — retiring session to rotate IP');
745 if (session) session.retire();
746 return false;
747 }
748
749 log.info('Slider captcha — attempting solve');
750 const solved = await this.dataDomeSolver.solveDataDome(page);
751 if (solved) {
752 await page.waitForLoadState('domcontentloaded', { timeout: 30000 }).catch(() => {});
753 return true;
754 }
755 log.warning('Slider solve failed — retiring session to rotate IP');
756 if (session) session.retire();
757 return false;
758 }
759
760
761 private async naturalDelay(minMs: number, maxMs: number): Promise<void> {
762 const mean = (minMs + maxMs) / 2;
763 const stdDev = (maxMs - minMs) / 6;
764 const u1 = Math.random(), u2 = Math.random();
765 const z0 = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
766 const delay = Math.max(minMs, Math.min(maxMs, mean + z0 * stdDev));
767 await new Promise((r) => setTimeout(r, delay));
768 }
769
770
771
772
773
774
775
776
777 private filterActive(v: number | undefined): v is number {
778 return typeof v === 'number' && v > 0;
779 }
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795 private applyFilters(p: EtsyProduct): 'keep' | 'dropped' | 'no-rating' | 'no-reviews' | 'no-price' {
796 if (this.filterActive(this.input.minRating)) {
797 if (p.rating === null) return 'no-rating';
798 if (p.rating < this.input.minRating!) return 'dropped';
799 }
800 if (this.filterActive(this.input.minReviews)) {
801 if (p.reviewCount === null) return 'no-reviews';
802 if (p.reviewCount < this.input.minReviews!) return 'dropped';
803 }
804 if (this.filterActive(this.input.priceMin) || this.filterActive(this.input.priceMax)) {
805 if (p.price === null) return 'no-price';
806 if (this.filterActive(this.input.priceMin) && p.price < this.input.priceMin!) return 'dropped';
807 if (this.filterActive(this.input.priceMax) && p.price > this.input.priceMax!) return 'dropped';
808 }
809 return 'keep';
810 }
811
812
813
814
815
816
817
818 private countUnfilterable(verdict: 'no-rating' | 'no-reviews' | 'no-price', meta?: CardMeta): void {
819 this.droppedUnfilterable++;
820 if (verdict === 'no-price') { this.unfilterablePrice++; return; }
821 if (verdict === 'no-rating') {
822
823 if (meta?.rating === 'absent') this.unfilterableRatingAbsent++;
824 else this.unfilterableRatingUnresolved++;
825 return;
826 }
827 if (meta?.reviewCount === 'absent') this.unfilterableReviewsAbsent++;
828 else this.unfilterableReviewsUnresolved++;
829 }
830
831
832
833
834
835
836 private unfilterableReasons(): Record<string, number> {
837 const out: Record<string, number> = {};
838 if (this.unfilterableRatingUnresolved > 0) out.ratingUnresolved = this.unfilterableRatingUnresolved;
839 if (this.unfilterableRatingAbsent > 0) out.ratingAbsent = this.unfilterableRatingAbsent;
840 if (this.unfilterableReviewsUnresolved > 0) out.reviewsUnresolved = this.unfilterableReviewsUnresolved;
841 if (this.unfilterableReviewsAbsent > 0) out.reviewsAbsent = this.unfilterableReviewsAbsent;
842 if (this.unfilterablePrice > 0) out.priceUnreadable = this.unfilterablePrice;
843 return out;
844 }
845
846
847 private unfilterableFaults(): number {
848 return this.unfilterableRatingUnresolved + this.unfilterableReviewsUnresolved + this.unfilterablePrice;
849 }
850
851
852
853
854
855
856 private describeUnfilterable(): string {
857 const parts: string[] = [];
858 if (this.unfilterableRatingUnresolved > 0) {
859 parts.push(`${this.unfilterableRatingUnresolved} published a rating this actor could not read `
860 + `(an extraction fault)`);
861 }
862 if (this.unfilterableRatingAbsent > 0) {
863 parts.push(`${this.unfilterableRatingAbsent} publish no rating at all (normal for a new listing)`);
864 }
865 if (this.unfilterableReviewsUnresolved > 0) {
866 parts.push(`${this.unfilterableReviewsUnresolved} published a review count this actor could not `
867 + `read (an extraction fault)`);
868 }
869 if (this.unfilterableReviewsAbsent > 0) {
870 parts.push(`${this.unfilterableReviewsAbsent} publish no review count at all (normal for a `
871 + `listing with no reviews yet)`);
872 }
873 if (this.unfilterablePrice > 0) {
874 parts.push(`${this.unfilterablePrice} had no price this actor could read (an extraction fault)`);
875 }
876 return parts.join('; ');
877 }
878
879
880
881
882
883
884
885
886 private async pushProduct(product: EtsyProduct): Promise<boolean> {
887 this.pushing = true;
888 try {
889 return await this.pushProductInner(product);
890 } finally {
891 this.pushing = false;
892 }
893 }
894
895
896 private countDelivered(product: EtsyProduct): void {
897 this.itemCount++;
898 this.deliveredIds.push(product.productId);
899 }
900
901 private async pushProductInner(product: EtsyProduct): Promise<boolean> {
902
903
904
905 try {
906 const charge = await Actor.pushData(product, 'product-scraped');
907
908
909
910
911
912 if (charge?.eventChargeLimitReached) {
913 if (charge.chargedCount === 0) {
914
915
916
917 this.chargeLimitReached = true;
918 this.chargeLimitDropped++;
919 return false;
920 }
921
922 this.chargeLimitReached = true;
923 }
924 this.countDelivered(product);
925 return true;
926 } catch (error: any) {
927 this.pushChargeFailures++;
928 log.warning(`Charged push rejected (${error?.message}); retrying without charge`);
929 try {
930 await Actor.pushData(product);
931 this.countDelivered(product);
932 return true;
933 } catch (e2: any) {
934 this.pushDropped++;
935 log.error(`Plain push also failed (${e2?.message}) — record dropped`);
936 return false;
937 }
938 }
939 }
940
941
942
943
944
945
946 private targetKey(): string {
947 return this.target.kind === 'query' ? `query:${this.target.query}` : `grid:${this.target.url}`;
948 }
949
950
951
952
953
954
955
956
957
958 private async resumePreviousAttempt(): Promise<void> {
959 const previous = await Actor.getValue<ResumeState>(RESUME_KEY).catch(() => null);
960
961
962
963 if (previous && previous.targetKey === this.targetKey()) {
964 this.attempt = (previous.attempts ?? 1) + 1;
965 }
966 if (!previous || previous.targetKey !== this.targetKey()) return;
967 if (previous.itemsDelivered <= 0 && previous.nextPage <= this.target.startPage) return;
968
969 this.resumedItems = previous.itemsDelivered;
970 this.itemCount = previous.itemsDelivered;
971
972
973 const rs = previous.ratingStats;
974 if (rs) {
975 this.ratingsMissing = rs.ratingsMissing ?? 0;
976 this.reviewCountsMissing = rs.reviewCountsMissing ?? 0;
977 this.ratingsAbsent = rs.ratingsAbsent ?? 0;
978 this.ratingsUnresolved = rs.ratingsUnresolved ?? 0;
979 this.reviewCountsAbsent = rs.reviewCountsAbsent ?? 0;
980 this.reviewCountsUnresolved = rs.reviewCountsUnresolved ?? 0;
981 this.ratingSources = { ...(rs.ratingSources ?? {}) };
982 this.reviewCountSources = { ...(rs.reviewCountSources ?? {}) };
983 }
984 this.deliveredIds = [...(previous.deliveredProductIds ?? [])];
985 for (const id of this.deliveredIds) this.seen.add(id);
986
987
988
989
990
991
992 for (const p of previous.pagesOpened ?? []) this.pagesOpened.add(p);
993 for (const p of previous.pagesDelivered ?? []) this.pagesDelivered.add(p);
994 this.nextPage = Math.max(this.target.startPage, previous.nextPage);
995
996 log.info(
997 `Resuming a previous attempt: ${this.resumedItems} product(s) are already in the dataset `
998 + `and were already charged. They will NOT be scraped or billed again — this run starts at `
999 + `results page ${this.nextPage} and collects the remaining `
1000 + `${Math.max(0, this.input.maxItems - this.itemCount)}.`,
1001 );
1002 }
1003
1004
1005
1006
1007
1008 private async maybeCheckpoint(): Promise<void> {
1009 if (!this.dirty || this.finalWritten || this.finalWriteStarted) return;
1010 const early = this.checkpointWrites < ALWAYS_CHECKPOINT_WRITES;
1011 if (!early && Date.now() - this.lastCheckpointAt < CHECKPOINT_INTERVAL_MS) return;
1012 await this.writeOutput(false);
1013 }
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039 private async writeOutput(isFinal: boolean): Promise<void> {
1040 if (this.finalWritten) return;
1041 if (this.finalWriteStarted && !isFinal) return;
1042 if (isFinal) this.finalWriteStarted = true;
1043 const output = this.buildRunOutput(isFinal);
1044 const resume: ResumeState = {
1045 targetKey: this.targetKey(),
1046 itemsDelivered: this.itemCount,
1047 nextPage: this.nextPage,
1048 deliveredProductIds: this.deliveredIds,
1049 pagesOpened: [...this.pagesOpened],
1050 pagesDelivered: [...this.pagesDelivered],
1051 attempts: this.attempt,
1052 ratingStats: this.ratingStats(),
1053 updatedAt: new Date().toISOString(),
1054 };
1055 const budget = isFinal ? SHUTDOWN_WRITE_TIMEOUT_MS : CHECKPOINT_INTERVAL_MS;
1056
1057
1058
1059
1060 const previous = this.writeChain;
1061 const run = (async () => {
1062 await withTimeout(previous, isFinal ? FINAL_WRITE_QUEUE_WAIT_MS : budget);
1063
1064
1065 if (!isFinal && (this.finalWriteStarted || this.finalWritten)) return;
1066
1067
1068 const written = await withTimeout(Actor.setValue(OUTPUT_KEY, output).then(() => true), budget);
1069 if (written === null) {
1070 log.warning('Could not write the OUTPUT summary record in time.');
1071 }
1072 await withTimeout(Actor.setValue(RESUME_KEY, resume).then(() => true), budget);
1073
1074 this.lastCheckpointAt = Date.now();
1075 this.checkpointWrites++;
1076 this.dirty = false;
1077 if (isFinal) this.finalWritten = true;
1078 })();
1079 this.writeChain = run.catch(() => { });
1080 await run;
1081 }
1082
1083
1084
1085
1086
1087
1088 private workStranded(): boolean {
1089 if (this.input.maxItems - this.itemCount <= 0) return false;
1090 if (this.interruption !== null) return true;
1091 if (this.chargeLimitReached) return true;
1092 if (STRANDING_STOP_REASONS.has(this.stopReason)) return true;
1093
1094
1095
1096
1097 if (this.pagesVisited === 0 && this.itemCount > 0) return true;
1098
1099
1100
1101 return this.lastFailure !== null && this.pagesVisited > 0;
1102 }
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113 private ratingStats(): RatingStats {
1114 return {
1115 ratingsMissing: this.ratingsMissing,
1116 reviewCountsMissing: this.reviewCountsMissing,
1117 ratingsAbsent: this.ratingsAbsent,
1118 ratingsUnresolved: this.ratingsUnresolved,
1119 reviewCountsAbsent: this.reviewCountsAbsent,
1120 reviewCountsUnresolved: this.reviewCountsUnresolved,
1121 ratingSources: { ...this.ratingSources },
1122 reviewCountSources: { ...this.reviewCountSources },
1123 };
1124 }
1125
1126
1127 private listingsAccounted(): number {
1128 return Object.values(this.ratingSources).reduce((a, b) => a + b, 0)
1129 + this.ratingsAbsent + this.ratingsUnresolved;
1130 }
1131
1132 private ratingFillRate(): number | null {
1133 const resolved = Object.values(this.ratingSources).reduce((a, b) => a + b, 0);
1134 const published = resolved + this.ratingsUnresolved;
1135 if (published === 0) return null;
1136 return Math.round((resolved / published) * 1000) / 1000;
1137 }
1138
1139
1140 private yieldPerPage(): number {
1141 if (this.pagesVisited === 0) return 48;
1142 const deliveredHere = this.itemCount - this.resumedItems;
1143
1144
1145 const basis = this.pageCutShort !== null || deliveredHere === 0
1146 ? this.listingsSeen
1147 : deliveredHere;
1148 return Math.max(1, Math.round(basis / this.pagesVisited));
1149 }
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160 private pagesNotRun(): number[] {
1161 if (!this.workStranded()) return [];
1162 const short = this.input.maxItems - this.itemCount;
1163 const needed = Math.min(MAX_PAGES, Math.max(1, Math.ceil(short / this.yieldPerPage())));
1164 const pages: number[] = [];
1165 for (let p = this.nextPage; pages.length < needed && p < this.nextPage + MAX_PAGES * 2; p++) {
1166 if (!this.pagesOpened.has(p)) pages.push(p);
1167 }
1168 return pages;
1169 }
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192 private rerunOverlapsDelivered(): boolean {
1193 return this.pagesDelivered.has(this.nextPage);
1194 }
1195
1196 private buildRerunInput(): Record<string, unknown> | null {
1197 if (!this.workStranded()) return null;
1198 const rerun: Record<string, unknown> = {
1199
1200 searchUrl: this.pageUrl(this.nextPage),
1201 maxItems: this.input.maxItems - this.itemCount,
1202 };
1203 if (this.filterActive(this.input.minRating)) rerun.minRating = this.input.minRating;
1204 if (this.filterActive(this.input.minReviews)) rerun.minReviews = this.input.minReviews;
1205 if (this.filterActive(this.input.priceMin)) rerun.priceMin = this.input.priceMin;
1206 if (this.filterActive(this.input.priceMax)) rerun.priceMax = this.input.priceMax;
1207 return rerun;
1208 }
1209
1210
1211
1212
1213
1214
1215 installShutdownHandlers(): void {
1216 Actor.on('persistState', () => {
1217 void this.maybeCheckpoint().catch(() => { });
1218 });
1219
1220
1221
1222
1223 Actor.on('aborting', () => this.shutdown('aborted'));
1224 Actor.on('migrating', () => this.shutdown('migrating'));
1225 for (const signal of ['SIGTERM', 'SIGINT'] as const) {
1226 process.on(signal, () => {
1227 void this.shutdown('signal').then(() => process.exit(0));
1228 });
1229 }
1230 }
1231
1232
1233 private shutdown(reason: InterruptionReason): Promise<void> {
1234 if (this.shutdownPromise) return this.shutdownPromise;
1235
1236 this.interruption = { reason, at: new Date().toISOString() };
1237 this.stopRequested = true;
1238 this.dirty = true;
1239
1240 setTimeout(() => process.exit(0), SHUTDOWN_TOTAL_TIMEOUT_MS).unref?.();
1241
1242 log.warning(
1243 `Run is being stopped (${reason}). Writing a summary of what was delivered and what is `
1244 + `still missing to the key-value store record "${OUTPUT_KEY}" before exiting...`,
1245 );
1246
1247 this.shutdownPromise = (async () => {
1248
1249
1250
1251 await this.settlePushes();
1252 await this.writeOutput(true).catch(() => { });
1253 log.info('Summary written. Exiting.');
1254 })();
1255 return this.shutdownPromise;
1256 }
1257
1258
1259 private async settlePushes(): Promise<void> {
1260 const until = Date.now() + SHUTDOWN_SETTLE_TIMEOUT_MS;
1261 while (this.pushing && Date.now() < until) {
1262 await new Promise((r) => setTimeout(r, 25));
1263 }
1264 }
1265
1266
1267
1268
1269
1270
1271 private buildRunOutput(isFinal: boolean): RunOutput {
1272 const requested = this.input.maxItems;
1273 const filtersActive = this.filterActive(this.input.minRating)
1274 || this.filterActive(this.input.minReviews)
1275 || this.filterActive(this.input.priceMin)
1276 || this.filterActive(this.input.priceMax);
1277 const notRun = this.pagesNotRun();
1278 const rerunInput = this.buildRerunInput();
1279
1280
1281 const overlaps = rerunInput !== null && this.rerunOverlapsDelivered();
1282
1283 let outcome: RunOutcome;
1284 let message: string;
1285
1286 if (this.interruption) {
1287
1288
1289 outcome = 'interrupted';
1290 const cause = INTERRUPTION_CAUSE_TEXT[this.interruption.reason];
1291 message = `This run was stopped before it finished. DELIVERED: ${this.itemCount} of the `
1292 + `${requested} product(s) you asked for — those are in the dataset and are complete. `
1293 + (rerunInput
1294 ? `NOT DELIVERED: ${requested - this.itemCount} product(s); results page `
1295 + `${this.nextPage} onwards was never finished. `
1296 : '')
1297 + `WHY: ${cause} `
1298 + (rerunInput
1299
1300
1301
1302
1303 ? 'WHAT TO DO: press RESURRECT on this run — it carries on where it stopped and does '
1304 + 'not charge again for any listing you already have. '
1305 + (overlaps
1306 ? 'Alternatively, "rerunInput" below restarts at results page '
1307 + `${this.nextPage}, which this run has already delivered listings from: `
1308 + 'those listings would be delivered — and charged — a second time.'
1309 : 'Alternatively, "rerunInput" below starts at results page '
1310 + `${this.nextPage}, which this run never delivered from, so it collects `
1311 + 'exactly what is missing without billing anything twice.')
1312 : 'Nothing is missing: everything you asked for was delivered before the stop.');
1313 } else if (!isFinal) {
1314
1315 outcome = 'in-progress';
1316 message = `Run in progress: ${this.itemCount} of ${requested} product(s) delivered so far `
1317 + `(${this.pagesVisited} results page(s) scraped). This record is refreshed as the run `
1318 + `proceeds, so it always reflects what is actually in the dataset.`;
1319 } else if (this.pagesVisited === 0 && this.itemCount === 0) {
1320 outcome = 'blocked';
1321 message = `Could not load any Etsy results page for ${this.target.label}. `
1322 + `Last error: ${this.lastFailure ?? 'unknown'}. Nothing was extracted; `
1323 + `re-run to get a different residential IP.`;
1324 } else if (this.pagesVisited === 0) {
1325
1326
1327 outcome = 'partial';
1328 message = this.lastFailure
1329 ? `Could not load results page ${this.nextPage} for ${this.target.label}, so nothing was `
1330 + `added to the ${this.itemCount} product(s) a previous attempt already delivered. `
1331 + `Last error: ${this.lastFailure}.`
1332
1333
1334 : `This attempt opened no results page and added nothing to the ${this.itemCount} `
1335 + `product(s) a previous attempt already delivered. No error was reported. `
1336 + `${requested - this.itemCount} of the ${requested} products you requested are `
1337 + `still missing — resurrect again, or use "rerunInput".`;
1338 } else if (this.listingsSeen === 0 && this.noResultsSignal) {
1339
1340 outcome = 'no-results';
1341 message = `Etsy returned no listings for ${this.target.label}. The page explicitly reported `
1342 + `no matches, so there is nothing to scrape — try a broader search term.`;
1343 } else if (this.listingsSeen === 0 && this.itemCount === 0 && this.lastFailure) {
1344
1345
1346 outcome = 'blocked';
1347 message = `Etsy served no listing grid for ${this.target.label} on any attempt — the requests were `
1348 + `blocked. Last error: ${this.lastFailure}. Nothing was extracted; re-run to get a `
1349 + `different residential IP.`;
1350 } else if (this.listingsSeen === 0 && this.itemCount === 0) {
1351 outcome = 'extraction-failed';
1352 message = `Loaded a page for ${this.target.label} but found no listing cards and no "no results" `
1353 + `message from Etsy. That means the page was blocked or Etsy's structure changed — `
1354 + `the run returned nothing rather than reporting a false success.`;
1355 } else if (this.uniqueExtracted === 0 && this.itemCount === 0) {
1356 outcome = 'extraction-failed';
1357 message = `Found ${this.listingsSeen} listing cards for ${this.target.label} but could not parse `
1358 + `a single one into a product record. Etsy's card markup has likely changed.`;
1359 } else if (this.itemCount === 0 && this.chargeLimitReached) {
1360
1361
1362 outcome = 'charge-limit-reached';
1363 message = `Found ${this.listingsSeen} listings for ${this.target.label} but saved none: this run's `
1364 + `maximum total charge was reached before the first product could be billed. Raise the run's `
1365 + `"Max total charge" limit (or your account budget) and re-run.`;
1366 } else if (this.itemCount === 0 && !filtersActive) {
1367 outcome = 'extraction-failed';
1368 message = `Parsed ${this.uniqueExtracted} listings for ${this.target.label} but none of them could `
1369 + `be saved to the dataset (${this.pushDropped} rejected on write). No filters were set, so `
1370 + `this is a storage or schema fault, not your input.`;
1371 } else if (this.itemCount === 0 && this.droppedUnfilterable > 0 && this.droppedByFilter === 0) {
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381 if (this.unfilterableFaults() > 0) {
1382 outcome = 'filters-unusable';
1383 message = `Found ${this.listingsSeen} listings for ${this.target.label}, but the value your `
1384 + `filter (${this.describeFilters()}) needs could not be READ from any of them: `
1385 + `${this.describeUnfilterable()}. ${this.unfilterableFaults()} of those `
1386 + `${this.droppedUnfilterable} withheld listing(s) are a fault in this actor, not in `
1387 + `your input — Etsy's card markup has probably changed. Nothing was returned rather `
1388 + `than guessing. Re-run without the filter to get the raw listings, and please report `
1389 + `this run.`;
1390 } else {
1391
1392
1393 outcome = 'zero-after-filter';
1394 message = `Found ${this.listingsSeen} listings for ${this.target.label}, but none of them `
1395 + `published the value your filter needs (${this.describeFilters()}) — all `
1396 + `${this.droppedUnfilterable} were withheld because ${this.describeUnfilterable()}. `
1397 + `This actor cannot claim a listing matches a filter whose value the listing never `
1398 + `published. Nothing is broken; drop the filter to get these listings, or search a `
1399 + `term with more established shops.`;
1400 }
1401 } else if (this.itemCount === 0) {
1402 outcome = 'zero-after-filter';
1403
1404
1405
1406
1407 message = `Found ${this.listingsSeen} listings for ${this.target.label}, but none matched your `
1408 + `filters (${this.describeFilters()}). ${this.droppedByFilter} were checked and out of `
1409 + `range`
1410 + (this.droppedUnfilterable > 0
1411 ? `, ${this.droppedUnfilterable} could not be checked at all: ${this.describeUnfilterable()}`
1412 : '')
1413 + `. Loosen the filters or re-run without them.`;
1414 } else if (this.workStranded()) {
1415
1416
1417
1418 outcome = 'partial';
1419 message = `Partial result: ${this.itemCount} of the ${requested} products you requested. `
1420 + `Stopped because ${this.describeStop()}. The ${requested - this.itemCount} still `
1421 + `missing are not in the dataset and you were not charged for them. `
1422
1423
1424
1425
1426 + `WHAT TO DO: press RESURRECT on this run — it carries on from results page `
1427 + `${this.nextPage} and does not charge again for any listing you already have.`
1428 + (overlaps
1429 ? ` Alternatively the "rerunInput" field below restarts at results page `
1430 + `${this.nextPage}, which this run has already delivered listings from: a fresh run has no `
1431 + `memory of this one's dataset, so it would re-deliver and re-charge the `
1432 + `listings already collected from that page. Resurrect avoids that.`
1433 : ` Alternatively the "rerunInput" field below is a ready-to-paste input that `
1434 + `collects exactly what is missing, starting at results page ${this.nextPage} `
1435 + `— a page this run never delivered from, so nothing is billed twice.`);
1436 } else {
1437 outcome = 'ok';
1438 message = `Scraped ${this.itemCount} products from ${this.target.label}.`
1439
1440
1441
1442
1443
1444
1445
1446
1447 + (this.droppedUnfilterable > 0
1448 ? ` ${this.droppedUnfilterable} further listing(s) were withheld because the value `
1449 + `your filter needs was not available for them: ${this.describeUnfilterable()}.`
1450 : '');
1451 }
1452
1453 const now = new Date().toISOString();
1454 return {
1455 outcome,
1456 message,
1457 target: this.target.label,
1458 itemsSaved: this.itemCount,
1459 itemsRequested: requested,
1460 listingsSeen: this.listingsSeen,
1461 droppedByFilter: this.droppedByFilter,
1462 droppedUnfilterable: this.droppedUnfilterable,
1463 unfilterableReasons: this.unfilterableReasons(),
1464 pagesVisited: this.pagesVisited,
1465 ratingsMissing: this.ratingsMissing,
1466 reviewCountsMissing: this.reviewCountsMissing,
1467
1468
1469
1470 ratingsAbsent: this.ratingsAbsent,
1471 ratingsUnresolved: this.ratingsUnresolved,
1472 reviewCountsAbsent: this.reviewCountsAbsent,
1473 reviewCountsUnresolved: this.reviewCountsUnresolved,
1474 ratingFillRate: this.ratingFillRate(),
1475 ratingSources: { ...this.ratingSources },
1476 reviewCountSources: { ...this.reviewCountSources },
1477 chargeLimitReached: this.chargeLimitReached,
1478 finishedAt: now,
1479
1480
1481 stopReason: this.interruption ? 'interrupted' : this.stopReason,
1482 interrupted: this.interruption !== null,
1483 interruptionReason: this.interruption?.reason ?? null,
1484 pagesNotRun: notRun,
1485 pagesNotRunIsApproximate: notRun.length > 0,
1486
1487
1488 pageInProgress: this.pageInProgress ?? this.pageCutShort,
1489 itemsNotDelivered: Math.max(0, requested - this.itemCount),
1490 itemsFromPreviousAttempt: this.resumedItems,
1491 isFinal,
1492 checkpointAt: now,
1493 rerunInput,
1494 rerunInputOverlapsDelivered: overlaps,
1495 };
1496 }
1497
1498
1499
1500
1501
1502 private async reportOutcome(): Promise<void> {
1503
1504 if (this.finalWritten) return;
1505
1506 const output = this.buildRunOutput(true);
1507 const fail = FAILING_OUTCOMES.has(output.outcome);
1508 await this.writeOutput(true);
1509
1510
1511
1512 if (this.listingsAccounted() > 0) {
1513 const resolved = Object.values(this.ratingSources).reduce((a, b) => a + b, 0);
1514 const rate = this.ratingFillRate();
1515 const via = Object.entries(this.ratingSources)
1516 .sort((a, b) => b[1] - a[1])
1517 .map(([k, v]) => `${k}=${v}`)
1518 .join(', ') || 'none';
1519 log.info(
1520 `Ratings: ${resolved}/${this.listingsAccounted()} listings resolved (via ${via}); `
1521 + `${this.ratingsAbsent} publish no rating at all (new listings — null is correct); `
1522 + `${this.ratingsUnresolved} published rating markup we could NOT read. `
1523 + `Fill rate over listings that published a rating: `
1524 + `${rate === null ? 'n/a (none published one)' : `${Math.round(rate * 100)}%`}.`,
1525 );
1526 if (this.ratingsUnresolved > 0) {
1527 log.warning(
1528 `${this.ratingsUnresolved} listing(s) published a rating this actor could not read — `
1529 + `those nulls are an EXTRACTION FAULT, not unrated listings. Etsy's card markup has `
1530 + `probably changed; the rating region of a sample is in the key-value store as `
1531 + `rating-unresolved-sample-page-*.html. Please report this run.`,
1532 );
1533 }
1534 if (this.reviewCountsUnresolved > 0) {
1535 log.warning(
1536 `${this.reviewCountsUnresolved} listing(s) published a review count this actor could `
1537 + `not read — those nulls are an extraction fault, not listings without reviews.`,
1538 );
1539 }
1540 }
1541 if (this.gridSettleMissedPages > 0) {
1542 log.warning(
1543 `${this.gridSettleMissedPages} page(s) were still rendering listing cards when the settle `
1544 + `budget expired, so they may hold fewer listings than Etsy would eventually have shown.`,
1545 );
1546 }
1547 if (this.pushChargeFailures > 0) {
1548 log.warning(
1549 `${this.pushChargeFailures} records failed dataset-schema validation on the charged push `
1550 + `and were saved uncharged. This is a schema mismatch, please report it.`,
1551 );
1552 }
1553 if (this.pushDropped > 0) {
1554 log.error(`${this.pushDropped} records could not be saved at all and are missing from the dataset.`);
1555 }
1556 if (this.chargeLimitDropped > 0) {
1557 log.warning(
1558 `${this.chargeLimitDropped} extracted listing(s) were discarded unsaved because this run's `
1559 + `maximum total charge was reached. They are NOT in the dataset and you were NOT billed `
1560 + `for them. Raise the run's charge limit to collect the rest.`,
1561 );
1562 }
1563
1564 if (fail) {
1565 log.error(output.message);
1566 await Actor.fail(output.message);
1567 return;
1568 }
1569
1570 if (output.outcome === 'zero-after-filter' || output.outcome === 'partial') {
1571 log.warning(output.message);
1572 } else {
1573 log.info(output.message);
1574 }
1575 log.info(`Done. ${this.itemCount} products in the dataset.`);
1576 if (output.pagesNotRun.length > 0) {
1577 log.warning(
1578 `Results page(s) ${output.pagesNotRun.join(', ')} were never scraped (an estimate — Etsy does `
1579 + `not say how many pages a search has). `
1580
1581
1582
1583
1584 + (output.rerunInputOverlapsDelivered
1585 ? `Press RESURRECT to collect the rest without paying twice: the OUTPUT record's `
1586 + `"rerunInput" restarts at page ${this.nextPage}, which this run has already `
1587 + `delivered listings from, so a fresh run would re-charge for listings you `
1588 + `already have.`
1589 : `The OUTPUT record holds a ready-to-paste "rerunInput" that collects exactly `
1590 + `what is missing.`),
1591 );
1592 }
1593 log.info(`Full run detail is in the key-value store record "${OUTPUT_KEY}".`);
1594
1595
1596 if (this.itemCount > 0 && !this.interruption && output.rerunInput === null) {
1597 log.info('----------------------------------------');
1598 log.info('Useful? A 30-second review helps others find this actor');
1599 log.info('and helps us keep it maintained:');
1600 log.info('https://apify.com/webdatalabs/etsy-scraper-pro');
1601 log.info('----------------------------------------');
1602 }
1603 }
1604
1605 private describeFilters(): string {
1606 const parts: string[] = [];
1607 if (this.filterActive(this.input.minRating)) parts.push(`minRating=${this.input.minRating}`);
1608 if (this.filterActive(this.input.minReviews)) parts.push(`minReviews=${this.input.minReviews}`);
1609 if (this.filterActive(this.input.priceMin)) parts.push(`priceMin=${this.input.priceMin}`);
1610 if (this.filterActive(this.input.priceMax)) parts.push(`priceMax=${this.input.priceMax}`);
1611 return parts.join(', ');
1612 }
1613
1614 private describeStop(): string {
1615 switch (this.stopReason) {
1616 case 'blocked': return 'Etsy blocked a later results page';
1617 case 'nav-timeout': return 'a results page stopped responding';
1618 case 'max-pages': return `the ${MAX_PAGES}-page safety cap was reached`;
1619 case 'time-budget': return 'the run reached its per-session time budget';
1620 case 'charge-limit': return "this run's maximum total charge was reached";
1621 default:
1622
1623
1624
1625 return this.lastFailure
1626 ? `Etsy stopped serving results pages part-way through (last error: ${this.lastFailure})`
1627 : 'the results ran out';
1628 }
1629 }
1630}
1631
1632
1633
1634
1635
1636
1637
1638
1639function resolveTarget(input: ValidatedInput): Target {
1640 const q = (input.query || '').trim();
1641
1642 if (q) {
1643 if (input.searchUrl) {
1644 try {
1645 const fromUrl = classifyEtsyUrl(input.searchUrl);
1646 const same = fromUrl.kind === 'query' && fromUrl.query.toLowerCase() === q.toLowerCase();
1647 if (!same) {
1648 log.warning(
1649 `Both "query" and "searchUrl" were provided and they disagree. Using query "${q}" `
1650 + `and ignoring searchUrl (${input.searchUrl}). Clear "query" to use the URL instead.`,
1651 );
1652 }
1653 } catch (e: any) {
1654 log.warning(
1655 `Ignoring searchUrl and using query "${q}" instead. searchUrl problem: `
1656 + `${String(e?.message).split('\n')[0]}`,
1657 );
1658 }
1659 }
1660 return { kind: 'query', query: q, label: `search "${q}"`, startPage: 1 };
1661 }
1662
1663 if (input.searchUrl) return classifyEtsyUrl(input.searchUrl);
1664
1665
1666 log.info('No search target provided — defaulting to query "handmade jewelry"');
1667 return {
1668 kind: 'query',
1669 query: 'handmade jewelry',
1670 label: 'search "handmade jewelry" (default)',
1671 startPage: 1,
1672 };
1673}
1674
1675async function failInvalidInput(message: string): Promise<void> {
1676 log.error(message);
1677 await Actor.setValue('OUTPUT', {
1678 outcome: 'invalid-input' as RunOutcome,
1679 message,
1680 target: null,
1681 itemsSaved: 0,
1682 finishedAt: new Date().toISOString(),
1683
1684
1685 interrupted: false,
1686 interruptionReason: null,
1687 pagesNotRun: [],
1688 pagesNotRunIsApproximate: false,
1689 itemsNotDelivered: 0,
1690 isFinal: true,
1691 rerunInput: null,
1692 rerunInputOverlapsDelivered: false,
1693 }).catch(() => { });
1694 await Actor.fail(message.split('\n')[0]);
1695}
1696
1697Actor.main(async () => {
1698 log.info('Starting Etsy Scraper Pro');
1699
1700 const rawInput = await Actor.getInput();
1701
1702 let input: ValidatedInput;
1703 try {
1704 input = InputSchema.parse(rawInput || {});
1705 } catch (error: any) {
1706 const issues = (error?.issues || [])
1707 .map((i: any) => `${(i.path || []).join('.') || '(root)'}: ${i.message}`)
1708 .join('; ');
1709 await failInvalidInput(
1710 `Invalid input: ${issues || error?.message}.\n`
1711 + ` A correct input looks like:\n`
1712 + ` { "query": "vintage watch", "maxItems": 48 }\n`
1713 + ` or { "searchUrl": "https://www.etsy.com/search?q=vintage%20watch", "maxItems": 48 }`,
1714 );
1715 return;
1716 }
1717
1718
1719
1720 let target: Target;
1721 try {
1722 target = resolveTarget(input);
1723 } catch (error: any) {
1724 if (error instanceof InputError) {
1725 await failInvalidInput(error.message);
1726 return;
1727 }
1728 throw error;
1729 }
1730
1731 const scraper = new EtsyScraper(input, target);
1732
1733 scraper.installShutdownHandlers();
1734 await scraper.execute();
1735});
![Etsy Reviews Pro [pay per event] avatar](https://images.apifyusercontent.com/YezdUg_vmrCiQJdCfW9Nq54VIEg9ewbPUPEGrUtHzXM/rs:fill:76:76/cb:1/aHR0cHM6Ly9hcGlmeS1pbWFnZS11cGxvYWRzLXByb2QuczMudXMtZWFzdC0xLmFtYXpvbmF3cy5jb20vdVdIRWc2cmM2NkdBMTdYYWgvYXdPRFM3TmM5MmUzWVYwQ3ctZXRzeV9sb2dvLnBuZw.webp){kind=avatar}
