1
2
3
4
5
6process.env.REBROWSER_PATCHES_RUNTIME_FIX_MODE ||= 'addBinding';
7process.env.REBROWSER_PATCHES_SOURCE_URL ||= 'app.js';
8process.env.REBROWSER_PATCHES_UTILITY_WORLD_NAME ||= 'util';
9
10import { Actor, log } from 'apify';
11import { PlaywrightCrawler, createPlaywrightRouter } from 'crawlee';
12import { chromium } from 'rebrowser-playwright';
13import {
14 InputSchema,
15 ValidatedInput,
16 EtsyProduct,
17 ExtractionResult,
18 RunOutput,
19 RunOutcome,
20} from './types.js';
21import { HumanBehavior } from './human-behavior.js';
22import { DataDomeSolver } from './datadome-solver.js';
23import { extractProductsInPage } from './extract.js';
24import { Target, InputError, classifyEtsyUrl, gridPageUrl } from './target.js';
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46const MAX_PAGES = 12;
47
48
49const HANDLER_BUDGET_MS = 240_000;
50
51type StopReason =
52 | 'reached-max-items'
53 | 'end-of-results'
54 | 'max-pages'
55 | 'blocked'
56 | 'nav-timeout'
57 | 'time-budget'
58 | 'charge-limit';
59
60class EtsyScraper {
61 private input: ValidatedInput;
62 private target: Target;
63 private dataDomeSolver: DataDomeSolver;
64
65 private itemCount = 0;
66 private seen = new Set<string>();
67 private pagesVisited = 0;
68 private listingsSeen = 0;
69 private uniqueExtracted = 0;
70 private droppedByFilter = 0;
71 private droppedUnfilterable = 0;
72 private ratingsMissing = 0;
73 private reviewCountsMissing = 0;
74 private pushChargeFailures = 0;
75 private pushDropped = 0;
76
77
78
79 private chargeLimitReached = false;
80 private chargeLimitDropped = 0;
81 private stopReason: StopReason = 'end-of-results';
82 private lastFailure: string | null = null;
83 private ratingSelectorMissedPages = 0;
84 private noResultsSignal = false;
85
86 constructor(input: ValidatedInput, target: Target) {
87 this.input = input;
88 this.target = target;
89 const apiKey = process.env.CAPSOLVER_API_KEY || '';
90 this.dataDomeSolver = new DataDomeSolver(apiKey);
91 }
92
93 async run(): Promise<void> {
94 const proxyConfiguration = await Actor.createProxyConfiguration({
95 groups: ['RESIDENTIAL'],
96 countryCode: 'US',
97 });
98 log.info('Using US residential proxies');
99
100 const router = createPlaywrightRouter();
101 router.addHandler('SEARCH', async ({ page, session }) => {
102 log.info(`Target: ${this.target.label}`);
103 const deadline = Date.now() + HANDLER_BUDGET_MS;
104
105
106 try {
107 await page.waitForLoadState('domcontentloaded', { timeout: 30000 });
108 } catch { }
109 await this.dumpState(page, 'homepage');
110 if (!(await this.handleBlock(page, session, 'homepage'))) {
111 throw new Error('Homepage blocked (DataDome) — rotating session');
112 }
113
114 const human = new HumanBehavior(page);
115 await human.initialize();
116 await this.naturalDelay(1500, 2500);
117 await human.naturalScroll(1);
118 await human.randomMouseMovements(2);
119
120
121 if (this.target.kind === 'query') {
122
123 const searchInput = await this.findSearchInput(page);
124 if (!searchInput) throw new Error('Search input not found on homepage');
125 await searchInput.click();
126 await this.naturalDelay(250, 500);
127 for (const ch of this.target.query) {
128 await page.keyboard.type(ch, { delay: 45 + Math.random() * 90 });
129 }
130 await this.naturalDelay(400, 800);
131 await page.keyboard.press('Enter');
132 } else {
133
134 await page.goto(this.target.url, { waitUntil: 'domcontentloaded', timeout: 60000 });
135 }
136 await page.waitForLoadState('domcontentloaded', { timeout: 30000 }).catch(() => {});
137
138
139
140 await this.dumpState(page, 'search-results');
141 if (!(await this.handleBlock(page, session, 'search-results'))) {
142 throw new Error('Search results blocked (DataDome) — rotating session');
143 }
144 if (!(await this.settleGrid(page, human))) {
145
146
147 await this.dumpState(page, 'search-results-no-grid', true);
148 if (!(await this.handleBlock(page, session, 'search-results-late'))) {
149 throw new Error('Search results blocked (DataDome, late) — rotating session');
150 }
151 }
152
153
154 const first = await this.scrapeCurrentPage(page, 1);
155 if (first.cardCount === 0) {
156 if (this.noResultsSignal) {
157
158 this.stopReason = 'end-of-results';
159 return;
160 }
161
162
163
164
165
166
167
168 await this.dumpState(page, 'no-grid-page-1', true);
169 throw new Error(
170 'Search results page rendered no listing grid and no "no results" message '
171 + '— retrying on a fresh session',
172 );
173 }
174
175 for (let pageNum = 2; pageNum <= MAX_PAGES; pageNum++) {
176 if (this.itemCount >= this.input.maxItems) {
177 this.stopReason = 'reached-max-items';
178 return;
179 }
180
181
182
183 if (this.chargeLimitReached) {
184 log.warning('Charge budget (maxTotalChargeUsd) is exhausted — stopping.');
185 this.stopReason = 'charge-limit';
186 return;
187 }
188
189
190 if (Date.now() > deadline) {
191 log.warning('Reached the per-session time budget — stopping with results so far.');
192 this.stopReason = 'time-budget';
193 return;
194 }
195 const nextUrl = this.target.kind === 'query'
196 ? `https://www.etsy.com/search?q=${encodeURIComponent(this.target.query)}&page=${pageNum}`
197 : gridPageUrl(this.target.url, pageNum);
198 log.info(`Page ${pageNum}...`);
199 try {
200 await page.goto(nextUrl, { waitUntil: 'domcontentloaded', timeout: 60000 });
201 } catch {
202 log.warning(`Pagination navigation to page ${pageNum} timed out — stopping with what we have.`);
203 this.stopReason = 'nav-timeout';
204 return;
205 }
206
207
208 const info = await this.getBlockInfo(page);
209 if (info.blocked) {
210 await this.dumpState(page, `block-page-${pageNum}`, true);
211 log.warning(`Page ${pageNum} blocked by DataDome — stopping with results so far.`);
212 this.stopReason = 'blocked';
213 return;
214 }
215 await this.settleGrid(page, human);
216 const res = await this.scrapeCurrentPage(page, pageNum);
217
218
219
220 if (res.cardCount === 0 || res.newUnique === 0) {
221 log.info('No further new listings — reached the end of the results.');
222 this.stopReason = 'end-of-results';
223 return;
224 }
225 this.stopReason = 'max-pages';
226 }
227 });
228
229 const crawler = new PlaywrightCrawler({
230 proxyConfiguration,
231 requestHandlerTimeoutSecs: 300,
232 maxRequestRetries: 5,
233 useSessionPool: true,
234 persistCookiesPerSession: true,
235 requestHandler: router,
236 failedRequestHandler: async ({ request }) => {
237 this.lastFailure = request.errorMessages?.slice(-1)[0]
238 ?? 'request failed with no error message';
239 log.error(`All ${request.retryCount + 1} attempts failed: ${this.lastFailure}`);
240 },
241 maxConcurrency: 1,
242 browserPoolOptions: {
243
244 useFingerprints: false,
245 },
246 sessionPoolOptions: {
247 blockedStatusCodes: [],
248 maxPoolSize: 10,
249 },
250 launchContext: {
251 launcher: chromium,
252 launchOptions: {
253 headless: false,
254
255
256
257 args: [
258 '--blink-settings=imagesEnabled=false',
259 '--disable-remote-fonts',
260 ],
261 },
262 },
263 preNavigationHooks: [
264 async ({ page }, gotoOptions) => {
265 await page.setViewportSize({ width: 1920, height: 1080 });
266 gotoOptions.waitUntil = 'domcontentloaded';
267 gotoOptions.timeout = 60000;
268 },
269 ],
270 });
271
272 log.info('Starting scraper');
273 await crawler.run([{ url: 'https://www.etsy.com', label: 'SEARCH' }]);
274
275 await this.reportOutcome();
276 }
277
278
279
280
281
282
283
284 private async settleGrid(page: any, human: HumanBehavior): Promise<boolean> {
285 const grid = await page
286 .waitForSelector('[data-palette-listing-id]', { timeout: 15000 })
287 .catch(() => null);
288 if (!grid) {
289 log.warning('Listing grid did not appear within 15s — the page is blocked, empty or restructured.');
290 return false;
291 }
292
293 await human.naturalScroll(1);
294
295
296
297
298
299
300
301
302 const settled = await page.waitForFunction(
303 () => {
304 const w = window as any;
305 const n = document.querySelectorAll(
306 '[aria-label*="star rating"], [aria-label*="out of 5 stars"]',
307 ).length;
308 if (n === 0) return false;
309 if (w.__etsyRatedPrev === n) {
310 w.__etsyRatedStable = (w.__etsyRatedStable || 0) + 1;
311 } else {
312 w.__etsyRatedPrev = n;
313 w.__etsyRatedStable = 0;
314 }
315 return w.__etsyRatedStable >= 3;
316 },
317 { timeout: 12000, polling: 400 },
318 ).catch(() => null);
319 if (!settled) this.ratingSelectorMissedPages++;
320 return true;
321 }
322
323
324 private async scrapeCurrentPage(
325 page: any,
326 pageNum: number,
327 ): Promise<{ cardCount: number; newUnique: number; saved: number }> {
328 let res: ExtractionResult;
329 try {
330 res = await page.evaluate(extractProductsInPage);
331 } catch (e: any) {
332 log.warning(`Extraction on page ${pageNum} failed: ${e?.message}`);
333 return { cardCount: 0, newUnique: 0, saved: 0 };
334 }
335
336 this.pagesVisited++;
337 this.listingsSeen += res.cardCount;
338 if (res.noResultsSignal) this.noResultsSignal = true;
339 log.info(
340 `Page ${pageNum}: ${res.cardCount} listing cards, ${res.items.length} parsed, ` +
341 `${res.ratingElCount} with a rating, ${res.reviewCountElCount} with a review count`,
342 );
343
344 if (res.cardCount > 0 && (res.ratingElCount === 0 || res.reviewCountElCount === 0)) {
345 log.warning(
346 `Page ${pageNum}: no listing exposed a ${res.ratingElCount === 0 ? 'rating' : 'review count'} ` +
347 `— that field will be null for these records (null means "not published on the card", ` +
348 `it is NOT zero). Saving a sample card to the key-value store as ` +
349 `rating-missing-sample-page-${pageNum}.html for diagnosis.`,
350 );
351 if (res.sampleCardHtml) {
352 await Actor.setValue(`rating-missing-sample-page-${pageNum}.html`, res.sampleCardHtml, {
353 contentType: 'text/plain; charset=utf-8',
354 }).catch(() => { });
355 }
356 }
357
358 let newUnique = 0;
359 let saved = 0;
360 for (const product of res.items) {
361 if (this.itemCount >= this.input.maxItems) break;
362
363 if (this.chargeLimitReached) break;
364 if (this.seen.has(product.productId)) continue;
365 this.seen.add(product.productId);
366 newUnique++;
367 this.uniqueExtracted++;
368 if (product.rating === null) this.ratingsMissing++;
369 if (product.reviewCount === null) this.reviewCountsMissing++;
370
371 const verdict = this.applyFilters(product);
372 if (verdict === 'dropped') { this.droppedByFilter++; continue; }
373 if (verdict === 'unfilterable') { this.droppedUnfilterable++; continue; }
374
375 if (await this.pushProduct(product)) {
376 this.itemCount++;
377 saved++;
378 }
379 }
380 log.info(`Page ${pageNum}: saved ${saved} (total ${this.itemCount}/${this.input.maxItems})`);
381 return { cardCount: res.cardCount, newUnique, saved };
382 }
383
384 private async findSearchInput(page: any): Promise<any> {
385 const selectors = [
386 'input#global-enhancements-search-query',
387 'input[name="search_query"]',
388 'input[type="search"]',
389 'input[placeholder*="Search"]',
390 ];
391 for (const sel of selectors) {
392 const el = await page.$(sel);
393 if (el && await el.isVisible().catch(() => false)) return el;
394 }
395 return null;
396 }
397
398
399
400
401
402
403
404 private async dumpState(page: any, label: string, force = false): Promise<void> {
405 try {
406 const info = await page.evaluate(() => ({
407 url: location.href,
408 title: document.title,
409 listings: document.querySelectorAll('[data-palette-listing-id]').length,
410 rated: document.querySelectorAll(
411 '[aria-label*="star rating"], [aria-label*="out of 5 stars"]',
412 ).length,
413 }));
414 log.info(`[${label}] "${info.title}" | listings=${info.listings} | rated=${info.rated}`);
415 } catch (e: any) {
416 log.warning(`[${label}] page probe failed (${e?.message}) — saving artifacts anyway`);
417 }
418
419 if (!force && !this.input.debug) return;
420 try {
421 const png = await page.screenshot({ fullPage: false, timeout: 10000 }).catch(() => null);
422 if (png) await Actor.setValue(`debug-${label}.png`, png, { contentType: 'image/png' });
423 const html = await page.content().catch(() => '');
424 if (html) await Actor.setValue(`debug-${label}.html`, html, { contentType: 'text/html' });
425 } catch (e: any) {
426 log.warning(`dumpState[${label}] persist failed: ${e?.message}`);
427 }
428 }
429
430 private async getBlockInfo(page: any): Promise<{ blocked: boolean; type: 'interstitial' | 'captcha' | 'none'; raw: string | null }> {
431 try {
432 return await page.evaluate(() => {
433 const html = document.documentElement.outerHTML;
434 const ddMatch = html.match(/var dd=\{.*?\}/s);
435 const raw = ddMatch ? ddMatch[0] : null;
436 const hasIframe = !!document.querySelector('iframe[src*="captcha-delivery.com"]');
437 const blocked = hasIframe || html.includes('geo.captcha-delivery.com') || html.includes('dd={');
438 if (!blocked) return { blocked: false, type: 'none' as const, raw };
439 const isCaptcha = /\/captcha\//.test(html) || /'rt'\s*:\s*'c'/.test(raw || '');
440 return { blocked: true, type: isCaptcha ? ('captcha' as const) : ('interstitial' as const), raw };
441 });
442 } catch {
443 return { blocked: false, type: 'none', raw: null };
444 }
445 }
446
447
448 private async handleBlock(page: any, session: any, label: string): Promise<boolean> {
449 let info = await this.getBlockInfo(page);
450 if (!info.blocked) return true;
451
452 log.warning(`[${label}] DataDome block — type=${info.type}`);
453 await this.dumpState(page, `block-${label}`, true);
454
455 if (info.type === 'interstitial') {
456 for (let i = 0; i < 3; i++) {
457 await page.waitForTimeout(2500);
458 info = await this.getBlockInfo(page);
459 if (!info.blocked) {
460 log.info('Interstitial cleared on its own');
461 return true;
462 }
463 }
464 log.warning('Interstitial persisted — retiring session to rotate IP');
465 if (session) session.retire();
466 return false;
467 }
468
469 log.info('Slider captcha — attempting solve');
470 const solved = await this.dataDomeSolver.solveDataDome(page);
471 if (solved) {
472 await page.waitForLoadState('domcontentloaded', { timeout: 30000 }).catch(() => {});
473 return true;
474 }
475 log.warning('Slider solve failed — retiring session to rotate IP');
476 if (session) session.retire();
477 return false;
478 }
479
480
481 private async naturalDelay(minMs: number, maxMs: number): Promise<void> {
482 const mean = (minMs + maxMs) / 2;
483 const stdDev = (maxMs - minMs) / 6;
484 const u1 = Math.random(), u2 = Math.random();
485 const z0 = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
486 const delay = Math.max(minMs, Math.min(maxMs, mean + z0 * stdDev));
487 await new Promise((r) => setTimeout(r, delay));
488 }
489
490
491
492
493
494
495
496
497 private filterActive(v: number | undefined): v is number {
498 return typeof v === 'number' && v > 0;
499 }
500
501
502
503
504
505
506
507
508 private applyFilters(p: EtsyProduct): 'keep' | 'dropped' | 'unfilterable' {
509 if (this.filterActive(this.input.minRating)) {
510 if (p.rating === null) return 'unfilterable';
511 if (p.rating < this.input.minRating!) return 'dropped';
512 }
513 if (this.filterActive(this.input.minReviews)) {
514 if (p.reviewCount === null) return 'unfilterable';
515 if (p.reviewCount < this.input.minReviews!) return 'dropped';
516 }
517 if (this.filterActive(this.input.priceMin) || this.filterActive(this.input.priceMax)) {
518 if (p.price === null) return 'unfilterable';
519 if (this.filterActive(this.input.priceMin) && p.price < this.input.priceMin!) return 'dropped';
520 if (this.filterActive(this.input.priceMax) && p.price > this.input.priceMax!) return 'dropped';
521 }
522 return 'keep';
523 }
524
525
526 private async pushProduct(product: EtsyProduct): Promise<boolean> {
527
528
529
530 try {
531 const charge = await Actor.pushData(product, 'product-scraped');
532
533
534
535
536
537 if (charge?.eventChargeLimitReached) {
538 if (charge.chargedCount === 0) {
539
540
541
542 this.chargeLimitReached = true;
543 this.chargeLimitDropped++;
544 return false;
545 }
546
547 this.chargeLimitReached = true;
548 }
549 return true;
550 } catch (error: any) {
551 this.pushChargeFailures++;
552 log.warning(`Charged push rejected (${error?.message}); retrying without charge`);
553 try {
554 await Actor.pushData(product);
555 return true;
556 } catch (e2: any) {
557 this.pushDropped++;
558 log.error(`Plain push also failed (${e2?.message}) — record dropped`);
559 return false;
560 }
561 }
562 }
563
564
565
566
567
568 private async reportOutcome(): Promise<void> {
569 const requested = this.input.maxItems;
570 const filtersActive = this.filterActive(this.input.minRating)
571 || this.filterActive(this.input.minReviews)
572 || this.filterActive(this.input.priceMin)
573 || this.filterActive(this.input.priceMax);
574
575 let outcome: RunOutcome;
576 let message: string;
577 let fail = false;
578
579 if (this.pagesVisited === 0) {
580 outcome = 'blocked';
581 message = `Could not load any Etsy results page for ${this.target.label}. `
582 + `Last error: ${this.lastFailure ?? 'unknown'}. Nothing was extracted; `
583 + `re-run to get a different residential IP.`;
584 fail = true;
585 } else if (this.listingsSeen === 0 && this.noResultsSignal) {
586
587 outcome = 'no-results';
588 message = `Etsy returned no listings for ${this.target.label}. The page explicitly reported `
589 + `no matches, so there is nothing to scrape — try a broader search term.`;
590 } else if (this.listingsSeen === 0 && this.lastFailure) {
591
592
593 outcome = 'blocked';
594 message = `Etsy served no listing grid for ${this.target.label} on any attempt — the requests were `
595 + `blocked. Last error: ${this.lastFailure}. Nothing was extracted; re-run to get a `
596 + `different residential IP.`;
597 fail = true;
598 } else if (this.listingsSeen === 0) {
599 outcome = 'extraction-failed';
600 message = `Loaded a page for ${this.target.label} but found no listing cards and no "no results" `
601 + `message from Etsy. That means the page was blocked or Etsy's structure changed — `
602 + `the run returned nothing rather than reporting a false success.`;
603 fail = true;
604 } else if (this.uniqueExtracted === 0) {
605 outcome = 'extraction-failed';
606 message = `Found ${this.listingsSeen} listing cards for ${this.target.label} but could not parse `
607 + `a single one into a product record. Etsy's card markup has likely changed.`;
608 fail = true;
609 } else if (this.itemCount === 0 && this.chargeLimitReached) {
610
611
612 outcome = 'charge-limit-reached';
613 message = `Found ${this.listingsSeen} listings for ${this.target.label} but saved none: this run's `
614 + `maximum total charge was reached before the first product could be billed. Raise the run's `
615 + `"Max total charge" limit (or your account budget) and re-run.`;
616 fail = true;
617 } else if (this.itemCount === 0 && !filtersActive) {
618 outcome = 'extraction-failed';
619 message = `Parsed ${this.uniqueExtracted} listings for ${this.target.label} but none of them could `
620 + `be saved to the dataset (${this.pushDropped} rejected on write). No filters were set, so `
621 + `this is a storage or schema fault, not your input.`;
622 fail = true;
623 } else if (this.itemCount === 0 && this.droppedUnfilterable > 0 && this.droppedByFilter === 0) {
624 outcome = 'filters-unusable';
625 message = `Found ${this.listingsSeen} listings for ${this.target.label}, but the value your `
626 + `filter needs (rating / reviews / price) could not be extracted for any of them, so the `
627 + `filter could not be applied. Returning nothing rather than guessing. `
628 + `Re-run without the filter to get the raw listings.`;
629 fail = true;
630 } else if (this.itemCount === 0) {
631 outcome = 'zero-after-filter';
632 message = `Found ${this.listingsSeen} listings for ${this.target.label}, but none matched your `
633 + `filters (${this.describeFilters()}). ${this.droppedByFilter} were checked and out of `
634 + `range, ${this.droppedUnfilterable} could not be checked because the value was not `
635 + `published on the card. Loosen the filters or re-run without them.`;
636 } else if (
637 this.itemCount < requested
638 && (this.stopReason === 'blocked' || this.stopReason === 'nav-timeout'
639 || this.stopReason === 'max-pages' || this.stopReason === 'charge-limit'
640 || this.stopReason === 'time-budget')
641 ) {
642 outcome = 'partial';
643 message = `Partial result: ${this.itemCount} of the ${requested} products you requested. `
644 + `Stopped because ${this.describeStop()}.`;
645 } else {
646 outcome = 'ok';
647 message = `Scraped ${this.itemCount} products from ${this.target.label}.`;
648 }
649
650 const output: RunOutput = {
651 outcome,
652 message,
653 target: this.target.label,
654 itemsSaved: this.itemCount,
655 itemsRequested: requested,
656 listingsSeen: this.listingsSeen,
657 droppedByFilter: this.droppedByFilter,
658 droppedUnfilterable: this.droppedUnfilterable,
659 pagesVisited: this.pagesVisited,
660 ratingsMissing: this.ratingsMissing,
661 reviewCountsMissing: this.reviewCountsMissing,
662 chargeLimitReached: this.chargeLimitReached,
663 finishedAt: new Date().toISOString(),
664 };
665 await Actor.setValue('OUTPUT', output).catch((e: any) => {
666 log.warning(`Could not write OUTPUT record: ${e?.message}`);
667 });
668
669 if (this.uniqueExtracted > 0 && (this.ratingsMissing > 0 || this.reviewCountsMissing > 0)) {
670 log.warning(
671 `Of ${this.uniqueExtracted} listings, ${this.ratingsMissing} exposed no rating and `
672 + `${this.reviewCountsMissing} exposed no review count. Those fields are null in the dataset. `
673 + `Null means "not published on the card", not "zero".`,
674 );
675 }
676 if (this.pushChargeFailures > 0) {
677 log.warning(
678 `${this.pushChargeFailures} records failed dataset-schema validation on the charged push `
679 + `and were saved uncharged. This is a schema mismatch, please report it.`,
680 );
681 }
682 if (this.pushDropped > 0) {
683 log.error(`${this.pushDropped} records could not be saved at all and are missing from the dataset.`);
684 }
685 if (this.chargeLimitDropped > 0) {
686 log.warning(
687 `${this.chargeLimitDropped} extracted listing(s) were discarded unsaved because this run's `
688 + `maximum total charge was reached. They are NOT in the dataset and you were NOT billed `
689 + `for them. Raise the run's charge limit to collect the rest.`,
690 );
691 }
692
693 if (fail) {
694 log.error(message);
695 await Actor.fail(message);
696 return;
697 }
698
699 if (outcome === 'zero-after-filter') {
700 log.warning(message);
701 } else {
702 log.info(message);
703 }
704 log.info(`Done. ${this.itemCount} products in the dataset.`);
705
706 if (this.itemCount > 0) {
707 log.info('----------------------------------------');
708 log.info('Useful? A 30-second review helps others find this actor');
709 log.info('and helps us keep it maintained:');
710 log.info('https://apify.com/webdatalabs/etsy-scraper-pro');
711 log.info('----------------------------------------');
712 }
713 }
714
715 private describeFilters(): string {
716 const parts: string[] = [];
717 if (this.filterActive(this.input.minRating)) parts.push(`minRating=${this.input.minRating}`);
718 if (this.filterActive(this.input.minReviews)) parts.push(`minReviews=${this.input.minReviews}`);
719 if (this.filterActive(this.input.priceMin)) parts.push(`priceMin=${this.input.priceMin}`);
720 if (this.filterActive(this.input.priceMax)) parts.push(`priceMax=${this.input.priceMax}`);
721 return parts.join(', ');
722 }
723
724 private describeStop(): string {
725 switch (this.stopReason) {
726 case 'blocked': return 'Etsy blocked a later results page';
727 case 'nav-timeout': return 'a results page stopped responding';
728 case 'max-pages': return `the ${MAX_PAGES}-page safety cap was reached`;
729 case 'time-budget': return 'the run reached its per-session time budget';
730 case 'charge-limit': return "this run's maximum total charge was reached";
731 default: return 'the results ran out';
732 }
733 }
734}
735
736
737
738
739
740
741
742
743function resolveTarget(input: ValidatedInput): Target {
744 const q = (input.query || '').trim();
745
746 if (q) {
747 if (input.searchUrl) {
748 try {
749 const fromUrl = classifyEtsyUrl(input.searchUrl);
750 const same = fromUrl.kind === 'query' && fromUrl.query.toLowerCase() === q.toLowerCase();
751 if (!same) {
752 log.warning(
753 `Both "query" and "searchUrl" were provided and they disagree. Using query "${q}" `
754 + `and ignoring searchUrl (${input.searchUrl}). Clear "query" to use the URL instead.`,
755 );
756 }
757 } catch (e: any) {
758 log.warning(
759 `Ignoring searchUrl and using query "${q}" instead. searchUrl problem: `
760 + `${String(e?.message).split('\n')[0]}`,
761 );
762 }
763 }
764 return { kind: 'query', query: q, label: `search "${q}"` };
765 }
766
767 if (input.searchUrl) return classifyEtsyUrl(input.searchUrl);
768
769
770 log.info('No search target provided — defaulting to query "handmade jewelry"');
771 return { kind: 'query', query: 'handmade jewelry', label: 'search "handmade jewelry" (default)' };
772}
773
774async function failInvalidInput(message: string): Promise<void> {
775 log.error(message);
776 await Actor.setValue('OUTPUT', {
777 outcome: 'invalid-input' as RunOutcome,
778 message,
779 target: null,
780 itemsSaved: 0,
781 finishedAt: new Date().toISOString(),
782 }).catch(() => { });
783 await Actor.fail(message.split('\n')[0]);
784}
785
786Actor.main(async () => {
787 log.info('Starting Etsy Scraper Pro');
788
789 const rawInput = await Actor.getInput();
790
791 let input: ValidatedInput;
792 try {
793 input = InputSchema.parse(rawInput || {});
794 } catch (error: any) {
795 const issues = (error?.issues || [])
796 .map((i: any) => `${(i.path || []).join('.') || '(root)'}: ${i.message}`)
797 .join('; ');
798 await failInvalidInput(
799 `Invalid input: ${issues || error?.message}.\n`
800 + ` A correct input looks like:\n`
801 + ` { "query": "vintage watch", "maxItems": 48 }\n`
802 + ` or { "searchUrl": "https://www.etsy.com/search?q=vintage%20watch", "maxItems": 48 }`,
803 );
804 return;
805 }
806
807
808
809 let target: Target;
810 try {
811 target = resolveTarget(input);
812 } catch (error: any) {
813 if (error instanceof InputError) {
814 await failInvalidInput(error.message);
815 return;
816 }
817 throw error;
818 }
819
820 await new EtsyScraper(input, target).run();
821});
