1import { createHash } from 'node:crypto';
2import zlib from 'node:zlib';
3import { sendUsage } from './actor-telemetry.js';
4import { Actor } from 'apify';
5import { gotScraping } from 'got-scraping';
6import { request, ProxyAgent, interceptors as undiciInterceptors } from 'undici';
7import { CookieJar } from 'tough-cookie';
8import * as cheerio from 'cheerio';
9import c from 'ansi-colors';
10
11await Actor.init();
12
13
14
15const __reviveGuard = true;
16if (typeof Actor.isAtHome === 'function' && Actor.isAtHome() && typeof Actor.openDataset === 'function') {
17 try {
18 const __rgDs = await Actor.openDataset();
19 const __rgCount = (await __rgDs.getInfo())?.itemCount ?? 0;
20 if (__rgCount > 0) {
21 const { createHash } = await import('node:crypto');
22 const __rgVolatile = new Set(['observedAt', 'scrapedAt', 'queriedAt', 'capturedAt', 'fetchedAt',
23 'crawledAt', 'retrievedAt', 'collectedAt', 'extractedAt']);
24 const __rgNorm = (v) => {
25 if (Array.isArray(v)) return v.map(__rgNorm);
26 if (v && typeof v === 'object') {
27 const o = {};
28 for (const k of Object.keys(v).sort()) if (!__rgVolatile.has(k)) o[k] = __rgNorm(v[k]);
29 return o;
30 }
31 return v;
32 };
33 const __rgKey = (it) => createHash('sha1').update(JSON.stringify(__rgNorm(it))).digest('base64').slice(0, 22);
34 const __rgSeen = new Map();
35 const __rgMax = Math.min(__rgCount, 1000000);
36 for (let __off = 0; __off < __rgMax; __off += 10000) {
37 const { items } = await __rgDs.getData({ offset: __off, limit: 10000 });
38 if (!items || !items.length) break;
39 for (const it of items) { const k = __rgKey(it); __rgSeen.set(k, (__rgSeen.get(k) || 0) + 1); }
40 }
41 console.log(`note: this run was resumed (revive or platform migration). ${__rgCount} result(s) already delivered in this run will not be delivered or charged again.`);
42 let __rgSkipped = 0;
43 const __rgInner = Actor.pushData.bind(Actor);
44 Actor.pushData = async (data, eventName) => {
45 if (!__rgSeen.size) return __rgInner(data, eventName);
46 const isArr = Array.isArray(data);
47 const fresh = [];
48 let skipped = 0;
49 for (const it of (isArr ? data : [data])) {
50 let k = null;
51 try { k = __rgKey(it); } catch { }
52 const n = k ? __rgSeen.get(k) : 0;
53 if (n) { if (n === 1) __rgSeen.delete(k); else __rgSeen.set(k, n - 1); skipped++; } else fresh.push(it);
54 }
55 if (skipped) {
56 const before = __rgSkipped;
57 __rgSkipped += skipped;
58 if (before === 0 || Math.floor(before / 500) !== Math.floor(__rgSkipped / 500)) {
59 console.log(`note: skipped ${__rgSkipped} result(s) already delivered before the resume (not charged again).`);
60 }
61 }
62 const bonus = eventName ? skipped : 0;
63 if (!fresh.length) return { eventChargeLimitReached: false, chargedCount: bonus, chargeableWithinLimit: {} };
64 const cr = await __rgInner(isArr ? fresh : fresh[0], eventName);
65 if (bonus && cr && typeof cr.chargedCount === 'number') return { ...cr, chargedCount: cr.chargedCount + bonus };
66 return cr;
67 };
68 }
69 } catch (e) {
70 console.log('note: resume check skipped (' + String(e && e.message || e).slice(0, 80) + ')');
71 }
72}
73
74
75const __uipRaw = (typeof Actor !== 'undefined' && Actor.getEnv) ? Actor.getEnv()?.userIsPaying : undefined;
76const __isPayingUser = __uipRaw === '1' || __uipRaw === 1 || __uipRaw === true || process.env.APIFY_USER_IS_PAYING === '1';
77
78
79
80{
81 const __origPushData = Actor.pushData.bind(Actor);
82 let __chargeLimitHit = false;
83 Actor.pushData = async (data, eventName) => {
84 if (eventName && __chargeLimitHit) return { eventChargeLimitReached: true, chargedCount: 0, chargeableWithinLimit: {} };
85 const __cr = await __origPushData(data, eventName);
86 if (eventName && __cr && __cr.eventChargeLimitReached) __chargeLimitHit = true;
87 return __cr;
88 };
89
90
91 const __origCharge = Actor.charge.bind(Actor);
92 Actor.charge = async (spec) => {
93 if (__chargeLimitHit) return { eventChargeLimitReached: true, chargedCount: 0, chargeableWithinLimit: {} };
94 const __cr = await __origCharge(spec);
95 if (__cr && __cr.eventChargeLimitReached) __chargeLimitHit = true;
96 return __cr;
97 };
98}
99
100const input = (await Actor.getInput()) ?? {};
101sendUsage('mercadolibre-scraper', '0.1', input);
102
103
104
105
106try { await Actor.charge({ eventName: 'apify-actor-start' }); } catch { }
107
108const pct = (n) => c.dim(String(Math.round(n)).padStart(3) + '%') + ' ';
109
110
111const setStatus = (msg) => { Actor.setStatusMessage(msg).catch(() => {}); };
112
113async function mapPool(items, limit, fn) {
114 const out = new Array(items.length);
115 let i = 0;
116 await Promise.all(
117 Array.from({ length: Math.max(1, Math.min(limit, items.length)) }, async () => {
118 while (i < items.length) { const idx = i++; out[idx] = await fn(items[idx], idx); }
119 }),
120 );
121 return out;
122}
123
124console.log(pct(0) + c.cyan('scrapers_lat') + c.dim(' / mercadolibre-scraper'));
125console.log(c.dim(' ' + '─'.repeat(34)));
126const SECRET_INPUT_KEYS = new Set(['proxyUrl', 'proxySessionCookie']);
127for (const [k, v] of Object.entries(input)) if (v != null && v !== '') {
128 if (SECRET_INPUT_KEYS.has(k)) { console.log(c.dim(` ${k}:`) + ' ' + c.white('(set)')); continue; }
129 console.log(c.dim(` ${k}:`) + ' ' + c.white(typeof v === 'object' ? JSON.stringify(v) : String(v)));
130}
131console.log();
132
133let effectiveMaxItems = input.maxListings ?? input.maxItems ?? 10;
134const isPaying = __isPayingUser;
135
136
137
138const FREE_PLAN_CAP = 5;
139if (!isPaying && effectiveMaxItems > FREE_PLAN_CAP) { effectiveMaxItems = FREE_PLAN_CAP; console.log('note: free Apify plans are capped at ' + FREE_PLAN_CAP + ' products (listing-only) per run — upgrade for full product details and higher volume'); }
140const withDetails = isPaying ? (input.withDetails ?? false) : false;
141
142
143
144
145
146const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY;
147const wantAiSummary = isPaying && input.aiSummary === true;
148if (input.aiSummary === true && !isPaying) {
149 console.log(c.dim('note: the AI summary add-on requires a paid Apify plan — disabled for this run'));
150}
151async function aiJson(system, user, { maxChars = 4000, maxTokens = 300 } = {}) {
152 if (!OPENROUTER_API_KEY) return null;
153 try {
154 const res = await fetch('https://openrouter.ai/api/v1/chat/completions', {
155 method: 'POST',
156 headers: { authorization: `Bearer ${OPENROUTER_API_KEY}`, 'content-type': 'application/json' },
157 body: JSON.stringify({
158 model: 'openai/gpt-4o-mini', max_tokens: maxTokens, temperature: 0,
159 response_format: { type: 'json_object' },
160 messages: [{ role: 'system', content: system }, { role: 'user', content: String(user).slice(0, maxChars) }],
161 }),
162 signal: AbortSignal.timeout(30000),
163 });
164 if (!res.ok) return null;
165 const j = await res.json();
166 const t = j?.choices?.[0]?.message?.content;
167 if (!t) return null;
168 try { return JSON.parse(t); } catch { return { summary: t }; }
169 } catch { return null; }
170}
171
172async function aiEnrich(rec) {
173 if (!wantAiSummary) return rec;
174 const attrText = Array.isArray(rec.attributes) ? rec.attributes.map((a) => `${a.name}:${a.value}`).join(', ') : '';
175 const base = [rec.title, rec.condition, rec.price != null ? `${rec.price} ${rec.currency || ''}` : '', rec.sellerName, attrText, rec.description]
176 .filter(Boolean).join(' | ').trim();
177 if (!base) return rec;
178 const s = await aiJson(
179 'You are an e-commerce analyst. From the product info, return JSON {"summary":"1-2 sentence neutral buyer-oriented summary","pros":["short selling point", ...max 4]}.',
180 base);
181 if (s && (s.summary || (Array.isArray(s.pros) && s.pros.length))) {
182 rec.aiSummary = s.summary ? String(s.summary) : null;
183 rec.aiPros = Array.isArray(s.pros) ? s.pros.map((p) => String(p)).slice(0, 4) : null;
184 try { await Actor.charge({ eventName: 'ai_summary' }); } catch { }
185 }
186 return rec;
187}
188
189
190const DOMAINS = {
191 ar: 'listado.mercadolibre.com.ar', mx: 'listado.mercadolibre.com.mx',
192 br: 'lista.mercadolivre.com.br', co: 'listado.mercadolibre.com.co',
193 cl: 'listado.mercadolibre.cl', pe: 'listado.mercadolibre.com.pe',
194 uy: 'listado.mercadolibre.com.uy', ve: 'listado.mercadolibre.com.ve',
195 bo: 'listado.mercadolibre.com.bo', py: 'listado.mercadolibre.com.py',
196 ec: 'listado.mercadolibre.com.ec', cr: 'listado.mercadolibre.co.cr',
197 pa: 'listado.mercadolibre.com.pa', do: 'listado.mercadolibre.com.do',
198 gt: 'listado.mercadolibre.com.gt', hn: 'listado.mercadolibre.com.hn',
199 ni: 'listado.mercadolibre.com.ni', sv: 'listado.mercadolibre.com.sv',
200};
201const CC_BY_TLD = [
202 ['com.ar', 'AR'], ['com.mx', 'MX'], ['com.br', 'BR'], ['com.co', 'CO'],
203 ['com.pe', 'PE'], ['com.uy', 'UY'], ['com.ve', 'VE'], ['com.bo', 'BO'],
204 ['com.py', 'PY'], ['com.ec', 'EC'], ['co.cr', 'CR'], ['com.pa', 'PA'],
205 ['com.do', 'DO'], ['com.gt', 'GT'], ['com.hn', 'HN'], ['com.ni', 'NI'],
206 ['com.sv', 'SV'], ['.cl', 'CL'],
207];
208
209function countryFromUrl(u) {
210 const host = (() => { try { return new URL(u).hostname; } catch { return ''; } })();
211 for (const [tld, cc] of CC_BY_TLD) if (host.endsWith(tld)) return cc;
212 return 'AR';
213}
214
215
216
217
218
219
220const ML_CONDITION_IDS = { new: '2230284', used: '2230581', refurbished: '2230582' };
221
222
223const ML_SORT_SEG = { price_asc: '_OrderId_PRICE', price_desc: '_OrderId_PRICE*DESC', newest: '_OrderId_BEGINS*DESC', relevance: '' };
224function buildFilterSegments(inp) {
225 let seg = '';
226 if (inp.condition && ML_CONDITION_IDS[inp.condition]) seg += `_ITEM*CONDITION_${ML_CONDITION_IDS[inp.condition]}`;
227 const pmin = inp.priceMin, pmax = inp.priceMax;
228 if ((pmin != null && pmin !== '') || (pmax != null && pmax !== '')) {
229 seg += `_PriceRange_${pmin != null && pmin !== '' ? pmin : 0}-${pmax != null && pmax !== '' ? pmax : 0}`;
230 }
231 if (inp.minDiscount) seg += `_Discount_${inp.minDiscount}-100`;
232 if (inp.freeShipping) seg += '_shipping_free';
233 if (inp.fullFulfillment) seg += '_shipping_fulfillment';
234 if (inp.officialStore) seg += '_Tienda_all';
235 const sid = String(inp.sellerId || '').replace(/[^0-9]/g, '');
236 if (sid) seg += `_CustId_${sid}`;
237 if (inp.sort && ML_SORT_SEG[inp.sort]) seg += ML_SORT_SEG[inp.sort];
238 return seg;
239}
240const CC = (input.country || 'ar');
241const DOM = DOMAINS[CC] || DOMAINS.ar;
242const sellerIdInput = String(input.sellerId || '').replace(/[^0-9]/g, '');
243const categoryInput = String(input.category || '').trim();
244function slugify(term) {
245 return term ? term.toLowerCase().replace(/\s+/g, '-').replace(/[^a-z0-9\-]/g, '') : 'ofertas';
246}
247
248function toList(v) {
249 if (Array.isArray(v)) return v.map((x) => String(x ?? '').trim()).filter(Boolean);
250 if (v == null || v === '') return [];
251 return [String(v).trim()].filter(Boolean);
252}
253
254
255
256
257const isPdpUrl = (u) => /\/p\/ML[A-Z]?\d+/i.test(u) || /articulo\.mercadoli/i.test(u) || /\/ML[A-Z]-?\d{6,}/i.test(u);
258const jobs = [];
259const directPdpUrls = [];
260for (const u of [...toList(input.startUrls), ...toList(input.startUrl)]) {
261 if (!/^https?:\/\//i.test(u)) continue;
262 if (isPdpUrl(u)) directPdpUrls.push(u.split('#')[0]);
263 else jobs.push({ startUrl: u, searchTerm: null, searchTermMode: false });
264}
265for (const term of [...toList(input.searchTerms), ...toList(input.searchTerm)]) {
266 let base = `https://${DOM}/${categoryInput ? '' : ''}${encodeURIComponent(slugify(term))}`;
267 if (sellerIdInput) base += `_CustId_${sellerIdInput}`;
268 jobs.push({ startUrl: base, searchTerm: term, searchTermMode: true });
269}
270
271if (!jobs.length && sellerIdInput && !directPdpUrls.length) {
272 jobs.push({ startUrl: `https://${DOM}/_CustId_${sellerIdInput}`, searchTerm: null, searchTermMode: false });
273}
274
275if (!jobs.length && !directPdpUrls.length && !input.bestSellers && !sellerIdInput) {
276 jobs.push({ startUrl: `https://${DOM}/${slugify('')}`, searchTerm: null, searchTermMode: true });
277}
278
279
280
281
282
283const proxyCountry = countryFromUrl(jobs[0]?.startUrl || directPdpUrls[0] || `https://${DOM}/`);
284
285
286
287
288
289
290
291
292
293
294
295const customProxyUrl = String(input.proxyUrl || process.env.ML_PROXY_URL || '').trim();
296const usingCustomProxy = !!customProxyUrl;
297
298
299
300const sessionCookieRaw = String(input.proxySessionCookie || process.env.ML_SESSION_COOKIE || '').trim();
301const proxyConfiguration = usingCustomProxy
302 ? await Actor.createProxyConfiguration({ proxyUrls: [customProxyUrl] })
303 : await Actor.createProxyConfiguration({ groups: ['RESIDENTIAL'], countryCode: proxyCountry });
304if (usingCustomProxy) console.log(c.dim(' using a custom proxy pool (advanced) — full retry logic enabled'));
305
306
307
308
309
310
311
312
313
314let unblockerProxy = null;
315if (isPaying) {
316 try { unblockerProxy = await Actor.createProxyConfiguration({ groups: ['UNBLOCKER'], countryCode: proxyCountry }); }
317 catch (e) { console.log(c.yellow(' UNBLOCKER fallback unavailable: ' + String(e?.message || e).slice(0, 120))); }
318}
319const allowUnblocker = isPaying && !!unblockerProxy;
320
321
322const servedVia = {};
323const recordVia = (v) => { servedVia[v] = (servedVia[v] || 0) + 1; };
324let unblockerEngaged = false;
325
326
327
328
329
330let unblockerWalled = false;
331
332
333
334
335const RUN_SALT = String(Actor.getEnv()?.actorRunId || Math.random().toString(36).slice(2)).slice(-8);
336
337
338
339
340const CH_VER = '124';
341const BROWSER_HEADERS = {
342 'User-Agent': `Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/${CH_VER}.0.0.0 Safari/537.36`,
343 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
344 'Accept-Language': 'es-419,es;q=0.9,en;q=0.8',
345 'Accept-Encoding': 'gzip, deflate, br',
346 'Cache-Control': 'max-age=0',
347 'Upgrade-Insecure-Requests': '1',
348 'sec-ch-ua': `"Chromium";v="${CH_VER}", "Google Chrome";v="${CH_VER}", "Not-A.Brand";v="99"`,
349 'sec-ch-ua-mobile': '?0',
350 'sec-ch-ua-platform': '"Windows"',
351 'Sec-Fetch-Dest': 'document',
352 'Sec-Fetch-Mode': 'navigate',
353 'Sec-Fetch-Site': 'none',
354 'Sec-Fetch-User': '?1',
355};
356
357
358
359
360
361const PDP_CRAWLER_UA = 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko); compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot';
362
363
364
365const AR_BESTSELLER_CATEGORIES = {
366 MLA1051: 'Celulares y Teléfonos', MLA1648: 'Computación', MLA1000: 'Electrónica, Audio y Video',
367 MLA5726: 'Electrodomésticos y Aires Ac.', MLA1430: 'Ropa y Accesorios', MLA5725: 'Accesorios para Vehículos',
368 MLA1276: 'Deportes y Fitness', MLA1574: 'Hogar, Muebles y Jardín', MLA1132: 'Juegos y Juguetes',
369 MLA1246: 'Belleza y Cuidado Personal', MLA1071: 'Animales y Mascotas', MLA1499: 'Industrias y Oficinas',
370 MLA1384: 'Bebés', MLA1039: 'Cámaras y Accesorios', MLA1367: 'Antigüedades', MLA1368: 'Arte, Librería y Mercería',
371 MLA1743: 'Autos, Motos y Otros', MLA1500: 'Construcción', MLA1168: 'Música, Películas y Series',
372 MLA3937: 'Joyas y Relojes', MLA1182: 'Instrumentos Musicales', MLA1196: 'Libros, Revistas y Comics',
373 MLA409431: 'Salud y Equipamiento Médico', MLA1512: 'Agro', MLA1540: 'Servicios', MLA1953: 'Más Categorías',
374};
375
376
377function isChallenge(body) {
378 return /verifyChallenge|_bmstate|_bm_skipml|snoopy-generation-web/.test(body || '');
379}
380
381
382
383
384
385
386function isWall(body, url) {
387 return /account-verification|Hubo un error accediendo|suspicious-traffic-frontend|negative_traffic|micro-landing-container/i.test(body || '')
388 || /\/gz\/account-verification/.test(url || '');
389}
390
391
392function solvePow(nonce, difficulty) {
393 const prefix = '0'.repeat(difficulty);
394 if (difficulty <= 0) return '0';
395 for (let i = 0; i < 50_000_000; i++) {
396 const h = createHash('sha256').update(nonce + i).digest('hex');
397 if (h.startsWith(prefix)) return String(i);
398 }
399 return null;
400}
401
402function topLevelDomain(host) {
403
404 const parts = host.split('.');
405 if (parts.length <= 2) return host;
406
407 const twoTld = ['ar', 'mx', 'br', 'co', 'pe', 'uy', 've', 'bo', 'py', 'ec', 'cr', 'pa', 'do', 'gt', 'hn', 'ni', 'sv'];
408 return twoTld.includes(parts[parts.length - 1]) && parts.length >= 3
409 ? parts.slice(-3).join('.')
410 : parts.slice(-2).join('.');
411}
412
413let sessionSeq = 0;
414
415
416
417const MAX_FETCH_ATTEMPTS = usingCustomProxy ? 8 : 2;
418
419
420
421
422
423
424async function fetchViaProxy(url, referer) {
425 let lastErr;
426 for (let attempt = 0; attempt < MAX_FETCH_ATTEMPTS; attempt++) {
427
428
429
430 const proxyUrl = proxyConfiguration
431 ? await proxyConfiguration.newUrl(`ml_${proxyCountry}_${RUN_SALT}_${sessionSeq}`)
432 : undefined;
433 const jar = new CookieJar();
434 const host = (() => { try { return new URL(url).hostname; } catch { return ''; } })();
435 const tld = topLevelDomain(host);
436
437 if (sessionCookieRaw) {
438 for (const part of sessionCookieRaw.split(';')) {
439 const kv = part.trim();
440 if (kv.includes('=')) { try { await jar.setCookie(`${kv}; Domain=.${tld}; Path=/`, `https://${host}/`); } catch { } }
441 }
442 }
443 const reqOpts = {
444 proxyUrl,
445 cookieJar: jar,
446 headerGeneratorOptions: { browsers: ['chrome'], operatingSystems: ['windows'], locales: ['es-419', 'es'] },
447 timeout: { request: 35000 },
448 followRedirect: true,
449 retry: { limit: 0 },
450 throwHttpErrors: false,
451 };
452
453
454
455 const navHeaders = referer
456 ? { ...BROWSER_HEADERS, 'Sec-Fetch-Site': 'same-site', Referer: referer }
457 : BROWSER_HEADERS;
458 try {
459
460
461
462 if (!referer && /^(listado|lista)\./.test(host)) {
463 try {
464 const homeHost = host.replace(/^listado\.|^lista\./, 'www.');
465 await gotScraping({ url: `https://${homeHost}/`, ...reqOpts, headers: { ...BROWSER_HEADERS, 'Sec-Fetch-Site': 'none' } });
466 } catch { }
467 }
468
469 let res = await gotScraping({ url, ...reqOpts, headers: navHeaders });
470 let body = res.body || '';
471
472
473 if (isWall(body, res.url)) {
474 lastErr = new Error('MercadoLibre suspicious-traffic wall');
475 await new Promise((r) => setTimeout(r, 300 + attempt * 350));
476 sessionSeq++;
477 continue;
478 }
479
480
481
482 const dom = '.' + tld;
483 for (let solve = 0; solve < 5 && isChallenge(body); solve++) {
484 const cookies = await jar.getCookies(res.url || url);
485 const bmCookie = cookies.filter((ck) => ck.key === '_bmstate').pop();
486 const bm = bmCookie ? bmCookie.value : null;
487 if (!bm) break;
488 const [nonce, diffRaw] = decodeURIComponent(bm).split(';');
489 const difficulty = Number(diffRaw) || 0;
490 const answer = solvePow(nonce, difficulty);
491 if (answer == null) break;
492
493
494 const replayUrl = res.url || url;
495 await jar.setCookie(`_bmc=${encodeURIComponent(`${nonce};${answer}`)}; Domain=${dom}; Path=/`, replayUrl);
496 await jar.setCookie(`_bm_skipml=true; Domain=${dom}; Path=/`, replayUrl);
497 res = await gotScraping({
498 url: replayUrl, ...reqOpts,
499 headers: { ...BROWSER_HEADERS, 'Sec-Fetch-Site': 'same-site', Referer: replayUrl },
500 });
501 body = res.body || '';
502 }
503
504 if (res.statusCode === 200 && !isChallenge(body) && !isWall(body, res.url)) {
505 return body;
506 }
507 if (isWall(body, res.url)) lastErr = new Error('MercadoLibre suspicious-traffic wall');
508 else lastErr = new Error(`status ${res.statusCode}${isChallenge(body) ? ' challenge' : ''}`);
509 } catch (e) {
510 lastErr = e;
511 }
512 await new Promise((r) => setTimeout(r, 300 + attempt * 350));
513 sessionSeq++;
514 }
515 throw lastErr || new Error('fetch failed');
516}
517
518
519
520
521
522
523
524async function fetchViaUnblocker(url, referer, opts = {}) {
525 const UNB_TRIES = opts.tries || 3;
526 let lastErr;
527 const navHeaders = {
528 ...BROWSER_HEADERS,
529 'Accept-Encoding': 'gzip, deflate, br',
530 ...(referer ? { 'Sec-Fetch-Site': 'same-site', Referer: referer } : {}),
531 ...(opts.headers || {}),
532 };
533
534
535 if (opts.ua) {
536 navHeaders['User-Agent'] = opts.ua;
537 delete navHeaders['sec-ch-ua']; delete navHeaders['sec-ch-ua-mobile']; delete navHeaders['sec-ch-ua-platform'];
538 }
539 if (opts.body != null && !navHeaders['Content-Type']) navHeaders['Content-Type'] = 'application/json';
540
541 if (sessionCookieRaw) navHeaders.Cookie = sessionCookieRaw;
542 for (let i = 0; i < UNB_TRIES; i++) {
543 sessionSeq++;
544 let agent;
545 try {
546 const uri = await unblockerProxy.newUrl(`mlu_${proxyCountry}_${RUN_SALT}_${sessionSeq}`);
547
548
549 agent = new ProxyAgent({ uri, requestTls: { rejectUnauthorized: false } })
550 .compose(undiciInterceptors.redirect({ maxRedirections: 5 }));
551 const res = await request(url, {
552 method: opts.method || 'GET', headers: navHeaders, dispatcher: agent,
553 ...(opts.body != null ? { body: opts.body } : {}),
554 headersTimeout: opts.timeout || 90000, bodyTimeout: opts.timeout || 90000,
555 });
556 const buf = Buffer.from(await res.body.arrayBuffer());
557 const enc = String(res.headers['content-encoding'] || '').toLowerCase();
558 let body;
559 try {
560 if (enc.includes('br')) body = zlib.brotliDecompressSync(buf).toString('utf8');
561 else if (enc.includes('gzip')) body = zlib.gunzipSync(buf).toString('utf8');
562 else if (enc.includes('deflate')) body = zlib.inflateSync(buf).toString('utf8');
563 else body = buf.toString('utf8');
564 } catch { body = buf.toString('utf8'); }
565 try { await agent.close(); } catch { }
566 agent = null;
567 if (res.statusCode === 200 && !isChallenge(body) && !isWall(body, url)) return body;
568 lastErr = isWall(body, url)
569 ? new Error('MercadoLibre suspicious-traffic wall')
570 : new Error(`unblocker status ${res.statusCode}${isChallenge(body) ? ' challenge' : ''}`);
571 } catch (e) {
572 lastErr = e;
573 } finally {
574 if (agent) { try { await agent.close(); } catch { } }
575 }
576 await new Promise((r) => setTimeout(r, 400 + i * 400));
577 }
578 throw lastErr || new Error('unblocker fetch failed');
579}
580
581
582
583async function fetchHtml(url, referer) {
584 let primaryErr;
585 try {
586 const body = await fetchViaProxy(url, referer);
587 recordVia(usingCustomProxy ? 'custom-proxy' : 'residential');
588 return body;
589 } catch (e) { primaryErr = e; }
590 if (allowUnblocker && !unblockerWalled) {
591 unblockerEngaged = true;
592 try {
593 const body = await fetchViaUnblocker(url, referer);
594 recordVia('unblocker');
595 return body;
596 } catch (e) {
597 primaryErr = e;
598
599
600 if (/suspicious-traffic wall/i.test(String(e?.message || e))) unblockerWalled = true;
601 }
602 }
603 throw primaryErr || new Error('fetch failed');
604}
605
606
607
608
609
610async function fetchPdpHtml(url, referer) {
611 if (allowUnblocker && !unblockerWalled) {
612 unblockerEngaged = true;
613 try {
614 const body = await fetchViaUnblocker(url, referer, { ua: PDP_CRAWLER_UA, tries: 2, timeout: 115000 });
615 recordVia('unblocker-pdp');
616 return body;
617 } catch (e) {
618 if (/suspicious-traffic wall/i.test(String(e?.message || e))) unblockerWalled = true;
619 throw e;
620 }
621 }
622 return fetchHtml(url, referer);
623}
624
625
626const CURRENCY_SYMBOL = {
627 ARS: '$', MXN: '$', BRL: 'R$', CLP: '$', COP: '$', PEN: 'S/', UYU: '$U',
628 VES: 'Bs', BOB: 'Bs', PYG: '₲', GTQ: 'Q', HNL: 'L', NIO: 'C$', DOP: 'RD$',
629 CRC: '₡', PAB: 'B/.', USD: 'US$',
630};
631
632function num(x) {
633 if (x == null) return null;
634 if (typeof x === 'number') return Number.isFinite(x) ? x : null;
635 const n = Number(String(x).replace(/[^\d.,-]/g, '').replace(/\.(?=\d{3}\b)/g, '').replace(',', '.'));
636 return Number.isFinite(n) ? n : null;
637}
638
639
640function cleanText(s) {
641 if (!s || typeof s !== 'string') return null;
642 const out = s.replace(/\{[a-z0-9_]+\}/gi, '').replace(/\s+/g, ' ').trim();
643 return out || null;
644}
645
646
647function extractPolycards(html) {
648 const cards = [];
649 const marker = '"polycard":{';
650 let idx = html.indexOf(marker);
651 while (idx !== -1) {
652 const start = idx + '"polycard":'.length;
653 let depth = 0;
654 let end = -1;
655 for (let i = start; i < html.length; i++) {
656 const ch = html[i];
657 if (ch === '{') depth++;
658 else if (ch === '}') {
659 depth--;
660 if (depth === 0) { end = i + 1; break; }
661 } else if (ch === '"') {
662
663 i++;
664 while (i < html.length && html[i] !== '"') {
665 if (html[i] === '\\') i++;
666 i++;
667 }
668 }
669 }
670 if (end !== -1) {
671 try { cards.push(JSON.parse(html.slice(start, end))); } catch { }
672 }
673 idx = html.indexOf(marker, end !== -1 ? end : idx + marker.length);
674 }
675 return cards;
676}
677
678
679
680function permalinkFromId(id, host) {
681 if (!id) return null;
682 const base = host.replace(/^listado\.|^lista\./, '');
683 const m = id.match(/^([A-Z]+?)(\d+)$/);
684 if (!m) return `https://${base}/p/${id}`;
685 return `https://articulo.${base}/${m[1]}-${m[2]}`;
686}
687
688function isTrackerUrl(u) {
689 return !u || /click\d*\.|\/mclics\/|\/clicks\/external\//.test(u);
690}
691
692
693function normalizeCondition(raw) {
694 if (!raw) return null;
695 const s = String(raw).toLowerCase();
696 if (/nuevo|new|novo/.test(s)) return 'new';
697 if (/reacond|refurb|recond/.test(s)) return 'refurbished';
698 if (/usado|used|usada/.test(s)) return 'used';
699 return s.trim() || null;
700}
701
702
703function mapCard(card, host, domToUrl) {
704 const meta = card.metadata || {};
705 const tracks = meta.tracks || {};
706 const comps = Array.isArray(card.components) ? card.components : [];
707 const comp = (t) => comps.find((c2) => c2 && c2.type === t) || null;
708
709 const id = meta.id || null;
710
711
712 const picId = card.pictures?.pictures?.[0]?.id || null;
713 const imageUrl = picId ? `https://http2.mlstatic.com/D_NQ_NP_2X_${picId}-F.webp` : null;
714
715 const titleC = comp('title');
716 const title = cleanText(titleC?.title?.text);
717
718 const priceC = comp('price');
719 const pc = priceC?.price || {};
720 const price = num(pc.current_price?.value);
721 const currency = pc.current_price?.currency || null;
722 const originalPrice = num(pc.previous_price?.value);
723 let discountPercentage = null;
724 const dl = pc.discount_label?.text;
725 if (dl) {
726 const dm = dl.match(/(\d+)\s*%/);
727 if (dm) discountPercentage = Number(dm[1]);
728 }
729 if (discountPercentage == null && price != null && originalPrice != null && originalPrice > price) {
730 discountPercentage = Math.round((1 - price / originalPrice) * 100);
731 }
732
733
734 let installments = null;
735 const inst = pc.installments;
736 if (inst) {
737 const amtV = (inst.values || []).find((v) => v.type === 'price');
738 const amount = num(amtV?.price?.value);
739 const qm = (inst.text || '').match(/(\d+)\s*(?:cuotas?|x)\b/i);
740 installments = {
741 quantity: qm ? Number(qm[1]) : null,
742 amount,
743 currency: amtV?.price?.currency || currency,
744 interestFree: inst.no_interest === true,
745 text: cleanText((inst.text || '').replace(/\{price\}/g, amount != null ? String(amount) : '')),
746 };
747 }
748
749 const sellerC = comp('seller');
750 const sellerName = cleanText(sellerC?.seller?.text);
751
752 const officialStore = !!sellerC && /icon_cockade|tienda oficial/i.test(JSON.stringify(sellerC));
753
754
755 let ratingAverage = null;
756 const revC = comp('review_compacted');
757 if (revC?.review_compacted?.values) {
758 const lab = revC.review_compacted.values.find((v) => v.type === 'label');
759 const r = num(lab?.label?.text);
760 if (r != null) ratingAverage = r;
761 }
762
763
764 const condition = normalizeCondition(comp('item_condition')?.item_condition?.text);
765
766
767 const shipC = comp('shipping_v2') || comp('shipping');
768 let shippingText = null;
769 let fulfillment = false;
770 const sv = shipC?.shipping_v2;
771 if (Array.isArray(sv)) {
772 for (const blk of sv) for (const v of (blk.values || [])) {
773 if (v.type === 'pill' && v.pill?.text && !shippingText) shippingText = cleanText(v.pill.text);
774 if (/full/i.test(v.key || '') || /full/i.test(v.icon?.key || '')) fulfillment = true;
775 }
776 }
777 const shipJson = shipC ? JSON.stringify(shipC).toLowerCase() : '';
778 const freeShipping = /grat[ií]s|free.?ship|llega gratis/.test(shipJson);
779 const fastShipping = shippingText ? /mañana|hoy|manha|hoje|today|tomorrow|día|dia/i.test(shippingText) : false;
780
781
782 const promoC = comp('promotions');
783 let promotions = null;
784 if (Array.isArray(promoC?.promotions)) {
785 promotions = promoC.promotions.map((p) => ({ type: p.type || null, text: cleanText(p.text) })).filter((p) => p.text);
786 if (!promotions.length) promotions = null;
787 }
788
789
790 const cbtC = comp('cbt');
791 let internationalItem = false;
792 let originCountry = null;
793 if (cbtC?.cbt) {
794 internationalItem = true;
795 const cv = (cbtC.cbt.values || []).find((v) => v.type === 'label');
796 originCountry = cleanText(cv?.label?.text);
797 }
798
799
800 let dealType = null;
801 for (const [k, v] of Object.entries(tracks)) if (/other_highlight_winner/.test(k)) dealType = v;
802 const dealEndsAt = card.float_highlight_generic?.countdown?.period_end || null;
803
804 const category = meta.domain_id || meta.category_id || null;
805
806 const domUrl = id ? domToUrl.get(id) : null;
807 const url = (domUrl && !isTrackerUrl(domUrl)) ? domUrl : permalinkFromId(id, host);
808
809
810 const cardStr = JSON.stringify(card);
811 const isSponsored = /"is_ad":\s*true|"advertising_id"|"pad_id"|"Patrocinado"|"Publicidad"|catamarketing/i.test(cardStr);
812
813 return {
814
815 imageUrl,
816 title,
817 price,
818 currency,
819 currencySymbol: currency ? (CURRENCY_SYMBOL[currency] || null) : null,
820 isSponsored,
821
822 searchQuery: null,
823 countryCode: null,
824 pageNumber: null,
825 totalResults: null,
826 originalPrice,
827 discountPercentage,
828 sellerName,
829 freeShipping,
830 category,
831 listingId: id,
832 url,
833
834 catalogProductId: meta.product_id || null,
835 categoryId: meta.category_id || null,
836 domainId: meta.domain_id || null,
837 position: meta.item_position != null ? Number(meta.item_position) : null,
838
839 installments,
840 promotions,
841 dealType,
842 dealEndsAt,
843
844 condition,
845 ratingAverage,
846 unitsSold: Number.isFinite(tracks.sold_quantity) ? tracks.sold_quantity : null,
847
848 officialStore,
849 shippingText,
850 fulfillment,
851 fastShipping,
852 internationalItem,
853 originCountry,
854 };
855}
856
857
858function domUrlMap(html) {
859 const $ = cheerio.load(html);
860 const map = new Map();
861 $('a.poly-component__title, a.ui-search-link, .ui-search-result__wrapper a[href]').each((_, a) => {
862 const href = $(a).attr('href');
863 if (!href || !/mercadoli|mercadoliv/.test(href) || isTrackerUrl(href)) return;
864 const m = href.match(/(ML[A-Z]\d+|MCO\d+|MPE\d+|MLU\d+|MLC\d+|MLM\d+|MLB\d+)/);
865 if (m && !map.has(m[1])) map.set(m[1], href.split('#')[0]);
866 });
867 return map;
868}
869
870function parseSearch(html, host) {
871 const cards = extractPolycards(html);
872 if (!cards.length) return [];
873 const domToUrl = domUrlMap(html);
874 const seen = new Set();
875 const out = [];
876 for (const card of cards) {
877 const item = mapCard(card, host, domToUrl);
878 if (!item.listingId || seen.has(item.listingId)) continue;
879 seen.add(item.listingId);
880 if (item.title || item.url) out.push(item);
881 }
882 return out;
883}
884
885
886function extractObjectAround(html, token) {
887 const ti = html.indexOf(token);
888 if (ti === -1) return null;
889 let start = ti;
890 let depth = 0;
891 for (let i = ti; i >= 0 && i > ti - 200000; i--) {
892 if (html[i] === '}') depth++;
893 else if (html[i] === '{') { if (depth === 0) { start = i; break; } depth--; }
894 }
895 depth = 0;
896 let end = -1;
897 for (let i = start; i < html.length && i < start + 200000; i++) {
898 const ch = html[i];
899 if (ch === '{') depth++;
900 else if (ch === '}') { depth--; if (depth === 0) { end = i + 1; break; } }
901 else if (ch === '"') { i++; while (i < html.length && html[i] !== '"') { if (html[i] === '\\') i++; i++; } }
902 }
903 if (end === -1) return null;
904 try { return JSON.parse(html.slice(start, end)); } catch { return null; }
905}
906
907
908function extractForwardObject(html, marker) {
909 const mi = html.indexOf(marker);
910 if (mi === -1) return null;
911 const start = html.indexOf('{', mi + marker.length - 1);
912 if (start === -1) return null;
913 let depth = 0;
914 for (let i = start; i < html.length && i < start + 200000; i++) {
915 const ch = html[i];
916 if (ch === '{') depth++;
917 else if (ch === '}') { depth--; if (depth === 0) { try { return JSON.parse(html.slice(start, i + 1)); } catch { return null; } } }
918 else if (ch === '"') { i++; while (i < html.length && html[i] !== '"') { if (html[i] === '\\') i++; i++; } }
919 }
920 return null;
921}
922
923
924
925function parsePdpData(html) {
926 const out = {
927 reviewsCount: null, ratingAverage: null,
928 availableStock: null, soldQuantity: null,
929 variants: null, seller: null,
930
931 condition: null, brandId: null,
932 catalogParentId: null, catalogProductId: null,
933 categoryId: null, categoryPath: null, categoryPathNames: null, catalogUrl: null,
934 listingTypeId: null, logisticType: null, shippingMode: null,
935 localPickup: null, storePickup: null, freeShippingConfirmed: null,
936 hasExtendedWarranty: null, stockType: null, tags: null,
937 tradeInEligible: null, itemOrigins: null,
938 reviewsWithComment: null, reviewsWithPhotos: null,
939 seoKeywords: null, unitsSold: null,
940 trackPrice: null, trackOriginalPrice: null, trackCurrency: null,
941 };
942
943 const t = extractObjectAround(html, '"reputation_level"');
944 if (t && typeof t === 'object') {
945 if (t.reviews && typeof t.reviews === 'object') {
946 out.reviewsCount = Number.isFinite(t.reviews.count) ? t.reviews.count : null;
947 out.ratingAverage = Number.isFinite(t.reviews.rate) ? t.reviews.rate : null;
948 out.reviewsWithComment = Number.isFinite(t.reviews.reviews_with_comment) ? t.reviews.reviews_with_comment : null;
949 out.reviewsWithPhotos = Number.isFinite(t.reviews.pictures_quantity) ? t.reviews.pictures_quantity : null;
950 }
951 if (out.ratingAverage == null && Number.isFinite(t.review_rate)) out.ratingAverage = t.review_rate;
952 if (Number.isFinite(t.quantity)) out.availableStock = t.quantity;
953
954
955 out.condition = normalizeCondition(t.item_condition);
956 out.brandId = t.item_attribute_brand_id != null ? String(t.item_attribute_brand_id) : null;
957 out.catalogParentId = t.catalog_parent_id && t.catalog_parent_id !== 'not_apply' ? String(t.catalog_parent_id) : null;
958 out.catalogProductId = t.catalog_product_id && t.catalog_product_id !== 'not_apply' ? String(t.catalog_product_id) : null;
959 out.categoryId = t.category_id || null;
960 out.listingTypeId = t.listing_type_id || null;
961 out.logisticType = t.logistic_type && t.logistic_type !== 'not_apply' ? t.logistic_type : null;
962 out.shippingMode = t.shipping_mode && t.shipping_mode !== 'not_apply' ? t.shipping_mode : null;
963 out.localPickup = typeof t.local_pick_up === 'boolean' ? t.local_pick_up : null;
964 out.storePickup = typeof t.store_pick_up === 'boolean' ? t.store_pick_up : null;
965 out.freeShippingConfirmed = typeof t.free_shipping === 'boolean' ? t.free_shipping : null;
966 out.hasExtendedWarranty = typeof t.has_garex === 'boolean' ? t.has_garex : null;
967 out.stockType = t.stock_type && t.stock_type !== 'not_apply' ? t.stock_type : null;
968 out.tags = Array.isArray(t.tags) && t.tags.length ? t.tags : null;
969 out.tradeInEligible = t.trade_in_status ? t.trade_in_status === 'eligible' : null;
970 out.itemOrigins = Array.isArray(t.item_origins) && t.item_origins.length ? t.item_origins : null;
971 out.unitsSold = Number.isFinite(t.sold_quantity) ? t.sold_quantity : null;
972 out.trackPrice = num(t.price);
973 out.trackOriginalPrice = num(t.original_price);
974 out.trackCurrency = t.currency_id || null;
975 if (Array.isArray(t.seo?.seo_top_keywords)) {
976 const kws = t.seo.seo_top_keywords.map((k) => k.keyword).filter(Boolean);
977 if (kws.length) out.seoKeywords = [...new Set(kws)];
978 }
979
980 if (t.seller_id != null || t.seller_name) {
981 out.seller = {
982 sellerName: t.seller_name || null,
983 sellerId: t.seller_id != null ? String(t.seller_id) : null,
984 reputationLevel: t.reputation_level || null,
985 powerSellerStatus: t.power_seller_status || null,
986 isOfficialStore: t.official_store_id != null,
987 officialStoreId: t.official_store_id != null ? String(t.official_store_id) : null,
988 };
989 }
990
991
992
993
994 if (t.pickers && typeof t.pickers === 'object') {
995 const labels = t.pickers_attributes || {};
996 const variants = [];
997 for (const [attr, list] of Object.entries(t.pickers)) {
998 if (!Array.isArray(list)) continue;
999 for (const v of list) {
1000 if (!v || typeof v !== 'object') continue;
1001 variants.push({
1002 attribute: labels[attr]?.picker_label || attr,
1003 value: v.value ?? null,
1004 sku: v.catalog_product_id || v.attribute_id || null,
1005 selected: v.selected === true,
1006 price: null,
1007 stock: null,
1008 });
1009 }
1010 }
1011 if (variants.length) out.variants = variants;
1012 }
1013 }
1014
1015
1016 const header = extractObjectAround(html, '"subtitle":"Nuevo');
1017 let subtitle = header?.subtitle;
1018 if (!subtitle) {
1019 const sm = html.match(/"subtitle":"([^"]*vendido[^"]*)"/i);
1020 if (sm) subtitle = sm[1];
1021 }
1022 if (typeof subtitle === 'string') {
1023 const vm = subtitle.match(/([+\d.,]+)\s*vendidos?/i);
1024 if (vm) out.soldQuantity = vm[0].replace(/\s+/g, ' ').trim();
1025 }
1026
1027
1028 const cpm = html.match(/"category_path":\[([^\]]*)\]/);
1029 if (cpm) {
1030 const ids = cpm[1].split(',').map((s) => s.replace(/[^A-Z0-9]/gi, '')).filter(Boolean);
1031 if (ids.length) out.categoryPath = ids;
1032 }
1033
1034
1035 const bc = extractForwardObject(html, '"breadcrumb":');
1036 if (bc) {
1037 if (Array.isArray(bc.categories)) {
1038 const names = bc.categories.map((c2) => cleanText(c2.label?.text)).filter(Boolean);
1039 if (names.length) out.categoryPathNames = names;
1040 }
1041 const backUrl = bc.back?.target;
1042 if (backUrl && /\/p\//.test(backUrl)) out.catalogUrl = backUrl.split('?')[0];
1043 }
1044
1045 return out;
1046}
1047
1048
1049
1050function attrValue(attributes, re) {
1051 if (!Array.isArray(attributes)) return null;
1052 for (const a of attributes) {
1053 if (a && a.name && re.test(String(a.name)) && a.value) return cleanText(String(a.value));
1054 }
1055 return null;
1056}
1057
1058
1059function parseDetail(html) {
1060 const out = { attributes: null, description: null, images: null, title: null };
1061 const $ = cheerio.load(html);
1062
1063
1064 out.title = cleanText($('h1.ui-pdp-title').first().text())
1065 || cleanText(($('title').first().text() || '').split('|')[0]);
1066
1067
1068 const rows = [];
1069 $('.ui-pdp-specs__table tr, .andes-table__row').each((_, tr) => {
1070 const k = $(tr).find('th, .andes-table__header').first().text().trim();
1071 const v = $(tr).find('td, .andes-table__column--value').first().text().trim();
1072 if (k && v) rows.push({ name: k, value: v });
1073 });
1074 $('.ui-vpp-highlighted-specs__key-value, .ui-pdp-highlighted-specs-res__key-value').each((_, el) => {
1075 const t = $(el).text().trim();
1076 const m = t.split(':');
1077 if (m.length >= 2) rows.push({ name: m[0].trim(), value: m.slice(1).join(':').trim() });
1078 });
1079 if (rows.length) {
1080 const seen = new Set();
1081 out.attributes = rows.filter((r) => {
1082 const key = r.name.toLowerCase();
1083 if (seen.has(key)) return false;
1084 seen.add(key);
1085 return true;
1086 });
1087 }
1088
1089
1090 const d = $('.ui-pdp-description__content, [data-testid="content"]').first().text().trim();
1091 if (d) out.description = d;
1092
1093
1094 const imgs = new Set();
1095 $('.ui-pdp-gallery img, figure.ui-pdp-gallery__figure img, .ui-pdp-image').each((_, im) => {
1096 let s = $(im).attr('data-zoom') || $(im).attr('data-src') || $(im).attr('src');
1097 if (s && /mlstatic/.test(s)) {
1098 s = s.replace(/-[A-Z]\.(jpg|webp|png)/, '-O.$1');
1099 imgs.add(s);
1100 }
1101 });
1102 if (imgs.size) out.images = [...imgs];
1103
1104 return out;
1105}
1106
1107
1108function pageUrl(base, offset) {
1109 if (offset <= 0) return base;
1110 try {
1111 const u = new URL(base);
1112
1113 if (/_Desde_\d+/.test(u.pathname)) {
1114 u.pathname = u.pathname.replace(/_Desde_\d+/, `_Desde_${offset + 1}`);
1115 } else if (/_NoIndex_True$/.test(u.pathname)) {
1116
1117 u.pathname = u.pathname.replace(/_NoIndex_True$/, `_Desde_${offset + 1}_NoIndex_True`);
1118 } else {
1119 const segs = u.pathname.split('/').filter(Boolean);
1120
1121 u.pathname = '/' + segs.join('/') + `_Desde_${offset + 1}`;
1122 }
1123 return u.toString().replace(/%2A/gi, '*');
1124 } catch {
1125 return base;
1126 }
1127}
1128
1129
1130
1131
1132
1133
1134async function resolveFilteredStartUrl(job) {
1135 if (!job.searchTermMode) return;
1136 const seg = buildFilterSegments(input);
1137 if (!seg) return;
1138 let html;
1139 try { html = await fetchHtml(job.startUrl); } catch { return; }
1140 const flat = html.replace(/\\u002F/g, '/').replace(/\\\//g, '/');
1141
1142 let sample = null;
1143 const ms = flat.match(/"sort":\{[\s\S]{0,600}?"url":"(https?:[^"]+?)"/);
1144 if (ms) sample = ms[1];
1145 if (!sample) {
1146 const mf = flat.match(/https?:\/\/[^"\s\\]*?_(?:OrderId|PriceRange|ITEM\*CONDITION|Tienda|Discount|CustId)[^"\s\\]*/);
1147 if (mf) sample = mf[0];
1148 }
1149 if (!sample) return;
1150 const narrowed = sample.split(/_OrderId|_PriceRange|_ITEM\*CONDITION|_NoIndex|_Tienda|_Discount|_shipping|_CustId|_Deal|_Desde|[?#]/)[0];
1151 if (!/^https?:\/\//.test(narrowed)) return;
1152 job.startUrl = `${narrowed}${seg}_NoIndex_True`;
1153 console.log(c.dim(' filters applied → ') + c.white(job.startUrl.replace(/^https?:\/\//, '')));
1154}
1155
1156
1157function parseTotalResults(html) {
1158 const m = html.match(/([\d.,]+)\s*resultad|([\d.,]+)\s*result/i);
1159 if (!m) return null;
1160 const n = num(m[1] || m[2]);
1161 return Number.isFinite(n) ? n : null;
1162}
1163
1164
1165
1166function bucketSold(n) {
1167 if (!Number.isFinite(n) || n <= 0) return null;
1168 const steps = [100000, 50000, 10000, 5000, 1000, 500, 100, 50, 25, 10, 5];
1169 for (const s of steps) {
1170 if (n >= s) return `+${s >= 1000 ? (s / 1000) + 'mil' : s} vendidos`;
1171 }
1172 return `${n} vendidos`;
1173}
1174
1175
1176
1177
1178
1179
1180const BS_HOST = 'https://www.mercadolibre.com.ar';
1181const BS_POST_URL = `${BS_HOST}/ofertas/api/dynamic-component?scope=mla`;
1182
1183
1184function resolveBestSellerCategories(rawList) {
1185 const entries = Object.entries(AR_BESTSELLER_CATEGORIES);
1186 if (!rawList || !rawList.length) return entries.map(([id, name]) => ({ id, name }));
1187 const out = [];
1188 const norm = (s) => String(s).toLowerCase().normalize('NFD').replace(/[̀-ͯ]/g, '');
1189 for (const raw of rawList) {
1190 const r = String(raw).trim();
1191 if (/^MLA\d+$/i.test(r)) { out.push({ id: r.toUpperCase(), name: AR_BESTSELLER_CATEGORIES[r.toUpperCase()] || null }); continue; }
1192 const hit = entries.find(([, name]) => norm(name).includes(norm(r)) || norm(r).includes(norm(name)));
1193 if (hit) out.push({ id: hit[0], name: hit[1] });
1194 else console.log(c.yellow(` best-seller category not recognised: "${r}" — skipped`));
1195 }
1196 return out;
1197}
1198
1199function mapBestSellerItem(it, cat, rankFallback) {
1200 const permalink = cleanText(it.permalink || it.url || (it.target && it.target.url));
1201 const catalogId = permalink ? (permalink.match(/\/p\/(MLA\d+)/) || [])[1] || null : null;
1202 const priceObj = (it.price && typeof it.price === 'object') ? it.price : {};
1203 const price = num(it.price?.value ?? it.price?.amount ?? it.price);
1204 const originalPrice = num(it.original_price ?? it.price?.original_value ?? it.price?.previous_price);
1205 const shipping = it.shipping || {};
1206 return {
1207 url: permalink,
1208 listingId: catalogId || cleanText(it.id),
1209 catalogProductId: catalogId,
1210 title: cleanText(it.title),
1211 price,
1212 originalPrice,
1213 currency: cleanText(it.currency_id || it.price?.currency_id || 'ARS'),
1214 discountPercent: num(it.discount ?? it.price?.discount),
1215 freeShipping: shipping.free_shipping === true || /gratis/i.test(cleanText(shipping.title) || ''),
1216 installments: cleanText(typeof it.installments === 'object' ? it.installments?.text : it.installments),
1217 rankPosition: num(it.best_sellers_position) ?? rankFallback,
1218 categoryId: cat.id,
1219 categoryName: cat.name,
1220 rankingCategory: cat.name || cat.id,
1221 thumbnail: cleanText(it.thumbnail || it.picture || (it.pictures && it.pictures[0])),
1222 hasVariations: it.has_variations === true || undefined,
1223 };
1224}
1225
1226
1227
1228
1229function parseRankingPage(html) {
1230 const h = html.replace(/\\u002F/g, '/');
1231
1232 const parts = h.split('<li class="ui-search-layout__item"').slice(1);
1233 const out = [];
1234 const seen = new Set();
1235 parts.forEach((seg, idx) => {
1236 const pid = (seg.match(/\/p\/(MLA\d+)/) || [])[1] || null;
1237 const title = cleanText((seg.match(/poly-component__title[^>]*>([^<]{3,})</) || [])[1]);
1238 if (!pid && !title) return;
1239 const key = pid || title;
1240 if (seen.has(key)) return;
1241 seen.add(key);
1242 const prices = [...seg.matchAll(/andes-money-amount__fraction[^>]*>([\d.]+)/g)].map((m) => num(m[1]));
1243 const price = prices.length ? prices[0] : null;
1244 const prevPrice = prices.length > 1 ? Math.max(...prices) : null;
1245 const rankBadge = num((seg.match(/(\d+)[º°]\s*M[ÁA]S VENDIDO/i) || [])[1]);
1246 const img = (seg.match(/poly-component__picture"[^>]*\ssrc="([^"]+)"/) || [])[1]
1247 || (seg.match(/(https:\/\/http2\.mlstatic\.com\/[^"]+\.(?:webp|jpg))/) || [])[1] || null;
1248 const installments = cleanText((seg.match(/poly-price__installments[^>]*>(.*?)<\/(?:span|div|p)>/s) || [])[1]?.replace(/<[^>]+>/g, ' '));
1249 out.push({
1250 rankPosition: rankBadge || (idx + 1),
1251 catalogProductId: pid,
1252 title,
1253 imageUrl: img,
1254 price,
1255 originalPrice: prevPrice && prevPrice > (price || 0) ? prevPrice : null,
1256 soldQuantity: cleanText((seg.match(/(\+?[\d.]+(?:mil)?\s*vendidos)/) || [])[1]),
1257 ratingAverage: num((seg.match(/Calificaci[oó]n\s+([\d.]+)\s+de\s+5/) || [])[1]),
1258 sellerName: cleanText((seg.match(/poly-component__seller[^>]*>([^<]{2,})</) || [])[1]),
1259 freeShipping: /Env[ií]o gratis/i.test(seg),
1260 officialStore: /Tienda oficial/i.test(seg),
1261 installments,
1262 });
1263 });
1264 return out;
1265}
1266
1267function mapRankingCard(card, cat) {
1268 const url = card.catalogProductId ? `${BS_HOST}/p/${card.catalogProductId}` : null;
1269 const discountPercent = (card.originalPrice && card.price)
1270 ? Math.round((1 - card.price / card.originalPrice) * 100) : null;
1271 return {
1272 imageUrl: card.imageUrl,
1273 url,
1274 listingId: card.catalogProductId,
1275 catalogProductId: card.catalogProductId,
1276 title: card.title,
1277 price: card.price,
1278 originalPrice: card.originalPrice,
1279 currency: 'ARS',
1280 currencySymbol: '$',
1281 discountPercentage: discountPercent && discountPercent > 0 ? discountPercent : null,
1282 freeShipping: card.freeShipping,
1283 officialStore: card.officialStore,
1284 installments: card.installments,
1285 ratingAverage: card.ratingAverage,
1286 soldQuantity: card.soldQuantity,
1287 sellerName: card.sellerName,
1288 rankPosition: card.rankPosition,
1289 categoryId: cat.id,
1290 categoryName: cat.name,
1291 rankingCategory: cat.name || cat.id,
1292 };
1293}
1294
1295
1296
1297let __bffTemplate; let __bffTemplateTried = false;
1298async function bffRanking(cat) {
1299 if (!__bffTemplateTried) {
1300 __bffTemplateTried = true;
1301 try {
1302 const mv = await fetchViaUnblocker(`${BS_HOST}/mas-vendidos`, undefined, { tries: 2 });
1303 const tmpl = extractObjectAround(mv, '"CarouselDynamic"');
1304 if (tmpl && JSON.stringify(tmpl).includes('best_sellers')) __bffTemplate = tmpl;
1305 } catch { }
1306 }
1307 if (!__bffTemplate) return [];
1308 let comp;
1309 try { comp = JSON.parse(JSON.stringify(__bffTemplate)); } catch { return []; }
1310 if (comp.filters && comp.filters.dealOrUrl) comp.filters.dealOrUrl.categoryId = cat.id;
1311 else if (comp.filters) comp.filters.categoryId = cat.id;
1312 const body = JSON.stringify({ component: comp, siteId: 'MLA', vertical: comp.vertical || 'COMMUNICATIONALS', webView: false, url: 'www.mercadolibre.com.ar/mas-vendidos' });
1313 let json;
1314 try { json = JSON.parse(await fetchViaUnblocker(BS_POST_URL, `${BS_HOST}/mas-vendidos`, { method: 'POST', body, headers: { Accept: 'application/json' }, tries: 2 })); }
1315 catch { return []; }
1316 const items = [];
1317 (function walk(o) {
1318 if (!o || typeof o !== 'object') return;
1319 if (Array.isArray(o)) { o.forEach(walk); return; }
1320 if (o.best_sellers_position != null && (o.title || o.permalink)) items.push(o);
1321 for (const v of Object.values(o)) walk(v);
1322 })(json);
1323 items.sort((a, b) => (num(a.best_sellers_position) || 999) - (num(b.best_sellers_position) || 999));
1324 return items.map((it, i) => mapBestSellerItem(it, cat, i + 1)).filter((r) => r.url || r.title);
1325}
1326
1327async function runBestSellers() {
1328 if (!allowUnblocker) {
1329 await Actor.pushData([{
1330 error: 'Best-seller ranking mode requires premium unblocking, available on a paid Apify plan. Free runs use the standard proxy only, which MercadoLibre walls. Upgrade to a paid plan (or supply a clean residential/mobile proxy) and retry.',
1331 source: 'MercadoLibre', observedAt: new Date().toISOString(),
1332 }]);
1333 return [];
1334 }
1335 const cats = resolveBestSellerCategories(toList(input.bestSellerCategories));
1336 const out = [];
1337 setStatus(`Ranking de más vendidos: ${cats.length} categorías…`);
1338 let catsDone = 0;
1339 await mapPool(cats, 6, async (cat) => {
1340 if (out.length >= effectiveMaxItems) return;
1341
1342 let recs = [];
1343 try {
1344 const html = await fetchViaUnblocker(`${BS_HOST}/mas-vendidos/${cat.id}`, `${BS_HOST}/mas-vendidos`, { ua: PDP_CRAWLER_UA, tries: 2, timeout: 115000 });
1345 recs = parseRankingPage(html).map((card) => mapRankingCard(card, cat));
1346 if (recs.length) recordVia('unblocker-ranking');
1347 } catch (e) { }
1348
1349 if (!recs.length) {
1350 try { recs = await bffRanking(cat); if (recs.length) recordVia('unblocker-ranking-bff'); }
1351 catch { }
1352 }
1353 for (const rec of recs) {
1354 if (out.length >= effectiveMaxItems) break;
1355 out.push(rec);
1356 }
1357 catsDone++;
1358 console.log(pct((catsDone / cats.length) * 100) + c.dim(`[${cat.name || cat.id}]`) + ' ' + c.white(`${recs.length} ranked · ${out.length} total`));
1359 setStatus(`Ranking: ${out.length} productos · ${catsDone}/${cats.length} categorías…`);
1360 });
1361 return out;
1362}
1363
1364
1365const collected = [];
1366const seenIds = new Set();
1367const PER_PAGE = 48;
1368let wallErrorPushed = false;
1369
1370
1371const bestSellersMode = input.bestSellers === true;
1372if (bestSellersMode) {
1373 const ranked = await runBestSellers();
1374 for (const item of ranked) {
1375 if (collected.length >= effectiveMaxItems) break;
1376 const key = item.listingId || item.url;
1377 if (key && seenIds.has(key)) continue;
1378 if (key) seenIds.add(key);
1379 item.countryCode = 'ar';
1380 collected.push(item);
1381 }
1382}
1383
1384
1385
1386for (const u of directPdpUrls) {
1387 if (collected.length >= effectiveMaxItems) break;
1388 const catalogId = (u.match(/\/p\/(ML[A-Z]?\d+)/i) || [])[1] || null;
1389 const itemId = (u.match(/\/(ML[A-Z]-?\d{6,})/i) || [])[1] || null;
1390 const key = catalogId || itemId || u;
1391 if (seenIds.has(key)) continue;
1392 seenIds.add(key);
1393 collected.push({ url: u, catalogProductId: catalogId, listingId: catalogId || itemId, countryCode: countryFromUrl(u) });
1394}
1395const effectiveWithDetails = withDetails || directPdpUrls.length > 0;
1396
1397
1398
1399
1400for (const job of (bestSellersMode ? [] : jobs)) {
1401 if (collected.length >= effectiveMaxItems) break;
1402 await resolveFilteredStartUrl(job);
1403 const jobCountry = countryFromUrl(job.startUrl);
1404 const remaining = effectiveMaxItems - collected.length;
1405 const totalPages = Math.max(1, Math.ceil(remaining / PER_PAGE));
1406 const offsets = Array.from({ length: totalPages }, (_, p) => p * PER_PAGE);
1407 let pagesDone = 0;
1408 let jobTotalResults = null;
1409 const label = job.searchTerm ? `“${job.searchTerm}”` : job.startUrl.replace(/^https?:\/\//, '').slice(0, 40);
1410 setStatus(`Buscando ${label} en MercadoLibre (${jobCountry})…`);
1411 console.log(c.dim(` searching ${jobCountry} · ${label} · up to ${remaining} more`));
1412 await mapPool(offsets, Math.min(totalPages, 4), async (offset, p) => {
1413 if (collected.length >= effectiveMaxItems) return;
1414 const url = pageUrl(job.startUrl, offset);
1415 const host = (() => { try { return new URL(url).hostname; } catch { return 'www.mercadolibre.com.ar'; } })();
1416 try {
1417 const html = await fetchHtml(url, p === 0 ? undefined : job.startUrl);
1418 if (jobTotalResults == null) jobTotalResults = parseTotalResults(html);
1419 const listings = parseSearch(html, host);
1420 for (const item of listings) {
1421 if (collected.length >= effectiveMaxItems) break;
1422 const key = item.listingId || item.url;
1423 if (key && seenIds.has(key)) continue;
1424 if (key) seenIds.add(key);
1425 item.searchQuery = job.searchTerm || null;
1426 item.countryCode = jobCountry;
1427 item.pageNumber = p + 1;
1428 item.totalResults = jobTotalResults;
1429 collected.push(item);
1430 }
1431 } catch (e) {
1432 const msg = String(e.message || e);
1433
1434
1435
1436 if (/suspicious-traffic wall/i.test(msg)) {
1437 if (!wallErrorPushed) {
1438 wallErrorPushed = true;
1439 const wallMsg = usingCustomProxy
1440 ? 'MercadoLibre blocked this request with a suspicious-traffic verification (login) wall even through the custom proxy, so no products could be retrieved. You were NOT charged for this run. Try a cleaner residential/mobile proxy pool, and/or supply a logged-in session cookie, then retry.'
1441 : isPaying
1442 ? 'MercadoLibre blocked this request with a suspicious-traffic verification (login) wall on every transport (residential and premium unblocking), so no products could be retrieved. You were NOT charged for this run. This is a temporary IP-reputation block on MercadoLibre’s side; please retry shortly.'
1443 : 'MercadoLibre is currently blocking automated access with a suspicious-traffic verification (login) wall, so no products could be retrieved. You were NOT charged for this run. Free runs use the standard proxy only — upgrade to a paid Apify plan for delivery via premium unblocking, or supply your own clean residential/mobile proxy (proxyUrl) for a much higher success rate.';
1444 await Actor.pushData([{
1445 error: wallMsg,
1446 source: 'MercadoLibre', sourceUrl: url, observedAt: new Date().toISOString(),
1447 }]);
1448 }
1449 } else {
1450
1451
1452 const safe = msg
1453 .replace(/https?:\/\/\S+/g, '')
1454 .replace(/\b\d{1,3}(?:\.\d{1,3}){3}(?::\d+)?\b/g, '')
1455 .replace(/[\w.-]+:\d+/g, '')
1456 .replace(/\s+/g, ' ').trim().slice(0, 180) || 'request failed';
1457 await Actor.pushData([{ error: safe, source: 'MercadoLibre', sourceUrl: url, observedAt: new Date().toISOString() }]);
1458 }
1459 }
1460 pagesDone++;
1461 setStatus(`Buscando ${label}… ${pagesDone}/${totalPages} páginas · ${collected.length} encontrados`);
1462 console.log(pct((pagesDone / totalPages) * 100) + c.dim(`page ${pagesDone}/${totalPages}`) + ' ' + c.white(`${collected.length} products`));
1463 });
1464}
1465
1466
1467const toEmit = collected.slice(0, effectiveMaxItems);
1468let pushed = 0;
1469
1470
1471
1472let detailsOk = 0;
1473
1474if (effectiveWithDetails) {
1475
1476
1477
1478 setStatus(`Obteniendo detalle de ${toEmit.length} productos…`);
1479 await mapPool(toEmit, 8, async (item) => {
1480 let done = false;
1481 if (item.url) {
1482 try {
1483
1484 const referer = (() => {
1485 try { const h = new URL(item.url).hostname.replace(/^articulo\.|^www\./, ''); return `https://www.${h}/`; }
1486 catch { return undefined; }
1487 })();
1488 const dhtml = await fetchPdpHtml(item.url, referer);
1489 const pdp = parsePdpData(dhtml);
1490 const detail = parseDetail(dhtml);
1491 const rec = {
1492 ...item,
1493
1494 title: item.title ?? detail.title,
1495 price: item.price ?? pdp.trackPrice,
1496 originalPrice: item.originalPrice ?? pdp.trackOriginalPrice,
1497 currency: item.currency ?? pdp.trackCurrency,
1498 condition: item.condition ?? pdp.condition,
1499 ratingAverage: pdp.ratingAverage ?? item.ratingAverage,
1500 unitsSold: item.unitsSold ?? pdp.unitsSold,
1501 catalogProductId: item.catalogProductId ?? pdp.catalogProductId,
1502 categoryId: item.categoryId ?? pdp.categoryId,
1503 freeShipping: item.freeShipping || pdp.freeShippingConfirmed === true,
1504 reviewsCount: pdp.reviewsCount,
1505 availableStock: pdp.availableStock,
1506 soldQuantity: pdp.soldQuantity ?? item.soldQuantity ?? bucketSold(item.unitsSold ?? pdp.unitsSold),
1507 seller: pdp.seller,
1508 attributes: detail.attributes,
1509 description: detail.description,
1510 variants: pdp.variants,
1511 images: detail.images,
1512
1513
1514 brand: attrValue(detail.attributes, /^(marca|brand)$/i),
1515 model: attrValue(detail.attributes, /^(modelo|model|modelo del producto)$/i),
1516 warranty: attrValue(detail.attributes, /garant|warranty/i),
1517
1518 brandId: pdp.brandId,
1519 catalogParentId: pdp.catalogParentId,
1520 categoryPath: pdp.categoryPath,
1521 categoryPathNames: pdp.categoryPathNames,
1522 catalogUrl: pdp.catalogUrl,
1523 listingTypeId: pdp.listingTypeId,
1524 logisticType: pdp.logisticType,
1525 shippingMode: pdp.shippingMode,
1526 localPickup: pdp.localPickup,
1527 storePickup: pdp.storePickup,
1528 hasExtendedWarranty: pdp.hasExtendedWarranty,
1529 stockType: pdp.stockType,
1530 tags: pdp.tags,
1531 tradeInEligible: pdp.tradeInEligible,
1532 itemOrigins: pdp.itemOrigins,
1533 reviewsWithComment: pdp.reviewsWithComment,
1534 reviewsWithPhotos: pdp.reviewsWithPhotos,
1535 seoKeywords: pdp.seoKeywords,
1536 observedAt: new Date().toISOString(),
1537 error: null,
1538 };
1539 await aiEnrich(rec);
1540 await Actor.pushData([rec], 'result');
1541
1542
1543
1544 await Actor.charge({ eventName: 'details' });
1545 done = true;
1546 detailsOk++;
1547 } catch { }
1548 }
1549
1550
1551
1552 if (!done) { const rec = { ...item, observedAt: new Date().toISOString(), error: null }; await aiEnrich(rec); await Actor.pushData([rec], 'result'); }
1553 pushed++;
1554 if (pushed % 5 === 0 || pushed === toEmit.length) {
1555 setStatus(`Procesando detalles… ${pushed}/${toEmit.length} (${detailsOk} con detalle)`);
1556 console.log(pct((pushed / toEmit.length) * 100) + c.dim(`detail ${pushed}/${toEmit.length} · ${detailsOk} ok`));
1557 }
1558 });
1559
1560
1561
1562
1563 if (toEmit.length && detailsOk === 0) {
1564 await Actor.pushData([{
1565 error: 'Product detail pages could not be retrieved for this run, so these rows contain listing-level fields only. You were NOT charged for the "details" add-on. This is a temporary block on MercadoLibre’s side; retry later, or supply your own clean residential/mobile proxy (proxyUrl) for a higher success rate.',
1566 source: 'MercadoLibre', observedAt: new Date().toISOString(),
1567 }]);
1568 setStatus(`Listo: ${pushed} productos (sin detalle — bloqueado, no se cobró el add-on)`);
1569 console.log(pct(100) + c.yellow('warn') + c.dim(' detail add-on unavailable this run; not charged'));
1570 }
1571} else {
1572 for (const item of toEmit) {
1573 const rec = { ...item, observedAt: new Date().toISOString(), error: null };
1574 await aiEnrich(rec);
1575 await Actor.pushData([rec], 'result');
1576 pushed++;
1577 }
1578}
1579
1580const viaSummary = Object.entries(servedVia).map(([k, v]) => `${k}:${v}`).join(', ') || 'none';
1581console.log(c.dim(' transport served → ') + c.white(viaSummary) + c.dim(` (unblocker ${allowUnblocker ? 'enabled' : 'disabled — free run'}${unblockerEngaged ? ', engaged' : ''})`));
1582setStatus(`Listo: ${pushed} productos extraídos.`);
1583console.log(pct(100) + c.green('done') + c.dim(` ${pushed} products`));
1584
1585await Actor.exit().catch(() => {});
1586setTimeout(() => process.exit(0), 2000).unref?.();
