1
2
3
4
5
6import { log } from 'apify';
7import OpenAI from 'openai';
8
9import { BRAND_CAPS } from './constants.js';
10
11const BATCH_SIZE = 10;
12const MAX_RETRIES = 3;
13const INITIAL_RETRY_DELAY_MS = 1000;
14const DEFAULT_CONCURRENCY = 5;
15const LARGE_REQUEST_THRESHOLD = 100;
16const LARGE_REQUEST_CONCURRENCY = 10;
17const MAX_TOKENS = 2048;
18
19
20const OPENROUTER_BASE_URL = 'https://openrouter.apify.actor/api/v1';
21
22const OPENROUTER_DIRECT_URL = 'https://openrouter.ai/api/v1';
23
24
25
26
27const STYLE_MODIFIERS = {
28 clean: `Extract the CORE product name, removing noise but KEEPING the product identity:
29
30REMOVE:
31- Promotional text: BESTSELLER, SALE, FREE SHIPPING, LIMITED, EXCLUSIVE, HOT DEAL, etc.
32- Prices and discounts: $19.99, 50% OFF, SAVE $10, etc.
33- Size/variant info: Large, Blue, Pack of 3, 12oz, etc.
34- Amazon ASINs and prefixed codes: "B08N5WRWNW - ", "ASIN: B0BSHF7WHB | "
35- Store/brand repetition: "Nike Nike Air Max" -> "Nike Air Max"
36- Category breadcrumbs: "Clothing > Men > T-Shirts >"
37- Shipping info: Ships Free, 2-Day Delivery, etc.
38- Stock status: In Stock, Only 3 Left, etc.
39- Tags like [REFURBISHED], ***NEW***, emojis
40
41KEEP (critical — do NOT strip these):
42- Brand name (once, at start if present). "New Balance" is a brand, NOT promotional text.
43- Product model numbers that ARE the product name: WH-1000XM5, RTX 4090, A7 IV, X1 Carbon
44 * Model numbers with letters and digits are product identifiers, NOT SKU codes
45 * "Sony WH-1000XM5" -> "Sony WH-1000XM5" (KEEP the model number)
46 * "Roomba j7+" -> "Roomba j7+" (KEEP the model)
47- Key identifying features that are part of the product identity
48
49IMPORTANT: Never return an empty string. If input contains a recognizable product, extract it.
50- "ASIN: B0BSHF7WHB | Anker 735 Charger (GaNPrime 65W)" -> "Anker 735 Charger"
51- "B08N5WRWNW - Apple AirTag (4 Pack)" -> "Apple AirTag"
52
53Examples:
54- "BESTSELLER! Nike Air Max 90 | Men's | Size 10 - $129.99" -> "Nike Air Max 90"
55- "Apple iPhone 15 Pro Max 256GB Space Black - Apple Store" -> "Apple iPhone 15 Pro Max"
56- "New Balance 990v6 M990GL6" -> "New Balance 990v6" (New Balance is a BRAND)
57- "Sony WH-1000XM5/B Wireless NC Headphones - Black" -> "Sony WH-1000XM5 Headphones"
58- "🔥 HOT DEAL Sony WH-1000XM5 Headphones $279" -> "Sony WH-1000XM5 Headphones"
59- "Clothing > Men > Shoes > Nike Running Shoes" -> "Nike Running Shoes"
60
61Output proper title case. Preserve brand capitalizations (iPhone, PlayStation, MacBook).`,
62
63 full: `Clean and properly FORMAT the product name, keeping descriptive attributes:
64
65FIX:
66- ALL CAPS -> Title Case (preserve brand caps: iPhone, MacBook, PlayStation)
67- Encoding issues: ’ -> ', & -> &
68- Extra whitespace
69- Inconsistent punctuation
70
71REMOVE:
72- Promotional text: BESTSELLER, SALE, NEW, FREE SHIPPING, etc.
73- Prices and discounts
74- SKU/product codes
75- Store/site names at the end
76- Stock status
77
78KEEP:
79- Brand name
80- Product model/name
81- Color, size, material when they're part of the product identity
82- Quantity/pack size (12oz, Pack of 6)
83- Model numbers that are part of the name
84
85Examples:
86- "nike air max 90 mens white" -> "Nike Air Max 90 Men's White"
87- "ORGANIC COFFEE BEANS 12OZ MEDIUM ROAST" -> "Organic Coffee Beans 12oz Medium Roast"
88- "iphone 15 pro max 256gb" -> "iPhone 15 Pro Max 256GB"
89- "SONY WH-1000XM5 WIRELESS HEADPHONES BLACK" -> "Sony WH-1000XM5 Wireless Headphones Black"
90- "mens blue cotton t-shirt large" -> "Men's Blue Cotton T-Shirt Large"`,
91
92 searchable: `Normalize product name for SEARCH and DEDUPLICATION:
93
94OUTPUT RULES:
95- Lowercase everything
96- Remove special characters (keep only letters, numbers, spaces, hyphens)
97- Remove sizes, colors, variants
98- Remove promotional text
99- Remove prices, SKUs
100- Keep only core product identity for matching
101
102GOAL: Two variations of the same product should produce identical output.
103
104Examples:
105- "Nike Air Max 90 (Men's Size 10)" -> "nike air max 90"
106- "NIKE AIR MAX 90 - White/Black" -> "nike air max 90"
107- "ORGANIC COFFEE BEANS - 12oz Medium Roast" -> "organic coffee beans"
108- "Organic Coffee Beans (Dark Roast, 16oz)" -> "organic coffee beans"
109- "Apple iPhone 15 Pro Max 256GB" -> "apple iphone 15 pro max"
110- "iPhone 15 Pro Max - 512GB Space Black" -> "apple iphone 15 pro max"
111
112Output lowercase, minimal punctuation, for database matching.`,
113};
114
115
116
117
118const STYLE_GOALS = {
119 clean: 'Core product name only, suitable for CRM, catalogs, and display.',
120 full: 'Properly formatted product name with attributes, for detailed listings.',
121 searchable: 'Normalized lowercase for deduplication, matching, and search indexes.',
122};
123
124
125
126
127const STYLE_TESTS = {
128 clean: 'Product catalog entry: [your answer]',
129 full: 'E-commerce listing title: [your answer]',
130 searchable: 'Database matching key: [your answer]',
131};
132
133
134
135
136
137
138function buildSystemPrompt(style) {
139 const effectiveStyle = STYLE_MODIFIERS[style] ? style : 'clean';
140
141 return `You are normalizing product names from e-commerce data, web scraping, and catalog exports.
142
143GOAL: ${STYLE_GOALS[effectiveStyle]}
144
145RULES:
146${STYLE_MODIFIERS[effectiveStyle]}
147
148BRAND CAPITALIZATION:
149Preserve these brand-specific capitalizations:
150- Apple: iPhone, iPad, iPod, iMac, MacBook, AirPods, iOS, macOS
151- Sony: PlayStation
152- Microsoft: Xbox, PowerPoint, OneDrive
153- Tech terms: WiFi, Bluetooth, HDMI, USB, LED, LCD, OLED, HD, 4K, 1080p
154
155EDGE CASES:
156- Generic product with no brand: "Blue T-Shirt" -> keep as is
157- Multiple separators (|, -, /, ::): extract the product name segment
158- Input with ASIN/SKU prefix: strip the code, keep the product name after it
159- Non-English with English portion: extract and normalize the English part
160- NEVER return an empty string if a product name can be extracted
161
162TEST: Before returning, mentally check: "${STYLE_TESTS[effectiveStyle]}" - does it look right?
163
164OUTPUT FORMAT (strict JSON, no comments):
165[{"index": 0, "normalized": "Product Name", "confidence": 0.95}]
166Return ONLY the JSON array with objects containing "index", "normalized", and "confidence" keys.`;
167}
168
169
170
171
172
173
174
175
176
177export function createNormalizer(model, style = 'clean', openRouterApiKey = null) {
178 let openai;
179
180 if (openRouterApiKey) {
181 log.info('Using direct OpenRouter API (bring your own key)');
182 openai = new OpenAI({
183 baseURL: OPENROUTER_DIRECT_URL,
184 apiKey: openRouterApiKey,
185 timeout: 120_000,
186 });
187 } else {
188 const apifyToken = process.env.APIFY_TOKEN;
189 if (!apifyToken) {
190 throw new Error('APIFY_TOKEN environment variable is required for OpenRouter access');
191 }
192 openai = new OpenAI({
193 baseURL: OPENROUTER_BASE_URL,
194 apiKey: 'apify-openrouter',
195 timeout: 120_000,
196 defaultHeaders: {
197 Authorization: `Bearer ${apifyToken}`,
198 },
199 });
200 }
201
202 const systemPrompt = buildSystemPrompt(style);
203
204 return {
205
206
207
208
209
210
211 async normalize(items) {
212
213 const batches = [];
214 for (let i = 0; i < items.length; i += BATCH_SIZE) {
215 batches.push({
216 items: items.slice(i, i + BATCH_SIZE),
217 startIndex: i,
218 });
219 }
220
221 const totalBatches = batches.length;
222
223
224 const concurrency =
225 items.length >= LARGE_REQUEST_THRESHOLD ? LARGE_REQUEST_CONCURRENCY : DEFAULT_CONCURRENCY;
226
227 log.info(`Processing ${items.length} items in ${totalBatches} batches (concurrency: ${concurrency})`);
228
229
230 const results = new Array(items.length);
231 for (let i = 0; i < batches.length; i += concurrency) {
232 const chunk = batches.slice(i, i + concurrency);
233 const chunkStart = i + 1;
234 const chunkEnd = Math.min(i + concurrency, totalBatches);
235
236 log.info(`Processing batches ${chunkStart}-${chunkEnd}/${totalBatches}`);
237
238 const chunkResults = await Promise.allSettled(
239 chunk.map((batch) => normalizeBatch(openai, model, systemPrompt, batch.items)),
240 );
241
242
243 chunk.forEach((batch, idx) => {
244 const settled = chunkResults[idx];
245 if (settled.status === 'fulfilled') {
246 settled.value.forEach((result, resultIdx) => {
247 results[batch.startIndex + resultIdx] = result;
248 });
249 } else {
250 log.error(`Batch ${i + idx + 1} failed: ${settled.reason?.message}`);
251 batch.items.forEach((item, resultIdx) => {
252 results[batch.startIndex + resultIdx] = {
253 output: item.preProcessed || item.original,
254 confidence: 0,
255 error: settled.reason?.message || 'Batch failed',
256 };
257 });
258 }
259 });
260 }
261
262 return results;
263 },
264 };
265}
266
267
268
269
270
271
272function sleep(ms) {
273 return new Promise((resolve) => {
274 setTimeout(resolve, ms);
275 });
276}
277
278
279
280
281
282
283function isRetryableError(error) {
284
285 if (error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT' || error.code === 'ENOTFOUND') {
286 return true;
287 }
288
289 if (error.status === 429 || error.status >= 500) {
290 return true;
291 }
292
293 if (error.message?.includes('timeout') || error.message?.includes('network')) {
294 return true;
295 }
296 return false;
297}
298
299
300
301
302
303
304
305
306
307
308async function normalizeBatch(openai, model, systemPrompt, items) {
309 const prompt = buildPrompt(items);
310 let lastError;
311
312 for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
313 try {
314 const completion = await openai.chat.completions.create({
315 model,
316 messages: [
317 { role: 'system', content: systemPrompt },
318 { role: 'user', content: prompt },
319 ],
320 temperature: 0.3,
321 max_tokens: MAX_TOKENS,
322 timeout: 60000,
323 });
324
325 const responseContent = completion.choices[0]?.message?.content;
326 if (!responseContent) {
327 throw new Error('Empty response from LLM');
328 }
329
330 return parseResponse(responseContent, items);
331 } catch (error) {
332 lastError = error;
333
334 if (attempt < MAX_RETRIES && isRetryableError(error)) {
335 const delayMs = INITIAL_RETRY_DELAY_MS * 2 ** (attempt - 1);
336 log.warning(`LLM request failed (attempt ${attempt}/${MAX_RETRIES}), retrying in ${delayMs}ms...`, {
337 error: error.message,
338 });
339 await sleep(delayMs);
340 } else if (attempt < MAX_RETRIES) {
341
342 log.error('LLM request failed with non-retryable error', { error: error.message });
343 break;
344 }
345 }
346 }
347
348
349
350
351
352 const errorMessage = lastError?.message || 'Unknown error';
353 log.error(`LLM request failed after all retries, returning pass-through for ${items.length} items`, { error: errorMessage });
354 return items.map((item) => ({
355 ...item,
356 normalized: postProcessProductName(item.preProcessed),
357 confidence: 0,
358 error: errorMessage,
359 }));
360}
361
362
363
364
365
366
367function buildPrompt(items) {
368 const itemList = items.map((item, i) => `${i}. "${item.preProcessed}"`).join('\n');
369
370 return `Normalize the following ${items.length} product names:\n\n${itemList}`;
371}
372
373
374
375
376
377
378
379function repairTruncatedJson(jsonStr) {
380
381 try {
382 JSON.parse(jsonStr);
383 return jsonStr;
384 } catch {
385
386 }
387
388
389 if (!jsonStr.trimStart().startsWith('[')) {
390 return jsonStr;
391 }
392
393
394
395 let lastValidEnd = -1;
396
397
398 const lastCommaObj = jsonStr.lastIndexOf('},');
399 if (lastCommaObj > 0) {
400 lastValidEnd = lastCommaObj + 1;
401 }
402
403
404 const lastBrace = jsonStr.lastIndexOf('}');
405 if (lastBrace > lastValidEnd) {
406
407 const afterBrace = jsonStr.slice(lastBrace + 1).trim();
408 if (afterBrace === '' || afterBrace === ']' || afterBrace.startsWith(',') || afterBrace.startsWith(']')) {
409 lastValidEnd = lastBrace;
410 }
411 }
412
413 if (lastValidEnd > 0) {
414 const repaired = `${jsonStr.slice(0, lastValidEnd + 1)}]`;
415 try {
416 JSON.parse(repaired);
417 log.debug('Repaired truncated JSON response', {
418 originalLength: jsonStr.length,
419 repairedLength: repaired.length,
420 });
421 return repaired;
422 } catch {
423
424 }
425 }
426
427 return jsonStr;
428}
429
430
431
432
433
434
435
436function parseResponse(content, originalItems) {
437 try {
438
439 let jsonStr = content.trim();
440 if (jsonStr.includes('```')) {
441 jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```/g, '').trim();
442 }
443
444
445 jsonStr = jsonStr.replace(/\/\/[^\n]*/g, '');
446 jsonStr = jsonStr.replace(/\/\*[\s\S]*?\*\//g, '');
447
448
449 jsonStr = jsonStr.replace(
450 /\{"(\d+)",\s*"([^"]*)",\s*([\d.]+)\}/g,
451 '{"index": $1, "normalized": "$2", "confidence": $3}',
452 );
453
454
455 jsonStr = repairTruncatedJson(jsonStr);
456
457 const parsed = JSON.parse(jsonStr);
458
459 if (!Array.isArray(parsed)) {
460 throw new Error('Response is not an array');
461 }
462
463
464 return originalItems.map((item, i) => {
465 const result = parsed.find((r) => r.index === i);
466 if (result) {
467 return {
468 ...item,
469 normalized: postProcessProductName(result.normalized || item.preProcessed),
470 confidence: clampConfidence(result.confidence),
471 reasoning: result.reasoning || '',
472 };
473 }
474
475 return {
476 ...item,
477 normalized: postProcessProductName(item.preProcessed),
478 confidence: 0.5,
479 reasoning: 'Item not found in LLM response',
480 };
481 });
482 } catch (error) {
483 log.warning('Failed to parse LLM response', { error: error.message, content: content.slice(0, 200) });
484
485
486 return originalItems.map((item) => ({
487 ...item,
488 normalized: postProcessProductName(item.preProcessed),
489 confidence: 0.4,
490 reasoning: `Failed to parse LLM response: ${error.message}`,
491 }));
492 }
493}
494
495
496
497
498
499
500function clampConfidence(value) {
501 if (typeof value !== 'number' || Number.isNaN(value)) {
502 return 0.5;
503 }
504 return Math.max(0, Math.min(1, value));
505}
506
507
508
509
510
511
512function postProcessProductName(text) {
513 if (!text || typeof text !== 'string') {
514 return text;
515 }
516
517 let result = text.trim();
518
519
520 result = result.replace(/(\w)'S\b/g, "$1's");
521
522
523 for (const [lower, proper] of Object.entries(BRAND_CAPS)) {
524 const pattern = new RegExp(`\\b${escapeRegexChars(lower)}\\b`, 'gi');
525 result = result.replace(pattern, proper);
526 }
527
528
529 result = result.replace(/\s+/g, ' ').trim();
530
531 return result;
532}
533
534
535
536
537
538
539function escapeRegexChars(str) {
540 return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
541}
