1
2
3
4
5
6import { log } from 'apify';
7import OpenAI from 'openai';
8
9const BATCH_SIZE = 10;
10const MAX_RETRIES = 3;
11const INITIAL_RETRY_DELAY_MS = 1000;
12const DEFAULT_CONCURRENCY = 5;
13const LARGE_REQUEST_THRESHOLD = 100;
14const LARGE_REQUEST_CONCURRENCY = 10;
15const MAX_TOKENS = 2048;
16
17
18const OPENROUTER_BASE_URL = 'https://openrouter.apify.actor/api/v1';
19
20const OPENROUTER_DIRECT_URL = 'https://openrouter.ai/api/v1';
21
22
23
24
25const STYLE_MODIFIERS = {
26 casual: `Simplify to the shortest RECOGNIZABLE form - the name must still be identifiable:
27 - "Farquhar & Gill Insurance Group" -> "Farquhar & Gill" (distinctive name, drop generic descriptor)
28 - "Samsung Electronics Co., Ltd." -> "Samsung" (globally known brand)
29 - For globally recognized brands, use common abbreviations:
30 * "Johnson & Johnson" -> "J&J"
31 * "Procter & Gamble" -> "P&G"
32 * "Hewlett-Packard" -> "HP"
33 * "International Business Machines" -> "IBM"
34 - When input contains a parenthetical abbreviation, USE IT:
35 * "U.S. Department of Energy (DOE)" -> "DOE"
36 * "Federal Bureau of Investigation (FBI)" -> "FBI"
37 * "World Health Organization (WHO)" -> "WHO"
38 - Strip country prefixes from government agencies:
39 * "U.S. Department of Energy" -> "DOE" (if known) or "Department of Energy"
40 - Remove industry descriptors ONLY when the core name is distinctive: Insurance, Solutions, Technologies, Industries, Group, Holdings
41 * "Decision Resources Group" -> "Decision Resources"
42 * "Farquhar & Gill Insurance Group" -> "Farquhar & Gill"
43 * "Acme Holdings Group" -> "Acme" (strip ALL generic descriptors)
44 * "Smith Industries Group" -> "Smith" (strip Industries AND Group)
45 * "CRC Group" -> "CRC" (strip Group even from acronyms)
46 - Strip parent company annotations like "part of X", "a X company", "an X brand":
47 * "Decision Resources Group, part of Clarivate" -> "Decision Resources"
48 - Strip divisions and subsidiaries to the recognizable PARENT brand:
49 * "J.P. Morgan Asset Management" -> "J.P. Morgan"
50 * "AT&T Cybersecurity" -> "AT&T"
51 * "Berkshire Hathaway HomeServices" -> "Berkshire Hathaway"
52 * "Google Cloud" -> "Google"
53 - BUT keep descriptors if removing them leaves just a person's name, an acronym, or a single vague word:
54 * "Flame Steakhouse" stays "Flame Steakhouse" (just "Flame" is too vague)
55 * "VRCC Veterinary Specialty and Emergency Hospital" -> keep full name (just "VRCC" is meaningless)
56 * "Crystal Dental" stays "Crystal Dental" (just "Crystal" loses the business type)
57 * "Fred's Wrigleyville Garage & Auto Repair" stays as-is (service type is the identity)
58 * "Cory Fein Law Firm" stays "Cory Fein Law Firm" (just "Cory Fein" is a person, not a business)
59 * "Open Hair Miami Salon" stays "Open Hair Miami Salon"
60 * "RKS Photography" stays "RKS Photography" (initials alone are meaningless)
61 - LOCATIONS: Always strip city/neighborhood names and "at/in [Location]" after the business name:
62 * "Publix Super Market at Gateway Village" -> "Publix Super Market"
63 * "exhale Spa Midtown Atlanta" -> "exhale Spa"
64 * "Onelife Fitness - Olney" -> "Onelife Fitness"
65 * "MHC Kenworth - Memphis" -> "MHC Kenworth"
66 - COLON-SEPARATED NAMES: Keep both parts (the subtitle is the identity):
67 * "Replenish: The Spa Co-op" -> "Replenish: The Spa Co-op"
68 - ALWAYS keep "Association", "Foundation", "Trust", "Society" for non-profits (they define the org type)
69 - UNIVERSITIES/COLLEGES: Keep recognizable full names, do NOT use nicknames:
70 * "California State University-Sacramento" -> "CSU Sacramento" or "Cal State Sacramento" (NOT "Sacramento State")
71 * "University of California, Los Angeles" -> "UCLA"
72 * "Massachusetts Institute of Technology" -> "MIT"
73 * Only use widely-known acronyms (UCLA, MIT, USC) - otherwise keep the full name
74 - DOMAIN NAMES: Strip ALL domain TLDs (.com, .io, .ai, .org, .net, etc.):
75 * "acquire.com" -> "Acquire"
76 * "Covidence.org" -> "Covidence"
77 * "scale.ai" -> "Scale"
78 - Remove ALL legal suffixes (Inc., LLC, Ltd., Corp., GmbH, AG, S.A., etc.)`,
79 formal: `Keep the FULL company name but remove legal entity suffixes:
80 - "Farquhar & Gill Insurance Group, LLC" -> "Farquhar & Gill Insurance Group"
81 - "Samsung Electronics Co., Ltd." -> "Samsung Electronics"
82 - "The Coca-Cola Company" -> "The Coca-Cola Company" (keep "The" if part of official name)
83 - "The Walt Disney Company" -> "The Walt Disney Company"
84 - Remove ONLY legal suffixes: Inc., LLC, Ltd., Corp., GmbH, AG, S.A., Co., P.L.C., N.V., etc.
85 - Strip country prefixes from government agencies but keep full name:
86 * "U.S. Department of Energy (DOE)" -> "Department of Energy"
87 * "U.S. Securities and Exchange Commission" -> "Securities and Exchange Commission"
88 - Strip parenthetical abbreviations (they're annotations, not part of the name)
89 - Keep ALL descriptive words: Group, Holdings, Industries, International, etc.
90 - DOMAIN NAMES: Strip ALL domain TLDs (.com, .io, .ai, .org, .net, etc.):
91 * "acquire.com" -> "Acquire"
92 * "Covidence.org" -> "Covidence"`,
93 legal: `Keep the COMPLETE legal company name with EXACT formatting as provided:
94 - "Farquhar & Gill Insurance Group, LLC" -> "Farquhar & Gill Insurance Group, LLC"
95 - "Samsung Electronics Co., Ltd." -> "Samsung Electronics Co., Ltd."
96 - "Apple Inc." -> "Apple Inc."
97 - "ACME CORPORATION, INC." -> "Acme Corporation, Inc." (normalize capitalization only)
98 - "GOOGLE LLC" -> "Google LLC" (keep LLC exactly - do NOT change to Inc.)
99
100 CRITICAL - PRESERVE EXACT FORMATTING:
101 - NEVER add punctuation: "Kforce Inc" stays "Kforce Inc" (do NOT add period to make "Kforce Inc.")
102 - NEVER change abbreviation style: "US Federal Government" stays "US Federal Government" (do NOT change to "U.S.")
103 - NEVER change the legal entity type (LLC, Inc., Corp., Ltd. are NOT interchangeable!)
104 - The ONLY change allowed is normalizing capitalization (all-caps to title case)
105
106 DASH-SEPARATED NAMES (ACRONYM - Full Name):
107 - Keep the FULL form, not the acronym: "IFC - International Finance Corporation" -> "International Finance Corporation"
108 - Be CONSISTENT: always choose the longer descriptive form for legal documents
109 - "NASA - National Aeronautics and Space Administration" -> "National Aeronautics and Space Administration"
110 - "WHO - World Health Organization" -> "World Health Organization"
111
112 PARENTHETICALS:
113 - For government agencies, strip the parenthetical: "U.S. Department of Energy (DOE)" -> "U.S. Department of Energy"
114 - For companies, KEEP the parenthetical: "Amazon Web Services (AWS)" -> "Amazon Web Services (AWS)"
115
116 NEVER ADD anything not in the input:
117 - "Google" -> "Google" (do NOT add "LLC")
118 - "Apple" -> "Apple" (do NOT add "Inc.")
119 - "Microsoft" -> "Microsoft" (do NOT add "Corporation")
120
121 DOMAIN NAMES: Keep TLDs in legal mode (they may be part of the legal name):
122 - "acquire.com" -> "Acquire.com"
123 - "Scale.ai" -> "Scale.ai"`,
124};
125
126
127
128
129const STYLE_GOALS = {
130 casual: 'Short, punchy names that sound natural in "Hi Sarah, I noticed [COMPANY] is hiring..."',
131 formal: 'Complete, professional company names suitable for CRM records and business correspondence.',
132 legal: 'Full legal company names suitable for contracts, invoices, and official documents.',
133};
134
135
136
137
138const STYLE_TESTS = {
139 casual: 'Hi Sarah, I noticed [your answer] is hiring...',
140 formal: 'Account: [your answer]',
141 legal: 'Invoice to: [your answer]',
142};
143
144
145
146
147
148
149function buildSystemPrompt(style) {
150 const effectiveStyle = STYLE_MODIFIERS[style] ? style : 'casual';
151 const isLegal = effectiveStyle === 'legal';
152 const isCasual = effectiveStyle === 'casual';
153
154
155 let contextDescription;
156 if (isCasual) {
157 contextDescription = 'cold email personalization';
158 } else if (isLegal) {
159 contextDescription = 'legal documents';
160 } else {
161 contextDescription = 'CRM and business records';
162 }
163
164 return `You are normalizing company names for ${contextDescription}.
165
166GOAL: ${STYLE_GOALS[effectiveStyle]}
167
168RULES:
1691. PRESERVE original capitalization pattern for brand names:
170 - All-caps brands stay all-caps: "3CD" stays "3CD", "NVIDIA" stays "NVIDIA", "BMW" stays "BMW", "IBM" stays "IBM"
171 - Alphanumeric codes: "3CD" stays "3CD" (NOT "3cd"), "H&M" stays "H&M"
172 - Mixed case brands: "eBay", "FedEx", "McKinsey", "SoCalGas"
173 - Possessives: "Stone's" (NOT "Stone'S"), "McDonald's", "Lowe's"
174 - ${isLegal ? 'In LEGAL mode, normalize all-caps to title case: "ACME CORPORATION" -> "Acme Corporation"' : 'Unknown all-caps brands should be preserved: "GLOBO" stays "GLOBO", "TBC" stays "TBC"'}
1752. Keep ampersands if part of brand name: "Johnson & Johnson", "AT&T", "H&M"
1763. NEVER expand acronyms or guess full company names - only normalize what's given:
177 - "LSEG" stays "LSEG" (do NOT expand to "London Stock Exchange Group")
178 - "IBM" stays "IBM" (do NOT expand to "International Business Machines")
179 - "BMW" stays "BMW" (do NOT expand to "Bayerische Motoren Werke")
180 - NEVER add legal suffixes (Inc., LLC, plc) that weren't in the original input
1814. ${isLegal ? 'KEEP all legal suffixes EXACTLY as written - do NOT add or change punctuation: "Inc" stays "Inc" (not "Inc."), "LLC" stays "LLC"' : 'Remove legal suffixes (Inc., LLC, Ltd., Corp., GmbH, AG, S.A., etc.)'}
182${isLegal ? `4a. LEGAL MODE - PRESERVE EXACT FORMATTING:
183 - NEVER add periods: "Kforce Inc" -> "Kforce Inc" (NOT "Kforce Inc.")
184 - NEVER change abbreviation style: "US" stays "US" (NOT "U.S."), "UK" stays "UK" (NOT "U.K.")
185 - ONLY normalize capitalization (all-caps to title case), nothing else
186` : ''}5. Handle "The" carefully:
187${isCasual ? ` - Keep "The" ONLY when the brand is commonly known WITH "The": "The Home Depot", "The New York Times"
188 - Remove "The" when it's just a formal prefix and the brand works without it:
189 * "THE COCA-COLA COMPANY" -> "Coca-Cola" (brand is "Coca-Cola", not "The Coca-Cola")
190 * "The Walt Disney Company" -> "Disney" (brand is "Disney")
191 * "The Boeing Company" -> "Boeing" (brand is "Boeing")
192 - Test: Would you say "I work at The X" or just "I work at X"? If the latter, drop "The"` : ` - Keep "The" when it's part of the official company name: "The Coca-Cola Company", "The Walt Disney Company"
193 - Only remove "The" if it's clearly not part of the official name`}
1946. ${STYLE_MODIFIERS[effectiveStyle]}
195
196TEST: Before returning, mentally check: "${STYLE_TESTS[effectiveStyle]}" - does it look right?
197
198CONFIDENCE SCORING (be critical, don't default to high scores):
199- 0.95-1.0: Minimal changes, high certainty (fixed capitalization, removed obvious suffix)
200- 0.8-0.9: Standard normalization with clear result
201- 0.6-0.75: Significant transformation OR institutional names (churches, universities, government) that may have preferred forms
202- 0.4-0.55: Ambiguous input, multiple valid interpretations, or controversial shortening
203- Below 0.4: Very uncertain, likely needs human review
204
205OUTPUT FORMAT (strict JSON, no comments):
206[{"index": 0, "normalized": "Example Corp", "confidence": 0.95}, {"index": 1, "normalized": "Another Inc", "confidence": 0.9}]
207Return ONLY the JSON array with objects containing "index", "normalized", and "confidence" keys.`;
208}
209
210
211
212
213
214
215
216
217
218export function createNormalizer(model, style = 'casual', openRouterApiKey = null) {
219 let openai;
220
221 if (openRouterApiKey) {
222 log.info('Using direct OpenRouter API (bring your own key)');
223 openai = new OpenAI({
224 baseURL: OPENROUTER_DIRECT_URL,
225 apiKey: openRouterApiKey,
226 timeout: 120_000,
227 });
228 } else {
229 const apifyToken = process.env.APIFY_TOKEN;
230 if (!apifyToken) {
231 throw new Error('APIFY_TOKEN environment variable is required for OpenRouter access');
232 }
233 openai = new OpenAI({
234 baseURL: OPENROUTER_BASE_URL,
235 apiKey: 'apify-openrouter',
236 timeout: 120_000,
237 defaultHeaders: {
238 Authorization: `Bearer ${apifyToken}`,
239 },
240 });
241 }
242
243 const systemPrompt = buildSystemPrompt(style);
244
245 return {
246
247
248
249
250
251
252 async normalize(items) {
253
254 const batches = [];
255 for (let i = 0; i < items.length; i += BATCH_SIZE) {
256 batches.push({
257 items: items.slice(i, i + BATCH_SIZE),
258 startIndex: i,
259 });
260 }
261
262 const totalBatches = batches.length;
263
264
265 const concurrency = items.length >= LARGE_REQUEST_THRESHOLD
266 ? LARGE_REQUEST_CONCURRENCY
267 : DEFAULT_CONCURRENCY;
268
269 log.info(`Processing ${items.length} items in ${totalBatches} batches (concurrency: ${concurrency})`);
270
271
272 const results = new Array(items.length);
273 for (let i = 0; i < batches.length; i += concurrency) {
274 const chunk = batches.slice(i, i + concurrency);
275 const chunkStart = i + 1;
276 const chunkEnd = Math.min(i + concurrency, totalBatches);
277
278 log.info(`Processing batches ${chunkStart}-${chunkEnd}/${totalBatches}`);
279
280 const chunkResults = await Promise.allSettled(
281 chunk.map((batch) => normalizeBatch(openai, model, systemPrompt, batch.items)),
282 );
283
284
285 chunk.forEach((batch, idx) => {
286 const settled = chunkResults[idx];
287 if (settled.status === 'fulfilled') {
288 settled.value.forEach((result, resultIdx) => {
289 results[batch.startIndex + resultIdx] = result;
290 });
291 } else {
292 log.error(`Batch ${i + idx + 1} failed: ${settled.reason?.message}`);
293 batch.items.forEach((item, resultIdx) => {
294 results[batch.startIndex + resultIdx] = {
295 output: item.preProcessed || item.original,
296 confidence: 0,
297 error: settled.reason?.message || 'Batch failed',
298 };
299 });
300 }
301 });
302 }
303
304 return results;
305 },
306 };
307}
308
309
310
311
312
313
314function sleep(ms) {
315 return new Promise((resolve) => {
316 setTimeout(resolve, ms);
317 });
318}
319
320
321
322
323
324
325function isRetryableError(error) {
326
327 if (error.code === 'ECONNRESET' || error.code === 'ETIMEDOUT' || error.code === 'ENOTFOUND') {
328 return true;
329 }
330
331 if (error.status === 429 || error.status >= 500) {
332 return true;
333 }
334
335 if (error.message?.includes('timeout') || error.message?.includes('network')) {
336 return true;
337 }
338 return false;
339}
340
341
342
343
344
345
346
347
348
349
350async function normalizeBatch(openai, model, systemPrompt, items) {
351 const prompt = buildPrompt(items);
352 let lastError;
353
354 for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {
355 try {
356 const completion = await openai.chat.completions.create({
357 model,
358 messages: [
359 { role: 'system', content: systemPrompt },
360 { role: 'user', content: prompt },
361 ],
362 temperature: 0.3,
363 max_tokens: MAX_TOKENS,
364 timeout: 60000,
365 });
366
367 const responseContent = completion.choices[0]?.message?.content;
368 if (!responseContent) {
369 throw new Error('Empty response from LLM');
370 }
371
372 return parseResponse(responseContent, items);
373 } catch (error) {
374 lastError = error;
375
376 if (attempt < MAX_RETRIES && isRetryableError(error)) {
377 const delayMs = INITIAL_RETRY_DELAY_MS * (2 ** (attempt - 1));
378 log.warning(`LLM request failed (attempt ${attempt}/${MAX_RETRIES}), retrying in ${delayMs}ms...`, {
379 error: error.message,
380 });
381 await sleep(delayMs);
382 } else if (attempt < MAX_RETRIES) {
383
384 log.error('LLM request failed with non-retryable error', { error: error.message });
385 break;
386 }
387 }
388 }
389
390
391
392 const errorMessage = lastError?.message || 'Unknown error';
393 log.error(`LLM request failed after all retries, returning pass-through for ${items.length} items`, { error: errorMessage });
394 return items.map((item) => ({
395 output: item.preProcessed || item.original,
396 confidence: 0,
397 error: errorMessage,
398 }));
399}
400
401
402
403
404
405
406function buildPrompt(items) {
407 const itemList = items.map((item, i) => {
408
409 if (item.input !== item.preProcessed) {
410 return `${i}. "${item.input}" (preprocessed: "${item.preProcessed}")`;
411 }
412 return `${i}. "${item.preProcessed}"`;
413 }).join('\n');
414
415 return `Normalize the following ${items.length} company names.
416
417Note: Some items show "(preprocessed: ...)" — normalize from the PREPROCESSED form, not the original. Legal suffixes (LLC, PLLC, P.C., Inc., etc.) have already been stripped per style rules. Do NOT re-add them. The original is shown only for context.
418
419${itemList}`;
420}
421
422
423
424
425
426
427
428function repairTruncatedJson(jsonStr) {
429
430 try {
431 JSON.parse(jsonStr);
432 return jsonStr;
433 } catch {
434
435 }
436
437
438 if (!jsonStr.trimStart().startsWith('[')) {
439 return jsonStr;
440 }
441
442
443
444 let lastValidEnd = -1;
445
446
447 const lastCommaObj = jsonStr.lastIndexOf('},');
448 if (lastCommaObj > 0) {
449 lastValidEnd = lastCommaObj + 1;
450 }
451
452
453 const lastBrace = jsonStr.lastIndexOf('}');
454 if (lastBrace > lastValidEnd) {
455
456 const afterBrace = jsonStr.slice(lastBrace + 1).trim();
457 if (afterBrace === '' || afterBrace === ']' || afterBrace.startsWith(',') || afterBrace.startsWith(']')) {
458 lastValidEnd = lastBrace;
459 }
460 }
461
462 if (lastValidEnd > 0) {
463 const repaired = `${jsonStr.slice(0, lastValidEnd + 1)}]`;
464 try {
465 JSON.parse(repaired);
466 log.debug('Repaired truncated JSON response', {
467 originalLength: jsonStr.length,
468 repairedLength: repaired.length,
469 });
470 return repaired;
471 } catch {
472
473 }
474 }
475
476 return jsonStr;
477}
478
479
480
481
482
483
484
485function parseResponse(content, originalItems) {
486 try {
487
488 let jsonStr = content.trim();
489 if (jsonStr.includes('```')) {
490 jsonStr = jsonStr.replace(/```json?\n?/g, '').replace(/```/g, '').trim();
491 }
492
493
494 jsonStr = jsonStr.replace(/\/\/[^\n]*/g, '');
495 jsonStr = jsonStr.replace(/\/\*[\s\S]*?\*\//g, '');
496
497
498 jsonStr = jsonStr.replace(
499 /\{"(\d+)",\s*"([^"]*)",\s*([\d.]+)\}/g,
500 '{"index": $1, "normalized": "$2", "confidence": $3}',
501 );
502
503
504 jsonStr = repairTruncatedJson(jsonStr);
505
506 const parsed = JSON.parse(jsonStr);
507
508 if (!Array.isArray(parsed)) {
509 throw new Error('Response is not an array');
510 }
511
512
513 return originalItems.map((item, i) => {
514 const result = parsed.find((r) => r.index === i);
515 if (result) {
516 return {
517 ...item,
518 normalized: postProcessCapitalization(result.normalized || item.preProcessed, item.original),
519 confidence: clampConfidence(result.confidence),
520 reasoning: result.reasoning || '',
521 };
522 }
523
524 return {
525 ...item,
526 normalized: postProcessCapitalization(item.preProcessed, item.original),
527 confidence: 0.5,
528 reasoning: 'Item not found in LLM response',
529 };
530 });
531 } catch (error) {
532 log.warning('Failed to parse LLM response', { error: error.message, content: content.slice(0, 200) });
533
534
535 return originalItems.map((item) => ({
536 ...item,
537 normalized: postProcessCapitalization(item.preProcessed, item.original),
538 confidence: 0.4,
539 reasoning: `Failed to parse LLM response: ${error.message}`,
540 }));
541 }
542}
543
544
545
546
547
548
549function clampConfidence(value) {
550 if (typeof value !== 'number' || Number.isNaN(value)) {
551 return 0.5;
552 }
553 return Math.max(0, Math.min(1, value));
554}
555
556
557
558
559
560
561
562function postProcessCapitalization(text, original = '') {
563 if (!text || typeof text !== 'string') {
564 return text;
565 }
566
567 let result = text;
568
569
570
571 result = result.replace(/(\w)'S\b/g, "$1's");
572
573
574
575
576 if (original) {
577
578 const alphanumericCodes = original.match(/\b(?=[A-Z0-9&]*[0-9])[A-Z0-9&]{2,5}\b/g) || [];
579
580 for (const code of alphanumericCodes) {
581 const lowerCode = code.toLowerCase();
582 const codePattern = new RegExp(`\\b${escapeRegexChars(lowerCode)}\\b`, 'gi');
583
584
585 if (original.includes(code)) {
586 result = result.replace(codePattern, code);
587 }
588 }
589 }
590
591 return result;
592}
593
594
595
596
597
598
599function escapeRegexChars(str) {
600 return str.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
601}