1import { Actor, log } from 'apify';
2import { gotScraping } from 'got-scraping';
3import { parseQuery, flatten, parentSummary, nameScore, nameVariants } from './lei.js';
4
5const EVENT = 'lei-record';
6const API = 'https://api.gleif.org/api/v1';
7const Q_FIELDS = ['lei', 'LEI', 'isin', 'bic', 'companyName', 'name', 'company', 'legalName'];
8const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
9const getPath = (o, path) => path.split('.').reduce((v, k) => (v && typeof v === 'object' ? v[k] : undefined), o);
10class InputError extends Error {}
11
12await Actor.init();
13try {
14 const input = (await Actor.getInput()) ?? {};
15 const { defaultCountry = '', includeParents = true, includeLegalFormName = true, onlyActive = false } = input;
16 const maxMatches = Math.max(1, Math.min(10, Math.floor(Number(input.maxMatchesPerName) || 1)));
17 const minScore = Math.max(0, Math.min(100, Number.isFinite(Number(input.minNameScore)) ? Number(input.minNameScore) : 80));
18 const raw = input.queries ?? input.leis ?? input.companies ?? [];
19 const entries = [];
20 for (const v of Array.isArray(raw) ? raw : String(raw).split(/\n+/)) { const s = typeof v === 'string' ? v.trim() : String(v?.lei ?? v?.name ?? '').trim(); if (s) entries.push(s); }
21 const dsId = String(input.datasetId || input?.payload?.resource?.defaultDatasetId || input?.resource?.defaultDatasetId || '').trim();
22 if (dsId) {
23 const max = Math.max(1, Math.floor(Number(input.datasetMaxItems) || 1000));
24 let qf = String(input.queryField || '').trim() || null; let cf = String(input.countryField || '').trim() || null; let offset = 0; let got = 0;
25 while (got < max) {
26 let page;
27 try { page = await Actor.newClient().dataset(dsId).listItems({ offset, limit: 1000, clean: true }); } catch (e) { throw new InputError(`Could not read dataset ${dsId}: ${e.message}. Use the dataset ID of a run you can access.`); }
28 const items = page.items ?? []; if (!items.length) break;
29 qf ||= Q_FIELDS.find((f) => items.some((it) => typeof getPath(it, f) === 'string' && getPath(it, f).trim())) ?? null;
30 if (!qf) break;
31 for (const it of items) {
32 const val = String(getPath(it, qf) ?? '').trim(); if (!val) continue;
33 const c = cf ? String(getPath(it, cf) ?? '').trim() : '';
34 entries.push(c && /^[A-Za-z]{2}$/.test(c) ? `${val} | ${c}` : val); got++; if (got >= max) break;
35 }
36 offset += items.length; if (items.length < 1000) break;
37 }
38 log.info(`Read ${got} values from dataset ${dsId} (field "${qf ?? 'none found'}").`);
39 if (!got && !entries.length) throw new InputError(`Dataset ${dsId} has no usable values. Set "queryField" (tried: ${Q_FIELDS.join(', ')}).`);
40 }
41 if (!entries.length) throw new InputError('Add at least one LEI, company name, ISIN or BIC, e.g. "HWUPKR0MPOU8FGXBT394" or "Siemens Aktiengesellschaft | DE", or pass a datasetId.');
42
43 const capRaw = input.maxQueries ?? input.maxItems ?? input.limit ?? input.maxResults;
44 let cap = Number.isFinite(Number(capRaw)) && Number(capRaw) > 0 ? Math.floor(Number(capRaw)) : 2000;
45 let paidCap = Number(process.env.ACTOR_MAX_PAID_DATASET_ITEMS); if (!Number.isFinite(paidCap) || paidCap <= 0) paidCap = Infinity;
46
47
48 let lastAt = 0; const stats = { requests: 0, retries: 0, throttled: 0 };
49 async function get(path, { allow404 = false } = {}) {
50 let last = null;
51 for (let a = 0; a < 4; a++) {
52 const wait = lastAt + 1050 - Date.now(); if (wait > 0) await sleep(wait); lastAt = Date.now();
53 try {
54 stats.requests++;
55 const res = await gotScraping({ url: API + path, responseType: 'json', headers: { accept: 'application/vnd.api+json' }, timeout: { request: 30000 }, throwHttpErrors: false, retry: { limit: 0 } });
56 if (res.statusCode === 200) return { body: res.body };
57 if (res.statusCode === 404 && allow404) return { body: null };
58 if (res.statusCode === 400 || res.statusCode === 422) return { error: `HTTP ${res.statusCode}`, fatal: true };
59 if (res.statusCode === 429) { stats.throttled++; await sleep(15000 * (a + 1)); }
60 last = `HTTP ${res.statusCode}`;
61 } catch (e) { last = `TIMEOUT/${e.code ?? e.message}`; }
62 stats.retries++; await sleep(Math.min(20000, 2000 * 2 ** a));
63 }
64 return { error: last };
65 }
66 const enc = encodeURIComponent;
67
68 const seen = new Set(); const queries = []; const localRows = []; let duplicates = 0;
69 for (const e of entries) {
70 const p = parseQuery(e, defaultCountry);
71 if (p.status) { localRows.push({ input: e.slice(0, 200), queryType: p.type ?? null, query: p.value ?? null, status: p.status, error: p.error }); continue; }
72 const key = `${p.type}:${p.value.toLowerCase()}:${p.country ?? ''}`;
73 if (seen.has(key)) { duplicates++; continue; }
74 seen.add(key); queries.push({ input: e.slice(0, 200), ...p });
75 }
76 const overCap = queries.length > cap ? queries.splice(cap) : [];
77 if (overCap.length) log.warning(`maxQueries is ${cap}: ${overCap.length} extra queries were not run.`);
78
79 const charging = Actor.getChargingManager();
80 const counts = {}; let chargeLimitHit = false; let charged = 0; const skipped = [];
81 const canCharge = () => {
82 if (chargeLimitHit || charged >= paidCap) { chargeLimitHit = true; return false; }
83 if (charging) { const allowed = charging.calculateMaxEventChargeCountWithinLimit(EVENT); if (Number.isFinite(allowed) && allowed < 1) { chargeLimitHit = true; log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.'); return false; } }
84 return true;
85 };
86 const formCache = new Map(); const parentCache = new Map();
87 async function legalFormName(code) {
88 if (!code || !includeLegalFormName) return null;
89 if (!formCache.has(code)) {
90 const r = await get(`/entity-legal-forms/${enc(code)}`, { allow404: true });
91 const names = r.body?.data?.attributes?.names ?? [];
92 formCache.set(code, (names.find((n) => n.languageCode === 'en') ?? names[0])?.localName ?? null);
93 }
94 return formCache.get(code);
95 }
96 async function parent(lei, kind) {
97 const k = `${kind}:${lei}`;
98 if (!parentCache.has(k)) { const r = await get(`/lei-records/${lei}/${kind}`, { allow404: true }); parentCache.set(k, r.error ? { error: r.error } : parentSummary(r.body?.data)); }
99 return parentCache.get(k);
100 }
101 async function emit(q, rec, extra = {}) {
102 if (!canCharge()) { skipped.push(q); return false; }
103 const f = flatten(rec);
104 const row = { input: q.input, queryType: q.type, query: q.value, status: 'found', ...extra, ...f };
105 row.legalForm = f.legalForm ?? await legalFormName(f.legalFormCode);
106 if (includeParents) {
107 row.directParent = f.hasDirectParent ? await parent(f.lei, 'direct-parent') : null;
108 row.ultimateParent = f.hasUltimateParent ? await parent(f.lei, 'ultimate-parent') : null;
109 }
110 row.checkedAt = new Date().toISOString();
111 counts.found = (counts.found ?? 0) + 1; charged++;
112 const r = await Actor.pushData(row, EVENT);
113 if (r?.eventChargeLimitReached) chargeLimitHit = true;
114 return true;
115 }
116 async function free(q, status, error) { counts[status] = (counts[status] ?? 0) + 1; await Actor.pushData({ input: q.input, queryType: q.type, query: q.value, status, error, checkedAt: new Date().toISOString() }); }
117 const statusOk = (rec) => !onlyActive || (rec?.attributes?.entity?.status === 'ACTIVE');
118
119
120 const leis = queries.filter((q) => q.type === 'lei');
121 for (let i = 0; i < leis.length; i += 100) {
122 const batch = leis.slice(i, i + 100);
123 if (chargeLimitHit) { skipped.push(...batch); continue; }
124 const r = await get(`/lei-records?filter[lei]=${batch.map((b) => b.value).join(',')}&page[size]=100`);
125 const byId = new Map((r.body?.data ?? []).map((d) => [d.id, d]));
126 for (const q of batch) {
127 if (chargeLimitHit) { skipped.push(q); continue; }
128 const rec = byId.get(q.value);
129 if (rec && statusOk(rec)) await emit(q, rec);
130 else if (rec) await free(q, 'not_found', 'LEI exists but the entity is not ACTIVE (onlyActive is on).');
131 else if (r.error) await free(q, 'service_unavailable', `GLEIF did not answer (${r.error}). Not charged. Run again later.`);
132 else await free(q, 'not_found', 'No LEI record with this code in GLEIF. Not charged.');
133 }
134 log.info(`Looked up ${Math.min(i + 100, leis.length)}/${leis.length} LEIs`);
135 }
136
137 let done = 0; const others = queries.filter((q) => q.type !== 'lei');
138 for (const q of others) {
139 if (chargeLimitHit) { skipped.push(q); continue; }
140 if (q.type === 'isin' || q.type === 'bic') {
141 const r = await get(`/lei-records?filter[${q.type}]=${enc(q.value)}&page[size]=5`);
142 const recs = (r.body?.data ?? []).filter(statusOk);
143 if (recs.length) { for (const rec of recs.slice(0, maxMatches)) await emit(q, rec); } else if (r.error && !r.fatal) await free(q, 'service_unavailable', `GLEIF did not answer (${r.error}). Not charged.`); else await free(q, 'not_found', `No LEI record linked to this ${q.type.toUpperCase()}. Not charged.`);
144 } else {
145 const cf = q.country ? `&filter[entity.legalAddress.country]=${q.country}` : '';
146 let r = await get(`/lei-records?filter[entity.legalName]=${enc(q.value)}${cf}&page[size]=25`);
147 let recs = r.body?.data ?? [];
148 const score = (list) => list.filter(statusOk).map((rec) => {
149 const e = rec.attributes?.entity ?? {};
150 const s = nameScore(q.value, [e.legalName?.name, ...(e.otherNames ?? []).map((n) => n.name), ...(e.transliteratedOtherNames ?? []).map((n) => n.name)]);
151 return { rec, s: s + (e.status === 'ACTIVE' ? 0.5 : 0) + (rec.attributes?.registration?.status === 'ISSUED' ? 0.3 : 0) };
152 }).filter((x) => x.s >= minScore).sort((a, b) => b.s - a.s);
153 let scored = score(recs);
154 if (!r.error && (!scored.length || scored[0].s < 95)) {
155 for (const v of nameVariants(q.value)) {
156 const r2 = await get(`/lei-records?filter[entity.legalName]=${enc(v)}${cf}&page[size]=25`);
157 if (r2.body?.data?.length) { recs = [...recs, ...r2.body.data]; const ids = new Set(); scored = score(recs).filter((x) => !ids.has(x.rec.id) && ids.add(x.rec.id)); }
158 }
159 }
160 if (!r.error && (!scored.length || scored[0].s < 95)) {
161 const r3 = await get(`/lei-records?filter[fulltext]=${enc(q.value)}${cf}&page[size]=25`);
162 if (r3.body?.data?.length) { recs = [...recs, ...r3.body.data]; const ids = new Set(); scored = score(recs).filter((x) => !ids.has(x.rec.id) && ids.add(x.rec.id)); }
163 }
164 scored = scored.slice(0, maxMatches);
165 if (scored.length) { let rank = 0; for (const x of scored) { rank++; if (!(await emit(q, x.rec, { matchScore: Math.floor(x.s), matchRank: rank }))) break; } } else if (r.error && !r.fatal) await free(q, 'service_unavailable', `GLEIF did not answer (${r.error}). Not charged.`);
166 else await free(q, 'not_found', recs.length ? `GLEIF returned ${recs.length} candidates but none scored >= minNameScore ${minScore}. Try the full legal name, add a country ("| DE"), or lower minNameScore. Not charged.` : 'No LEI record matches this name. Try the full legal name or add a country, e.g. "Siemens Aktiengesellschaft | DE". Not charged.');
167 }
168 done++; if (done % 25 === 0) log.info(`Searched ${done}/${others.length} names/ISINs/BICs`);
169 }
170 for (const q of skipped) await Actor.pushData({ input: q.input, queryType: q.type, query: q.value, status: 'skipped_charge_limit', error: 'Not run: the run reached the maximum cost you set. Raise the limit and run these again.', checkedAt: new Date().toISOString() });
171 if (skipped.length) counts.skipped_charge_limit = skipped.length;
172 for (const r of localRows) { counts[r.status] = (counts[r.status] ?? 0) + 1; await Actor.pushData({ ...r, checkedAt: new Date().toISOString() }); }
173
174 const found = counts.found ?? 0; const dsOut = Actor.getEnv().defaultDatasetId;
175 const summary = {
176 queriesRequested: entries.length, uniqueQueries: queries.length, recordsFound: found, byStatus: counts, duplicatesSkipped: duplicates,
177 notRunOverMaxQueries: overCap.length, chargeLimitReached: chargeLimitHit, sourceDatasetId: dsId || null, requests: stats,
178 zeroResultsReason: found ? null : `No LEI record found: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ') || 'no valid input'}. See each row's error.`,
179 finishedAt: new Date().toISOString(),
180 };
181 summary.nextSteps = [
182 'Filter by entityStatus (ACTIVE/INACTIVE) and registrationStatus (ISSUED is current; LAPSED means the LEI was not renewed).',
183 'Name searches return matchScore (0-100); review rows below 90 before relying on them.',
184 ...(counts.not_found ? ['For not_found names, try the full legal name with a country hint, e.g. "Siemens Aktiengesellschaft | DE".'] : []),
185 'Need VAT or EORI checks for EU companies? Use swillz/eu-vat-validator or swillz/eori-validator.',
186 `Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${dsOut}"}`,
187 ];
188 await Actor.setValue('OUTPUT', summary);
189 const freeC = Object.entries(counts).filter(([k]) => k !== 'found');
190 await Actor.exit(`${found ? '' : '0 records: '}Found ${found} LEI records for ${queries.length} queries${freeC.length ? `; ${freeC.map(([k, v]) => `${v} ${k}`).join(', ')} not charged` : ''}.`);
191} catch (e) {
192 if (e instanceof InputError) { log.error(e.message); await Actor.fail(`Invalid input: ${e.message}`); } else { log.exception(e, 'Run failed'); await Actor.fail(e.message); }
193}
