1import { Actor, log } from 'apify';
2import { gotScraping } from 'got-scraping';
3import { parseOfac, parseUn, parseEu, parseUk } from './lists.js';
4import { NameIndex, countryToIso } from './match.js';
5
6const EVENT = 'name-screened';
7const CHARGED = new Set(['clear', 'potential_match']);
8const OFAC = 'https://sanctionslistservice.ofac.treas.gov/api/PublicationPreview/exports/';
9export const SOURCES = {
10 ofac_sdn: { label: 'OFAC SDN List (US Treasury)', urls: ['SDN.CSV', 'ALT.CSV', 'ADD.CSV'].map((f) => OFAC + f) },
11 ofac_consolidated: { label: 'OFAC Consolidated Non-SDN List (US Treasury)', urls: ['CONS_PRIM.CSV', 'CONS_ALT.CSV', 'CONS_ADD.CSV'].map((f) => OFAC + f) },
12 un: { label: 'UN Security Council Consolidated List', urls: ['https://scsanctions.un.org/resources/xml/en/consolidated.xml'] },
13 eu: { label: 'EU Consolidated Financial Sanctions List', urls: ['https://webgate.ec.europa.eu/fsd/fsf/public/files/csvFullSanctionsList_1_1/content?token=dG9rZW4tMjAxNw'] },
14 uk: { label: 'UK Sanctions List (FCDO)', urls: ['https://sanctionslist.fcdo.gov.uk/docs/UK-Sanctions-List.csv'] },
15};
16const SNAPSHOT_URL = 'https://api.apify.com/v2/key-value-stores/M258nvMTE4yWUjoVd/records/SNAPSHOT';
17const NAME_FIELDS = ['name', 'fullName', 'companyName', 'company', 'legalName', 'title', 'personName', 'beneficiary', 'counterparty'];
18const COUNTRY_FIELDS = ['country', 'countryCode', 'nationality', 'location'];
19const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
20const getPath = (o, path) => path.split('.').reduce((v, k) => (v && typeof v === 'object' ? v[k] : undefined), o);
21class InputError extends Error {}
22
23async function fetchText(url, timeoutMs) {
24 let last;
25 for (let a = 0; a < 3; a++) {
26 try {
27 const res = await gotScraping({ url, timeout: { request: timeoutMs }, throwHttpErrors: false, retry: { limit: 0 }, headerGeneratorOptions: { browsers: ['chrome'] }, decompress: true });
28 if (res.statusCode === 200 && res.body?.length > 200) return String(res.body);
29 last = `HTTP ${res.statusCode}`;
30 } catch (e) { last = e.code ?? e.message; }
31 await sleep(2000 * (a + 1));
32 }
33 throw new Error(`${url.split('?')[0]}: ${last}`);
34}
35
36await Actor.init();
37try {
38 const input = (await Actor.getInput()) ?? {};
39 const lists = (Array.isArray(input.lists) && input.lists.length ? input.lists : Object.keys(SOURCES)).map(String);
40 const bad = lists.filter((l) => !SOURCES[l]);
41 if (bad.length) throw new InputError(`Unknown list(s): ${bad.join(', ')}. Use any of: ${Object.keys(SOURCES).join(', ')}.`);
42 const minScore = Math.max(70, Math.min(100, Number(input.minScore) || 85));
43 const maxMatches = Math.max(1, Math.min(25, Math.floor(Number(input.maxMatchesPerName) || 5)));
44 const entityType = ['any', 'individual', 'entity'].includes(input.entityType) ? input.entityType : 'any';
45 const defaultCountry = countryToIso(input.defaultCountry) ?? null;
46
47 const entries = [];
48 const raw = input.names ?? input.queries ?? [];
49 for (const v of Array.isArray(raw) ? raw : String(raw).split(/[\n;]+/)) {
50 if (v && typeof v === 'object') { entries.push({ name: String(v.name ?? v.fullName ?? v.companyName ?? '').trim(), country: countryToIso(v.country ?? v.countryCode) ?? defaultCountry, type: v.type ?? null }); continue; }
51 const s = String(v ?? '').trim(); if (!s) continue;
52 const [n, ...c] = s.split('|');
53 entries.push({ name: n.trim(), country: countryToIso(c.join('|').trim()) ?? defaultCountry, type: null });
54 }
55 const dsId = String(input.datasetId || input?.payload?.resource?.defaultDatasetId || input?.resource?.defaultDatasetId || '').trim();
56 if (dsId) {
57 const max = Math.max(1, Math.floor(Number(input.datasetMaxItems) || 1000));
58 let nf = String(input.nameField || '').trim() || null; let cf = String(input.countryField || '').trim() || null; let offset = 0; let got = 0;
59 while (got < max) {
60 let page;
61 try { page = await Actor.newClient().dataset(dsId).listItems({ offset, limit: 1000, clean: true }); } catch (e) { throw new InputError(`Could not read dataset ${dsId}: ${e.message}. Use the dataset ID of a run you can access.`); }
62 const items = page.items ?? []; if (!items.length) break;
63 nf ||= NAME_FIELDS.find((f) => items.some((it) => typeof getPath(it, f) === 'string' && getPath(it, f).trim())) ?? null;
64 cf ||= COUNTRY_FIELDS.find((f) => items.some((it) => countryToIso(getPath(it, f)))) ?? null;
65 if (!nf) break;
66 for (const it of items) { const n = String(getPath(it, nf) ?? '').trim(); if (n) { entries.push({ name: n, country: (cf && countryToIso(getPath(it, cf))) || defaultCountry, type: null }); got++; } if (got >= max) break; }
67 offset += items.length; if (items.length < 1000) break;
68 }
69 log.info(`Read ${got} names from dataset ${dsId} (field "${nf ?? 'none found'}").`);
70 if (!got && !entries.length) throw new InputError(`Dataset ${dsId} has no name values. Set "nameField" (tried: ${NAME_FIELDS.join(', ')}).`);
71 }
72 if (!entries.length) throw new InputError('Add at least one name to screen, e.g. "Acme Trading LLC" or "John Doe | RU", or pass a datasetId from another Actor run.');
73
74 const capRaw = input.maxNames ?? input.maxItems ?? input.limit;
75 let cap = Number.isFinite(Number(capRaw)) && Number(capRaw) > 0 ? Math.floor(Number(capRaw)) : 10000;
76 const envCap = Number(process.env.ACTOR_MAX_PAID_DATASET_ITEMS);
77 if (Number.isFinite(envCap) && envCap > 0) cap = Math.min(cap, envCap);
78
79
80 const t0 = Date.now(); const listInfo = {}; const records = []; const unavailable = [];
81 const timeout = Math.max(30, Math.min(300, Number(input.downloadTimeoutSecs) || 150)) * 1000;
82 const maxAgeH = Math.max(1, Math.min(168, Number(input.maxListAgeHours) || 24));
83 let snap = null;
84 if (!input.alwaysDownloadFresh) {
85 try {
86 const res = await gotScraping({ url: SNAPSHOT_URL, responseType: 'json', timeout: { request: 60000 }, throwHttpErrors: false, retry: { limit: 1 } });
87 if (res.statusCode === 200 && res.body?.lists) snap = res.body; else log.warning(`List snapshot unavailable (HTTP ${res.statusCode}); downloading the lists directly.`);
88 } catch (e) { log.warning(`List snapshot unavailable (${e.code ?? e.message}); downloading the lists directly.`); }
89 }
90 const live = [];
91 for (const l of lists) {
92 const s = snap?.lists?.[l];
93 if (s?.records?.length && Date.now() - Date.parse(s.fetchedAt) < maxAgeH * 3600e3) {
94 records.push(...s.records); listInfo[l] = { label: SOURCES[l].label, available: true, records: s.records.length, listDate: s.listDate ?? null, fetchedAt: s.fetchedAt, via: 'snapshot' };
95 } else live.push(l);
96 }
97 const texts = await Promise.all(live.map(async (l) => { try { return [l, await Promise.all(SOURCES[l].urls.map((u) => fetchText(u, timeout)))]; } catch (e) { return [l, null, e.message]; } }));
98 const fetchedAt = new Date().toISOString();
99 for (let ti = 0; ti < texts.length; ti++) {
100 const [l, t, err] = texts[ti];
101 if (!t) { unavailable.push(l); listInfo[l] = { label: SOURCES[l].label, available: false, error: String(err).slice(0, 200) }; log.warning(`${SOURCES[l].label} unavailable: ${err}`); continue; }
102 try {
103 let recs; let generated = null;
104 if (l.startsWith('ofac')) recs = parseOfac(l, ...t);
105 else if (l === 'un') ({ records: recs, generated } = parseUn(t[0]));
106 else if (l === 'eu') ({ records: recs, generated } = parseEu(t[0]));
107 else ({ records: recs, generated } = parseUk(t[0]));
108 if (!recs.length) throw new Error('file parsed to 0 records');
109 records.push(...recs); listInfo[l] = { label: SOURCES[l].label, available: true, records: recs.length, listDate: generated, fetchedAt, via: 'official source' };
110 } catch (e) { unavailable.push(l); listInfo[l] = { label: SOURCES[l].label, available: false, error: `Parse failed: ${e.message}`.slice(0, 200) }; }
111 texts[ti] = [l];
112 }
113 const index = new NameIndex();
114 records.forEach((r, i) => { if (entityType === 'any' || (entityType === 'individual') === (r.type === 'individual')) for (const n of r.names) index.add(i, n); });
115 log.info(`Loaded ${records.length} listed records (${index.names.length} names) from ${lists.length - unavailable.length}/${lists.length} lists in ${Math.round((Date.now() - t0) / 1000)}s.`);
116 const listsChecked = lists.filter((l) => !unavailable.includes(l));
117
118 const charging = Actor.getChargingManager();
119 const counts = {}; const skipped = []; let chargeLimitHit = false; const seen = new Set(); let duplicates = 0; let overCap = 0; let n = 0;
120 async function save(row) {
121 if (!CHARGED.has(row.status)) { counts[row.status] = (counts[row.status] ?? 0) + 1; await Actor.pushData(row); return; }
122 if (chargeLimitHit) { skipped.push(row.name); return; }
123 if (charging) {
124 const allowed = charging.calculateMaxEventChargeCountWithinLimit(EVENT);
125 if (Number.isFinite(allowed) && allowed < 1) { chargeLimitHit = true; skipped.push(row.name); log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.'); return; }
126 }
127 counts[row.status] = (counts[row.status] ?? 0) + 1;
128 const r = await Actor.pushData(row, EVENT);
129 if (r?.eventChargeLimitReached) chargeLimitHit = true;
130 }
131 for (const e of entries) {
132 const base = { name: e.name.slice(0, 300), countryHint: e.country, screenedAt: new Date().toISOString() };
133 const key = `${e.name.toLowerCase()}|${e.country ?? ''}`;
134 if (seen.has(key)) { duplicates++; continue; } seen.add(key);
135 if (n >= cap) { overCap++; continue; }
136 if (e.name.replace(/[^\p{L}\p{N}]/gu, '').length < 2) { await save({ ...base, status: 'invalid_input', topScore: null, matchCount: 0, matches: [], error: 'Name is empty or too short to screen. Not charged.' }); continue; }
137 if (!listsChecked.length) { await save({ ...base, status: 'service_unavailable', topScore: null, matchCount: 0, matches: [], listsChecked, listsNotChecked: unavailable, error: 'None of the selected sanctions lists could be downloaded. Not charged. Run again later.' }); continue; }
138 n++;
139 const typeWanted = e.type && /indiv|person/i.test(e.type) ? 'individual' : e.type && /entit|company|org/i.test(e.type) ? 'entity' : null;
140 const byRec = new Map();
141 for (const h of index.search(e.name, minScore)) {
142 const rec = records[h.r];
143 if (typeWanted && (typeWanted === 'individual') !== (rec.type === 'individual')) continue;
144 if (!byRec.has(h.r)) byRec.set(h.r, { h, rec });
145 }
146 const ms = [...byRec.values()].map(({ h, rec }) => ({
147 score: h.score, list: rec.list, listName: SOURCES[rec.list].label, listId: rec.listId, listedName: rec.primaryName, matchedName: h.name, type: rec.type,
148 programs: rec.programs.slice(0, 10), countries: rec.countries.slice(0, 10), countryMatch: e.country && rec.countries.length ? rec.countries.includes(e.country) : null,
149 datesOfBirth: rec.datesOfBirth.slice(0, 5), listedOn: rec.listedOn, aliasCount: rec.names.length, remarks: rec.remarks,
150 })).sort((a, b) => b.score - a.score || (b.countryMatch === true) - (a.countryMatch === true));
151 const row = { ...base, status: ms.length ? 'potential_match' : 'clear', topScore: ms[0]?.score ?? null, matchCount: ms.length, matches: ms.slice(0, maxMatches), listsChecked, listsNotChecked: unavailable, error: null };
152 if (unavailable.length) { row.status = ms.length ? 'potential_match_incomplete' : 'clear_incomplete'; row.error = `Not charged: ${unavailable.join(', ')} could not be downloaded, so this result does not cover those lists. Run again later.`; }
153 await save(row);
154 if (n % 500 === 0) log.info(`Screened ${n} names`);
155 }
156 for (const s of skipped) await Actor.pushData({ name: s, status: 'skipped_charge_limit', topScore: null, matchCount: 0, matches: [], error: 'Not screened: the run reached the maximum cost you set. Raise the limit and run these again.', screenedAt: new Date().toISOString() });
157 if (skipped.length) counts.skipped_charge_limit = skipped.length;
158
159 const screened = (counts.clear ?? 0) + (counts.potential_match ?? 0) + (counts.clear_incomplete ?? 0) + (counts.potential_match_incomplete ?? 0);
160 const dsOut = Actor.getEnv().defaultDatasetId;
161 const summary = {
162 namesRequested: entries.length, screened, clear: (counts.clear ?? 0) + (counts.clear_incomplete ?? 0), potentialMatches: (counts.potential_match ?? 0) + (counts.potential_match_incomplete ?? 0), byStatus: counts,
163 duplicatesSkipped: duplicates, notScreenedOverMaxNames: overCap, chargeLimitReached: chargeLimitHit, minScore, entityType, lists: listInfo, listsNotChecked: unavailable, sourceDatasetId: dsId || null,
164 zeroResultsReason: screened ? null : `No name was screened: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ') || 'no valid input'}. See each row's status and error.`,
165 disclaimer: 'Screening aid based on the official public lists; not legal advice. Review every potential_match manually (compare dates of birth, countries and programs) before acting.',
166 finishedAt: new Date().toISOString(),
167 };
168 summary.nextSteps = [
169 'Filter by status: "clear" names had no listed name at or above minScore; "potential_match" rows need a manual review of matches[].',
170 'Too many weak matches? Raise minScore (e.g. 92) or set entityType. Worried about misses? Lower it to 80.',
171 ...(unavailable.length ? [`Re-run later: ${unavailable.join(', ')} could not be downloaded (rows were not charged).`] : []),
172 'Need the legal entity behind a company? Run swillz/lei-lookup with the same names.',
173 `Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${dsOut}"}`,
174 ];
175 await Actor.setValue('OUTPUT', summary);
176 log.info(`Done: ${screened} screened, ${summary.potentialMatches} potential matches, ${summary.clear} clear.`);
177 await Actor.exit();
178} catch (e) {
179 if (e instanceof InputError) { await Actor.setValue('OUTPUT', { error: e.message, zeroResultsReason: e.message, nextSteps: ['Fix the input and run again.'] }); await Actor.fail(e.message); }
180 else { log.exception(e, 'Run failed'); await Actor.fail(`Unexpected error: ${e.message}`); }
181}
