1import { Actor, log } from 'apify';
2import { gotScraping } from 'got-scraping';
3import { parseEori, soapRequest, parseSoap, parseHmrc, nameMatch, COUNTRY_NAMES } from './eori.js';
4
5const EVENT = 'eori-checked';
6const EU_API = 'https://ec.europa.eu/taxation_customs/dds2/eos/validation/services/validation';
7const HMRC_API = 'https://api.service.hmrc.gov.uk/customs/eori/lookup/check-multiple-eori';
8const CHARGED = new Set(['valid', 'invalid']);
9const EORI_FIELDS = ['eori', 'eoriNumber', 'EORI', 'eori_number', 'eoriNo', 'customsId'];
10const NAME_FIELDS = ['companyName', 'name', 'company', 'legalName', 'traderName', 'businessName'];
11const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
12const getPath = (o, path) => path.split('.').reduce((v, k) => (v && typeof v === 'object' ? v[k] : undefined), o);
13class InputError extends Error {}
14
15await Actor.init();
16try {
17 const input = (await Actor.getInput()) ?? {};
18 const { defaultCountry = '', maxRetries = 3, requestTimeoutSecs = 30 } = input;
19 const raw = input.eoriNumbers ?? input.eoris ?? input.numbers ?? [];
20 const entries = [];
21 for (const v of Array.isArray(raw) ? raw : String(raw).split(/[\n;]+/)) {
22 if (v && typeof v === 'object') { entries.push({ raw: String(v.eori ?? v.eoriNumber ?? ''), expectedName: v.companyName ?? v.name ?? null }); continue; }
23 const str = String(v ?? '').trim(); if (!str) continue;
24 const [num, ...name] = str.split('|');
25 entries.push({ raw: num.trim(), expectedName: name.join('|').trim() || null });
26 }
27 const dsId = String(input.datasetId || input?.payload?.resource?.defaultDatasetId || input?.resource?.defaultDatasetId || '').trim();
28 if (dsId) {
29 const max = Math.max(1, Math.floor(Number(input.datasetMaxItems) || 1000));
30 let ef = String(input.eoriField || '').trim() || null; let nf = String(input.nameField || '').trim() || null; let offset = 0; let got = 0;
31 while (got < max) {
32 let page;
33 try { page = await Actor.newClient().dataset(dsId).listItems({ offset, limit: 1000, clean: true }); } catch (e) { throw new InputError(`Could not read dataset ${dsId}: ${e.message}. Use the dataset ID of a run you can access.`); }
34 const items = page.items ?? []; if (!items.length) break;
35 ef ||= EORI_FIELDS.find((f) => items.some((it) => String(getPath(it, f) ?? '').trim())) ?? null;
36 nf ||= NAME_FIELDS.find((f) => items.some((it) => typeof getPath(it, f) === 'string')) ?? null;
37 if (!ef) break;
38 for (const it of items) { const val = String(getPath(it, ef) ?? '').trim(); if (val) { entries.push({ raw: val, expectedName: nf ? String(getPath(it, nf) ?? '').trim() || null : null }); got++; } if (got >= max) break; }
39 offset += items.length; if (items.length < 1000) break;
40 }
41 log.info(`Read ${got} EORI numbers from dataset ${dsId} (field "${ef ?? 'none found'}").`);
42 if (!got && !entries.length) throw new InputError(`Dataset ${dsId} has no EORI values. Set "eoriField" (tried: ${EORI_FIELDS.join(', ')}).`);
43 }
44 if (!entries.length) throw new InputError('Add at least one EORI number, e.g. "DE1234567" or "GB123456789000", or pass a datasetId from another Actor run.');
45
46 const capRaw = input.maxNumbers ?? input.maxItems ?? input.limit ?? input.maxResults;
47 let cap = Number.isFinite(Number(capRaw)) && Number(capRaw) > 0 ? Math.floor(Number(capRaw)) : 5000;
48 const envCap = Number(process.env.ACTOR_MAX_PAID_DATASET_ITEMS);
49 if (Number.isFinite(envCap) && envCap > 0) cap = Math.min(cap, envCap);
50
51 const seen = new Set(); const targets = []; const localRows = []; let duplicates = 0;
52 for (const e of entries) {
53 const p = parseEori(e.raw, defaultCountry);
54 const base = { input: e.raw.slice(0, 100), eori: p.eori ?? null, countryCode: p.country ?? null, countryName: COUNTRY_NAMES[p.country] ?? null, expectedName: e.expectedName };
55 if (p.status) { localRows.push({ ...base, status: p.status, valid: p.status === 'invalid_format' ? false : null, error: p.error }); continue; }
56 if (seen.has(p.eori)) { duplicates++; continue; }
57 seen.add(p.eori); targets.push({ ...base, source: p.source });
58 }
59 const overCap = targets.length > cap ? targets.splice(cap) : [];
60 if (overCap.length) log.warning(`maxNumbers is ${cap}: ${overCap.length} extra numbers were not checked.`);
61
62 const timeout = Math.max(5, Math.min(90, Number(requestTimeoutSecs) || 30)) * 1000;
63 const tries = Math.max(0, Math.min(6, Number.isFinite(Number(maxRetries)) ? Math.floor(Number(maxRetries)) : 3)) + 1;
64 const stats = { requests: 0, retries: 0 };
65 async function callEu(batch) {
66 let last = null;
67 for (let a = 0; a < tries; a++) {
68 try {
69 stats.requests++;
70 const res = await gotScraping({ url: EU_API, method: 'POST', body: soapRequest(batch.map((t) => t.eori)), headers: { 'content-type': 'text/xml; charset=utf-8', soapaction: '""' }, timeout: { request: timeout }, throwHttpErrors: false, retry: { limit: 0 } });
71 const parsed = parseSoap(String(res.body ?? ''));
72 if (res.statusCode < 500 && Object.keys(parsed.results).length) return { results: parsed.results, requestDate: parsed.requestDate };
73 last = parsed.error || `HTTP ${res.statusCode}`;
74 } catch (e) { last = `TIMEOUT/${e.code ?? e.message}`; }
75 if (a < tries - 1) { stats.retries++; await sleep(Math.min(20000, 1500 * 2 ** a) + Math.random() * 500); }
76 }
77 return { error: last };
78 }
79 async function callHmrc(batch) {
80 let last = null;
81 for (let a = 0; a < tries; a++) {
82 try {
83 stats.requests++;
84 const res = await gotScraping({ url: HMRC_API, method: 'POST', json: { eoris: batch.map((t) => t.eori) }, responseType: 'json', headers: { accept: 'application/vnd.hmrc.1.0+json' }, timeout: { request: timeout }, throwHttpErrors: false, retry: { limit: 0 } });
85 if (res.statusCode === 200 && Array.isArray(res.body)) return { results: parseHmrc(res.body) };
86 last = `HTTP ${res.statusCode}${res.body?.message ? ` ${String(res.body.message).slice(0, 120)}` : ''}`;
87 if (res.statusCode === 400) break;
88 } catch (e) { last = `TIMEOUT/${e.code ?? e.message}`; }
89 if (a < tries - 1) { stats.retries++; await sleep(Math.min(20000, 1500 * 2 ** a) + Math.random() * 500); }
90 }
91 return { error: last };
92 }
93
94 const charging = Actor.getChargingManager();
95 const counts = {}; const skipped = []; let chargeLimitHit = false;
96 async function save(row) {
97 if (!CHARGED.has(row.status)) { counts[row.status] = (counts[row.status] ?? 0) + 1; await Actor.pushData(row); return; }
98 if (chargeLimitHit) { skipped.push(row); return; }
99 if (charging) {
100 const allowed = charging.calculateMaxEventChargeCountWithinLimit(EVENT);
101 if (Number.isFinite(allowed) && allowed < 1) { chargeLimitHit = true; skipped.push(row); log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.'); return; }
102 }
103 counts[row.status] = (counts[row.status] ?? 0) + 1;
104 const r = await Actor.pushData(row, EVENT);
105 if (r?.eventChargeLimitReached) chargeLimitHit = true;
106 }
107 const batches = [];
108 for (const src of ['eu', 'hmrc']) { const list = targets.filter((t) => t.source === src); for (let i = 0; i < list.length; i += 10) batches.push({ src, items: list.slice(i, i + 10) }); }
109 let processed = 0;
110 for (const b of batches) {
111 if (chargeLimitHit) { skipped.push(...b.items); continue; }
112 const r = b.src === 'eu' ? await callEu(b.items) : await callHmrc(b.items);
113 for (const t of b.items) {
114 const hit = r.results?.[t.eori];
115 const common = { input: t.input, eori: t.eori, countryCode: t.countryCode, countryName: t.countryName, expectedName: t.expectedName, source: b.src === 'eu' ? 'EU EOS (European Commission)' : 'HMRC (UK)', checkedAt: new Date().toISOString() };
116 let row;
117 if (hit && typeof hit.valid === 'boolean') {
118 const address = hit.address ?? ([hit.street, hit.postalCode, hit.city, hit.country].filter(Boolean).join(', ') || null);
119 row = { ...common, status: hit.valid ? 'valid' : 'invalid', valid: hit.valid, name: hit.name, address, street: hit.street, postalCode: hit.postalCode, city: hit.city,
120 nameMatch: hit.valid ? nameMatch(t.expectedName, hit.name) : null, namePublished: !!hit.name, requestDate: r.requestDate ?? hit.processingDate ?? null, error: hit.valid ? null : (hit.errorReason || hit.statusDescr || null) };
121 } else if (r.error) {
122 row = { ...common, status: 'service_unavailable', valid: null, error: `${common.source} did not answer (${String(r.error).slice(0, 160)}). Not charged. Run this number again later.` };
123 } else {
124 row = { ...common, status: 'error', valid: null, error: 'The service answered but did not return this number. Not charged.' };
125 }
126 await save(row);
127 }
128 processed += b.items.length;
129 if (processed % 100 < 10) log.info(`Checked ${processed}/${targets.length} EORI numbers`);
130 await sleep(250);
131 }
132 for (const t of skipped) await Actor.pushData({ eori: t.eori, status: 'skipped_charge_limit', valid: null, error: 'Not checked: the run reached the maximum cost you set. Raise the limit and run these again.', checkedAt: new Date().toISOString() });
133 if (skipped.length) counts.skipped_charge_limit = skipped.length;
134 for (const r of localRows) { counts[r.status] = (counts[r.status] ?? 0) + 1; await Actor.pushData({ ...r, checkedAt: new Date().toISOString() }); }
135
136 const checked = (counts.valid ?? 0) + (counts.invalid ?? 0);
137 const dsOut = Actor.getEnv().defaultDatasetId;
138 const summary = {
139 numbersRequested: entries.length, uniqueChecked: checked, valid: counts.valid ?? 0, invalid: counts.invalid ?? 0, byStatus: counts, duplicatesSkipped: duplicates,
140 notCheckedOverMaxNumbers: overCap.length, chargeLimitReached: chargeLimitHit, sourceDatasetId: dsId || null, requests: stats,
141 zeroResultsReason: checked ? null : `No number got an answer: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ') || 'no valid input'}. See each row's status and error.`,
142 finishedAt: new Date().toISOString(),
143 };
144 summary.nextSteps = [
145 'Filter by status: "valid" EORI numbers are registered for customs today; "invalid" are not (or have expired).',
146 'Names and addresses appear only when the trader agreed to publish them; otherwise namePublished is false.',
147 ...(counts.service_unavailable ? ['Re-run the service_unavailable numbers later (not charged).'] : []),
148 'Also need VAT checks? Run swillz/eu-vat-validator with the same companies.',
149 `Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${dsOut}"}`,
150 ];
151 await Actor.setValue('OUTPUT', summary);
152 const free = Object.entries(counts).filter(([k]) => !CHARGED.has(k));
153 await Actor.exit(`${checked ? '' : '0 checked: '}Checked ${checked} EORI numbers (${counts.valid ?? 0} valid, ${counts.invalid ?? 0} invalid)${free.length ? `; ${free.map(([k, v]) => `${v} ${k}`).join(', ')} not charged` : ''}${duplicates ? `; ${duplicates} duplicates skipped` : ''}.`);
154} catch (e) {
155 if (e instanceof InputError) { log.error(e.message); await Actor.fail(`Invalid input: ${e.message}`); } else { log.exception(e, 'Run failed'); await Actor.fail(e.message); }
156}
