1import { Actor, log } from 'apify';
2import { gotScraping } from 'got-scraping';
3import { parseVat, nameMatch, cleanField, COUNTRY_NAMES } from './vat.js';
4
5const EVENT = 'vat-checked';
6const API = 'https://ec.europa.eu/taxation_customs/vies/rest-api';
7const CHARGED = new Set(['valid', 'invalid']);
8const RETRYABLE = new Set(['MS_UNAVAILABLE', 'MS_MAX_CONCURRENT_REQ', 'MS_MAX_CONCURRENT_REQ_TIME', 'GLOBAL_MAX_CONCURRENT_REQ', 'GLOBAL_MAX_CONCURRENT_REQ_TIME', 'SERVICE_UNAVAILABLE', 'TIMEOUT', 'SERVER_BUSY']);
9const VAT_FIELDS = ['vatNumber', 'vat', 'vatId', 'vat_number', 'vatID', 'taxId', 'tax_id', 'euVat', 'VAT'];
10const NAME_FIELDS = ['companyName', 'name', 'company', 'title', 'legalName', 'businessName'];
11const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
12const getPath = (o, path) => path.split('.').reduce((v, k) => (v && typeof v === 'object' ? v[k] : undefined), o);
13class InputError extends Error {}
14
15await Actor.init();
16try {
17 const input = (await Actor.getInput()) ?? {};
18 const { vatNumbers = [], defaultCountry = '', requesterVatNumber = '', concurrency = 4, perCountryConcurrency = 2, maxRetries = 3, requestTimeoutSecs = 30 } = input;
19 const entries = [];
20 const list = Array.isArray(vatNumbers) ? vatNumbers : String(vatNumbers).split(/[\n;]+/);
21 for (const v of list) {
22 if (v && typeof v === 'object') { entries.push({ raw: String(v.vatNumber ?? v.vat ?? v.vatId ?? ''), expectedName: v.companyName ?? v.name ?? null }); continue; }
23 const str = String(v ?? '').trim(); if (!str) continue;
24 const [num, ...name] = str.split('|');
25 entries.push({ raw: num.trim(), expectedName: name.join('|').trim() || null });
26 }
27 const dsId = String(input.datasetId || input?.payload?.resource?.defaultDatasetId || input?.resource?.defaultDatasetId || '').trim();
28 if (dsId) {
29 const max = Math.max(1, Math.floor(Number(input.datasetMaxItems) || 1000));
30 let vf = String(input.vatField || '').trim() || null; let nf = String(input.nameField || '').trim() || null; let offset = 0; let got = 0;
31 while (got < max) {
32 let page;
33 try { page = await Actor.newClient().dataset(dsId).listItems({ offset, limit: 1000, clean: true }); } catch (e) { throw new InputError(`Could not read dataset ${dsId}: ${e.message}. Use the dataset ID of a run you can access.`); }
34 const items = page.items ?? []; if (!items.length) break;
35 vf ||= VAT_FIELDS.find((f) => items.some((it) => String(getPath(it, f) ?? '').trim())) ?? null;
36 nf ||= NAME_FIELDS.find((f) => items.some((it) => typeof getPath(it, f) === 'string')) ?? null;
37 if (!vf) break;
38 for (const it of items) { const val = String(getPath(it, vf) ?? '').trim(); if (val) { entries.push({ raw: val, expectedName: nf ? String(getPath(it, nf) ?? '').trim() || null : null }); got++; } if (got >= max) break; }
39 offset += items.length; if (items.length < 1000) break;
40 }
41 log.info(`Read ${got} VAT numbers from dataset ${dsId} (field "${vf ?? 'none found'}"${nf ? `, names from "${nf}"` : ''}).`);
42 if (!got && !entries.length) throw new InputError(`Dataset ${dsId} has no VAT number values. Set "vatField" (tried: ${VAT_FIELDS.join(', ')}).`);
43 }
44 if (!entries.length) throw new InputError('Add at least one EU VAT number, e.g. "IE6388047V" or "DE 123 456 789", or pass a datasetId from another Actor run.');
45
46 let requester = null;
47 if (String(requesterVatNumber).trim()) {
48 const r = parseVat(requesterVatNumber);
49 if (!r.vatId || r.status) throw new InputError(`requesterVatNumber "${String(requesterVatNumber).slice(0, 30)}" is not a valid EU VAT number format (${r.error}). Leave it empty if you don't need consultation numbers.`);
50 requester = r;
51 }
52 const capRaw = input.maxNumbers ?? input.maxItems ?? input.limit ?? input.maxResults;
53 let cap = Number.isFinite(Number(capRaw)) && Number(capRaw) > 0 ? Math.floor(Number(capRaw)) : 5000;
54 const envCap = Number(process.env.ACTOR_MAX_PAID_DATASET_ITEMS);
55 if (Number.isFinite(envCap) && envCap > 0) cap = Math.min(cap, envCap);
56
57 const seen = new Set(); const targets = []; const localRows = []; let duplicates = 0;
58 for (const e of entries) {
59 const p = parseVat(e.raw, defaultCountry);
60 const base = { input: e.raw.slice(0, 100), vatId: p.vatId ?? null, countryCode: p.country ?? null, countryName: COUNTRY_NAMES[p.country] ?? null, vatNumber: p.number ?? null, expectedName: e.expectedName };
61 if (p.status) { localRows.push({ ...base, status: p.status, valid: p.status === 'invalid_format' ? false : null, error: p.error }); continue; }
62 if (seen.has(p.vatId)) { duplicates++; continue; }
63 seen.add(p.vatId); targets.push({ ...base, ...p });
64 }
65 const overCap = targets.length > cap ? targets.splice(cap) : [];
66 if (overCap.length) log.warning(`maxNumbers is ${cap}: ${overCap.length} extra numbers were not checked.`);
67
68
69 const down = new Set();
70 try {
71 const st = await gotScraping({ url: `${API}/check-status`, responseType: 'json', timeout: { request: 15000 }, throwHttpErrors: false, headers: { accept: 'application/json' } });
72 for (const c of st.body?.countries ?? []) if (c.availability && c.availability !== 'Available') down.add(c.countryCode);
73 if (down.size) log.warning(`VIES reports these member states unavailable now: ${[...down].join(', ')}. They will be retried once.`);
74 } catch { }
75
76 const charging = Actor.getChargingManager();
77 const counts = {}; const skipped = []; let chargeLimitHit = false; const stats = { requests: 0, retries: 0 };
78 const timeout = Math.max(5, Math.min(90, Number(requestTimeoutSecs) || 30)) * 1000;
79 const perCountry = new Map(); const pc = Math.max(1, Math.min(5, Math.floor(Number(perCountryConcurrency) || 2)));
80 async function withCountry(cc, fn) {
81 while ((perCountry.get(cc) ?? 0) >= pc) await sleep(100);
82 perCountry.set(cc, (perCountry.get(cc) ?? 0) + 1);
83 try { return await fn(); } finally { perCountry.set(cc, perCountry.get(cc) - 1); }
84 }
85 async function vies(t, useRequester) {
86 const body = { countryCode: t.country, vatNumber: t.number };
87 if (useRequester && requester) Object.assign(body, { requesterMemberStateCode: requester.country, requesterNumber: requester.number });
88 stats.requests++;
89 const res = await gotScraping({ url: `${API}/check-vat-number`, method: 'POST', json: body, responseType: 'json', timeout: { request: timeout }, throwHttpErrors: false, retry: { limit: 0 }, headers: { accept: 'application/json' } });
90 const b = res.body && typeof res.body === 'object' ? res.body : {};
91 const code = b.errorWrappers?.[0]?.error ?? (res.statusCode >= 500 ? 'SERVICE_UNAVAILABLE' : res.statusCode >= 400 ? `HTTP_${res.statusCode}` : null);
92 return { code, b };
93 }
94 async function check(t) {
95 const tries = Math.max(0, Math.min(6, Number.isFinite(Number(maxRetries)) ? Math.floor(Number(maxRetries)) : 3));
96 const attempts = down.has(t.country) ? 1 : tries + 1;
97 let useReq = !!requester; let lastCode = null; let requesterNote = null;
98 for (let a = 0; a < attempts; a++) {
99 let r;
100 try { r = await withCountry(t.country, () => vies(t, useReq)); } catch (e) { lastCode = 'TIMEOUT'; r = null; }
101 if (r && !r.code && typeof r.b.valid === 'boolean') {
102 const name = cleanField(r.b.name); const address = cleanField(r.b.address);
103 return {
104 input: t.input, vatId: t.vatId, countryCode: t.country, countryName: t.countryName, vatNumber: t.number,
105 status: r.b.valid ? 'valid' : 'invalid', valid: r.b.valid, name, address,
106 addressLines: address ? address.split(/\n+/).map((x) => x.trim()).filter(Boolean) : [],
107 expectedName: t.expectedName, nameMatch: r.b.valid ? nameMatch(t.expectedName, name) : null,
108 consultationNumber: cleanField(r.b.requestIdentifier), requestDate: r.b.requestDate ?? null,
109 requesterNote, source: 'EU VIES', error: null, checkedAt: new Date().toISOString(),
110 };
111 }
112 if (r) lastCode = r.code ?? 'UNEXPECTED_RESPONSE';
113 if (lastCode === 'INVALID_INPUT') return { ...pub(t), status: 'invalid_format', valid: false, error: 'VIES rejected the number format for this country.' };
114 if (lastCode === 'INVALID_REQUESTER_INFO' && useReq) { useReq = false; requesterNote = 'VIES rejected requesterVatNumber, so no consultation number was issued.'; a--; continue; }
115 if (!RETRYABLE.has(lastCode) && !/^TIMEOUT|^HTTP_429/.test(lastCode)) break;
116 if (a < attempts - 1) { stats.retries++; await sleep(Math.min(20000, 2000 * 2 ** a) + Math.random() * 1000); }
117 }
118 if (RETRYABLE.has(lastCode) || lastCode === 'HTTP_429') return { ...pub(t), status: 'service_unavailable', valid: null, error: `${t.countryName} VAT service did not answer (${lastCode}). Not charged. Run this number again later.` };
119 return { ...pub(t), status: 'error', valid: null, error: `VIES error ${lastCode}.` };
120 }
121 const pub = (t) => ({ input: t.input, vatId: t.vatId, countryCode: t.country, countryName: t.countryName, vatNumber: t.number, expectedName: t.expectedName, checkedAt: new Date().toISOString() });
122
123 async function save(row) {
124 if (!CHARGED.has(row.status)) { counts[row.status] = (counts[row.status] ?? 0) + 1; await Actor.pushData(row); return; }
125 if (chargeLimitHit) { skipped.push(row); return; }
126 if (charging) {
127 const allowed = charging.calculateMaxEventChargeCountWithinLimit(EVENT);
128 if (Number.isFinite(allowed) && allowed < 1) { chargeLimitHit = true; skipped.push(row); log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.'); return; }
129 }
130 counts[row.status] = (counts[row.status] ?? 0) + 1;
131 const r = await Actor.pushData(row, EVENT);
132 if (r?.eventChargeLimitReached) chargeLimitHit = true;
133 }
134
135 let idx = 0; let done = 0;
136 const conc = Math.max(1, Math.min(10, Math.floor(Number(concurrency) || 4)));
137 const worker = async () => {
138 while (idx < targets.length && !chargeLimitHit) {
139 const t = targets[idx++];
140 let row; try { row = await check(t); } catch (e) { row = { ...pub(t), status: 'error', valid: null, error: String(e.message).slice(0, 300) }; }
141 await save(row);
142 if (++done % 50 === 0) log.info(`Checked ${done}/${targets.length} VAT numbers`);
143 }
144 };
145 await Promise.all(Array.from({ length: Math.min(conc, targets.length) }, worker));
146 const notChecked = [...skipped.map((r) => r.vatId), ...targets.slice(idx).map((t) => t.vatId)];
147 for (const v of notChecked) await Actor.pushData({ vatId: v, status: 'skipped_charge_limit', valid: null, error: 'Not checked: the run reached the maximum cost you set. Raise the limit and run these again.', checkedAt: new Date().toISOString() });
148 if (notChecked.length) counts.skipped_charge_limit = notChecked.length;
149 for (const r of localRows) { counts[r.status] = (counts[r.status] ?? 0) + 1; await Actor.pushData({ ...r, checkedAt: new Date().toISOString() }); }
150
151 const checked = (counts.valid ?? 0) + (counts.invalid ?? 0);
152 const dsOut = Actor.getEnv().defaultDatasetId;
153 const summary = {
154 numbersRequested: entries.length, uniqueChecked: checked, valid: counts.valid ?? 0, invalid: counts.invalid ?? 0, byStatus: counts, duplicatesSkipped: duplicates,
155 notCheckedOverMaxNumbers: overCap.length, memberStatesUnavailable: [...down], consultationNumbers: !!requester, chargeLimitReached: chargeLimitHit, sourceDatasetId: dsId || null, requests: stats,
156 zeroResultsReason: checked ? null : `No number got an answer from VIES: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ') || 'no valid input'}. See each row's status and error.`,
157 finishedAt: new Date().toISOString(),
158 };
159 summary.nextSteps = [
160 'Filter the dataset by status: "valid" numbers are registered for intra-EU trade today; "invalid" are not.',
161 ...(counts.service_unavailable ? ['Re-run the service_unavailable numbers in a few minutes; member-state databases go offline for maintenance (not charged).'] : []),
162 ...(requester ? [] : ['Need audit proof? Add requesterVatNumber (your own EU VAT number) to get a VIES consultation number per check.']),
163 `Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${dsOut}"}`,
164 ];
165 await Actor.setValue('OUTPUT', summary);
166 const free = Object.entries(counts).filter(([k]) => !CHARGED.has(k));
167 await Actor.exit(`${checked ? '' : '0 checked: '}Checked ${checked} VAT numbers in VIES (${counts.valid ?? 0} valid, ${counts.invalid ?? 0} invalid)${free.length ? `; ${free.map(([k, v]) => `${v} ${k}`).join(', ')} not charged` : ''}${duplicates ? `; ${duplicates} duplicates skipped` : ''}.`);
168} catch (e) {
169 if (e instanceof InputError) { log.error(e.message); await Actor.fail(`Invalid input: ${e.message}`); } else { log.exception(e, 'Run failed'); await Actor.fail(e.message); }
170}
