1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16import { createHash } from 'node:crypto';
17import { Actor, log } from 'apify';
18
19const USER_AGENT = 'foundry-actor/0.1 (Apify Actor eu-ted-tenders; polite client)';
20const SEARCH_URL = 'https://api.ted.europa.eu/v3/notices/search';
21const MIN_INTERVAL_MS = 1000;
22const PAGE_LIMIT = 100;
23let lastRequestAt = 0;
24
25
26const DEFAULTS = {
27 keywords: '',
28 countries: [],
29 cpvCodes: [],
30 noticeCategory: 'competition',
31 onlyOpen: true,
32 onlyNewSinceLastRun: false,
33 publishedWithinDays: 7,
34 language: 'eng',
35 maxItems: 10,
36};
37
38
39const NOTICE_TYPES = {
40 competition: ['cn-standard', 'cn-social', 'cn-desg', 'pin-cfc-standard', 'pin-cfc-social', 'qu-sy', 'subco'],
41 planning: ['pin-only', 'pin-buyer', 'pin-rtl', 'pin-tran'],
42 result: ['can-standard', 'can-social', 'can-desg', 'can-tran', 'veat', 'can-modif'],
43};
44const CATEGORY_OF_TYPE = Object.fromEntries(
45 Object.entries(NOTICE_TYPES).flatMap(([cat, types]) => types.map((t) => [t, cat])),
46);
47
48const LANGUAGES = ['bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'est', 'fin', 'fra', 'gle', 'hrv', 'hun',
49 'ita', 'lav', 'lit', 'mlt', 'nld', 'pol', 'por', 'ron', 'slk', 'slv', 'spa', 'swe'];
50
51
52const ALPHA2 = {
53 AT: 'AUT', BE: 'BEL', BG: 'BGR', HR: 'HRV', CY: 'CYP', CZ: 'CZE', DK: 'DNK', EE: 'EST', FI: 'FIN',
54 FR: 'FRA', DE: 'DEU', GR: 'GRC', EL: 'GRC', HU: 'HUN', IE: 'IRL', IT: 'ITA', LV: 'LVA', LT: 'LTU',
55 LU: 'LUX', MT: 'MLT', NL: 'NLD', PL: 'POL', PT: 'PRT', RO: 'ROU', SK: 'SVK', SI: 'SVN', ES: 'ESP',
56 SE: 'SWE', NO: 'NOR', IS: 'ISL', LI: 'LIE', CH: 'CHE', MK: 'MKD', RS: 'SRB', ME: 'MNE', AL: 'ALB',
57 TR: 'TUR', UA: 'UKR', MD: 'MDA', GE: 'GEO', BA: 'BIH', GB: 'GBR', UK: 'GBR',
58};
59
60
61const FIELDS = [
62 'publication-number', 'notice-type', 'notice-title', 'publication-date',
63 'buyer-name', 'buyer-country', 'buyer-city',
64 'deadline-receipt-tender-date-lot', 'procedure-type', 'contract-nature',
65 'classification-cpv', 'place-of-performance',
66 'estimated-value-proc', 'estimated-value-cur-proc', 'estimated-value-lot', 'estimated-value-cur-lot',
67 'total-value', 'total-value-cur', 'description-proc', 'winner-name',
68];
69
70const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
71
72async function politePostJson(url, body, attempt = 1) {
73 const wait = lastRequestAt + MIN_INTERVAL_MS - Date.now();
74 if (wait > 0) await sleep(wait);
75 lastRequestAt = Date.now();
76
77 let res;
78 try {
79 res = await fetch(url, {
80 method: 'POST',
81 headers: { 'user-agent': USER_AGENT, accept: 'application/json', 'content-type': 'application/json' },
82 body: JSON.stringify(body),
83 signal: AbortSignal.timeout(60_000),
84 });
85 } catch (err) {
86 if (attempt >= 4) throw new Error(`Network error after ${attempt} attempts: ${url} (${err.message})`);
87 await sleep(1000 * 2 ** attempt);
88 return politePostJson(url, body, attempt + 1);
89 }
90 if (res.status === 429 || res.status >= 500) {
91 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
92 const retryAfter = Number(res.headers.get('retry-after'));
93 const delayMs = retryAfter > 0 ? Math.min(retryAfter, 60) * 1000 : 1000 * 2 ** attempt;
94 log.warning(`HTTP ${res.status} from TED; waiting ${Math.round(delayMs / 1000)} s before retrying.`);
95 await sleep(delayMs);
96 return politePostJson(url, body, attempt + 1);
97 }
98 if (res.status === 400) {
99 const text = await res.text().catch(() => '');
100 let message = text;
101 try {
102 message = JSON.parse(text).message ?? text;
103 } catch { }
104 throw new Error(`TED rejected the search (HTTP 400): ${String(message).slice(0, 300)}`);
105 }
106 if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}: ${url}`);
107 return res.json();
108}
109
110
111
112function remainingBudget() {
113 try {
114 const n = Actor.getChargingManager().calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
115 return Number.isFinite(n) ? n : Infinity;
116 } catch {
117 return Infinity;
118 }
119}
120
121
122
123function readInput(raw) {
124 const input = { ...DEFAULTS, ...Object.fromEntries(Object.entries(raw ?? {}).filter(([, v]) => v !== undefined && v !== null)) };
125
126 if (typeof input.keywords !== 'string') throw new Error('Input "keywords" must be a string (comma-separated words or phrases).');
127 const phrases = input.keywords.split(',').map((s) => s.replace(/["\\]/g, ' ').replace(/\s+/g, ' ').trim()).filter(Boolean);
128 for (const p of phrases) {
129 if (p.length > 100) throw new Error(`Input "keywords": "${p.slice(0, 40)}..." is longer than 100 characters.`);
130 }
131 if (phrases.length > 20) throw new Error('Input "keywords" accepts at most 20 comma-separated phrases.');
132
133 if (!Array.isArray(input.countries)) throw new Error('Input "countries" must be a list of country codes, e.g. ["DEU", "FRA"].');
134 const countries = [...new Set(input.countries.map((c) => {
135 const code = String(c).trim().toUpperCase();
136 return ALPHA2[code] ?? code;
137 }).filter(Boolean))];
138 for (const c of countries) {
139 if (!/^[A-Z]{3}$/.test(c)) throw new Error(`Input "countries" contains "${c}". Use ISO 3166-1 codes such as "DEU" or "DE".`);
140 }
141
142 if (!Array.isArray(input.cpvCodes)) throw new Error('Input "cpvCodes" must be a list of CPV codes, e.g. ["72000000"].');
143 const cpvCodes = [...new Set(input.cpvCodes.map((c) => String(c).trim().replace(/-\d$/, '')).filter(Boolean))];
144 for (const c of cpvCodes) {
145 if (!/^\d{2,8}$/.test(c)) throw new Error(`Input "cpvCodes" contains "${c}". Use 2 to 8 digits, e.g. "72" or "72000000".`);
146 }
147
148 const category = input.noticeCategory;
149 if (!['competition', 'planning', 'result', 'any'].includes(category)) {
150 throw new Error('Input "noticeCategory" must be one of: "competition", "planning", "result", "any".');
151 }
152
153 const days = Number(input.publishedWithinDays);
154 if (!Number.isInteger(days) || days < 1 || days > 365) throw new Error('Input "publishedWithinDays" must be a whole number from 1 to 365.');
155
156 const language = String(input.language).toLowerCase();
157 if (!LANGUAGES.includes(language)) throw new Error(`Input "language" must be one of: ${LANGUAGES.join(', ')}.`);
158
159 const maxItems = Number(input.maxItems);
160 if (!Number.isInteger(maxItems) || maxItems < 1 || maxItems > 10000) throw new Error('Input "maxItems" must be a whole number from 1 to 10000.');
161
162 if (typeof input.onlyNewSinceLastRun !== 'boolean') throw new Error('Input "onlyNewSinceLastRun" must be true or false.');
163
164 return {
165 phrases,
166 countries,
167 cpvCodes: cpvCodes.map((c) => c.padEnd(8, '0')),
168 category,
169 onlyOpen: input.onlyOpen !== false,
170 days,
171 language,
172 maxItems,
173 onlyNewSinceLastRun: input.onlyNewSinceLastRun,
174 };
175}
176
177
178
179
180function filterFingerprint(opts) {
181 return JSON.stringify({
182 phrases: [...opts.phrases].sort(), countries: [...opts.countries].sort(), cpvCodes: [...opts.cpvCodes].sort(),
183 category: opts.category, onlyOpen: opts.onlyOpen,
184 });
185}
186
187const tedDate = (d) => d.toISOString().slice(0, 10).replace(/-/g, '');
188
189function buildQuery(opts, now) {
190 const parts = [`publication-date>=${tedDate(new Date(now - opts.days * 86_400_000))}`];
191 if (opts.phrases.length) parts.push(`(${opts.phrases.map((p) => `FT~"${p}"`).join(' OR ')})`);
192 if (opts.countries.length) parts.push(`buyer-country IN (${opts.countries.join(' ')})`);
193 if (opts.cpvCodes.length) parts.push(`classification-cpv IN (${opts.cpvCodes.join(' ')})`);
194 if (opts.category !== 'any') parts.push(`notice-type IN (${NOTICE_TYPES[opts.category].join(' ')})`);
195 if (opts.onlyOpen && opts.category === 'competition') parts.push(`deadline-receipt-tender-date-lot>=${tedDate(new Date(now))}`);
196 return `${parts.join(' AND ')} SORT BY publication-number DESC`;
197}
198
199
200
201const asList = (v) => (Array.isArray(v) ? v : v === undefined || v === null ? [] : [v]);
202const uniq = (arr) => [...new Set(arr.filter((v) => v !== undefined && v !== null && v !== ''))];
203const dateOnly = (s) => (typeof s === 'string' && /^\d{4}-\d{2}-\d{2}/.test(s) ? s.slice(0, 10) : null);
204const toNumber = (v) => {
205 const n = typeof v === 'number' ? v : Number.parseFloat(String(v ?? ''));
206 return Number.isFinite(n) ? n : null;
207};
208const cleanLine = (s) => (typeof s === 'string' ? s.replace(/\s+/g, ' ').trim() || null : null);
209
210
211
212
213const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9-]+(?:\.[A-Z0-9-]+)*\.[A-Z]{2,}/gi;
214const INTL_PHONE_RE = /\+\d{1,3}[\s.-]?(?:\(0\)[\s.-]?)?\d[\d\s.-]{6,}\d/g;
215const LABELLED_PHONE_RE = /\b(tel(?:ephone)?|phone|mobile|mob)(\.?\s*(?:no\.?|number)?\s*[:.]?\s*)(\(?0[\d\s().-]{8,}\d)/gi;
216function redactContacts(text) {
217 if (typeof text !== 'string') return text;
218 return text.replace(EMAIL_RE, '[email removed]')
219 .replace(INTL_PHONE_RE, '[phone removed]')
220 .replace(LABELLED_PHONE_RE, (m, label, sep) => `${label}${sep}[phone removed]`);
221}
222
223
224
225function pickLang(map, language) {
226 if (!map || typeof map !== 'object' || Array.isArray(map)) return { text: null, lang: null };
227 const keys = Object.keys(map);
228 const key = [language, 'eng'].find((k) => k in map) ?? keys[0];
229 if (!key) return { text: null, lang: null };
230 const value = map[key];
231 const text = Array.isArray(value) ? value.find((v) => typeof v === 'string' && v.trim()) : value;
232 return { text: typeof text === 'string' ? text.trim() : null, lang: key };
233}
234
235function pickLink(links, kind, language) {
236 const group = links?.[kind];
237 if (!group || typeof group !== 'object') return null;
238 const want = language.toUpperCase();
239 return group[want] ?? group.ENG ?? Object.values(group)[0] ?? null;
240}
241
242function mapNotice(n, language) {
243 const title = pickLang(n['notice-title'], language);
244 const description = pickLang(n['description-proc'], language);
245 const deadlines = uniq(asList(n['deadline-receipt-tender-date-lot']).map(dateOnly)).sort();
246 const lotValues = uniq(asList(n['estimated-value-lot']).map(toNumber));
247 const procValue = toNumber(n['estimated-value-proc']);
248 const estimatedValue = procValue ?? (lotValues.length === 1 ? lotValues[0] : null);
249 const estimatedCurrency = estimatedValue === null ? null
250 : (asList(n['estimated-value-cur-proc'])[0] ?? asList(n['estimated-value-cur-lot'])[0] ?? null);
251 const totalValue = toNumber(asList(n['total-value'])[0]);
252 const winners = n['winner-name'] && typeof n['winner-name'] === 'object' && !Array.isArray(n['winner-name'])
253 ? Object.values(n['winner-name']).flatMap(asList)
254 : asList(n['winner-name']);
255
256 return {
257 publicationNumber: n['publication-number'] ?? null,
258 noticeType: n['notice-type'] ?? null,
259 noticeCategory: CATEGORY_OF_TYPE[n['notice-type']] ?? 'other',
260 title: redactContacts(cleanLine(title.text)),
261 titleLanguage: title.lang,
262 buyerName: cleanLine(pickLang(n['buyer-name'], language).text),
263 buyerCountry: asList(n['buyer-country'])[0] ?? null,
264 buyerCity: cleanLine(pickLang(n['buyer-city'], language).text),
265 publishedAt: dateOnly(n['publication-date']),
266 tenderDeadline: deadlines[0] ?? null,
267
268 daysToDeadline: deadlines[0] ? Math.ceil((Date.parse(`${deadlines[0]}T23:59:59Z`) - Date.now()) / 86_400_000) : null,
269 procedureType: n['procedure-type'] ?? null,
270 contractNature: uniq(asList(n['contract-nature'])),
271 cpvCodes: uniq(asList(n['classification-cpv']).map(String)),
272 placeOfPerformance: uniq(asList(n['place-of-performance']).map(String)),
273 estimatedValue,
274 estimatedValueCurrency: estimatedCurrency,
275 totalValue,
276 totalValueCurrency: totalValue === null ? null : (asList(n['total-value-cur'])[0] ?? null),
277 winnerNames: uniq(winners.map(cleanLine)),
278 description: redactContacts(description.text),
279 url: pickLink(n.links, 'html', language),
280 pdfUrl: pickLink(n.links, 'pdf', language),
281 };
282}
283
284
285
286await Actor.init();
287
288const opts = readInput(await Actor.getInput());
289const now = Date.now();
290const query = buildQuery(opts, now);
291
292
293
294
295
296
297
298const MONITOR_MAX_PAGES = 100;
299const MAX_MEMORY = 100_000;
300const monitor = opts.onlyNewSinceLastRun;
301const stateStore = monitor ? await Actor.openKeyValueStore('eu-ted-tenders-seen') : null;
302const stateKey = monitor ? `seen-${createHash('sha256').update(filterFingerprint(opts)).digest('hex').slice(0, 40)}` : null;
303let delivered = {};
304let prevComplete = false;
305if (monitor) {
306 const raw = (await stateStore.getValue(stateKey)) ?? {};
307 const items = raw.v === 2 ? (raw.items ?? {}) : raw;
308 prevComplete = raw.v === 2 && raw.complete === true;
309 const keepAfterMs = now - (Math.max(opts.days, 30) + 2) * 86_400_000;
310 for (const [k, v] of Object.entries(items)) {
311 if (typeof v === 'string' && (Date.parse(v) || 0) >= keepAfterMs) delivered[k] = v;
312 }
313 log.info(`Monitor mode: ${Object.keys(delivered).length} notice(s) delivered by earlier runs with these filters will be skipped.`);
314}
315async function saveMemory(complete) {
316 let entries = Object.entries(delivered);
317 if (entries.length > MAX_MEMORY) {
318 entries = entries.sort((a, b) => (Date.parse(b[1]) || 0) - (Date.parse(a[1]) || 0)).slice(0, MAX_MEMORY);
319 delivered = Object.fromEntries(entries);
320 }
321 await stateStore.setValue(stateKey, { v: 2, complete, items: delivered });
322}
323
324
325const limit = monitor ? PAGE_LIMIT : Math.min(PAGE_LIMIT, opts.maxItems);
326log.info(`TED query: ${query}`);
327
328let saved = 0;
329let page = 1;
330let total = null;
331let exhausted = false;
332let cut = false;
333let stoppedEarly = false;
334while (saved < opts.maxItems) {
335 const data = await politePostJson(SEARCH_URL, {
336 query,
337 fields: FIELDS,
338 page,
339 limit,
340 paginationMode: 'PAGE_NUMBER',
341 });
342 if (total === null) {
343 total = data.totalNoticeCount ?? null;
344 log.info(`TED reports ${total ?? 'an unknown number of'} matching notices.`);
345 }
346 if (data.timedOut) log.warning('TED reported that this search timed out; results may be incomplete.');
347 const notices = Array.isArray(data.notices) ? data.notices : [];
348 if (notices.length === 0) {
349 exhausted = true;
350 break;
351 }
352
353 const mapped = notices.map((n) => mapNotice(n, opts.language));
354 const fresh = monitor ? mapped.filter((n) => !Object.hasOwn(delivered, n.publicationNumber)) : mapped;
355 if (monitor && prevComplete && fresh.length === 0) {
356 stoppedEarly = true;
357 break;
358 }
359 const budget = remainingBudget();
360 const batch = fresh.slice(0, Math.min(opts.maxItems - saved, budget));
361 if (fresh.length > 0 && batch.length === 0) {
362 log.info('Reached the maximum charge set for this run; stopping.');
363 cut = true;
364 break;
365 }
366 if (batch.length > 0) {
367 await Actor.pushData(batch);
368 saved += batch.length;
369 if (monitor) {
370
371 for (const n of batch) delivered[n.publicationNumber] = n.publishedAt;
372 await saveMemory(false);
373 }
374 }
375 if (batch.length < fresh.length && saved < opts.maxItems) {
376 log.info('Reached the maximum charge set for this run; stopping.');
377 cut = true;
378 break;
379 }
380 if (total !== null && page * limit >= total) {
381 exhausted = true;
382 break;
383 }
384 if (monitor && page >= MONITOR_MAX_PAGES) {
385 log.warning(`Monitor mode reads at most ${MONITOR_MAX_PAGES * PAGE_LIMIT} notices per run; the rest is delivered next run. Narrow the filters to cover everything in one run.`);
386 cut = true;
387 break;
388 }
389 page += 1;
390}
391if (stoppedEarly) log.info('Monitor mode: a full page of already-delivered notices; the rest of the window was covered by earlier runs.');
392if (monitor) await saveMemory(stoppedEarly || (exhausted && !cut));
393
394log.info(`Saved ${saved} notices.`);
395if (saved === 0 && monitor) {
396 await Actor.setStatusMessage('No new notices since the last run with these filters.');
397} else if (saved === 0) {
398 await Actor.setStatusMessage('No notices matched these filters in the selected period. Try more days or fewer filters.');
399}
400await Actor.exit();
