1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16import { Actor, log } from 'apify';
17
18const USER_AGENT = 'foundry-actor/0.1 (Apify Actor eu-ted-tenders; polite client)';
19const SEARCH_URL = 'https://api.ted.europa.eu/v3/notices/search';
20const MIN_INTERVAL_MS = 1000;
21const PAGE_LIMIT = 100;
22let lastRequestAt = 0;
23
24
25const DEFAULTS = {
26 keywords: '',
27 countries: [],
28 cpvCodes: [],
29 noticeCategory: 'competition',
30 onlyOpen: true,
31 publishedWithinDays: 7,
32 language: 'eng',
33 maxItems: 10,
34};
35
36
37const NOTICE_TYPES = {
38 competition: ['cn-standard', 'cn-social', 'cn-desg', 'pin-cfc-standard', 'pin-cfc-social', 'qu-sy', 'subco'],
39 planning: ['pin-only', 'pin-buyer', 'pin-rtl', 'pin-tran'],
40 result: ['can-standard', 'can-social', 'can-desg', 'can-tran', 'veat', 'can-modif'],
41};
42const CATEGORY_OF_TYPE = Object.fromEntries(
43 Object.entries(NOTICE_TYPES).flatMap(([cat, types]) => types.map((t) => [t, cat])),
44);
45
46const LANGUAGES = ['bul', 'ces', 'dan', 'deu', 'ell', 'eng', 'est', 'fin', 'fra', 'gle', 'hrv', 'hun',
47 'ita', 'lav', 'lit', 'mlt', 'nld', 'pol', 'por', 'ron', 'slk', 'slv', 'spa', 'swe'];
48
49
50const ALPHA2 = {
51 AT: 'AUT', BE: 'BEL', BG: 'BGR', HR: 'HRV', CY: 'CYP', CZ: 'CZE', DK: 'DNK', EE: 'EST', FI: 'FIN',
52 FR: 'FRA', DE: 'DEU', GR: 'GRC', EL: 'GRC', HU: 'HUN', IE: 'IRL', IT: 'ITA', LV: 'LVA', LT: 'LTU',
53 LU: 'LUX', MT: 'MLT', NL: 'NLD', PL: 'POL', PT: 'PRT', RO: 'ROU', SK: 'SVK', SI: 'SVN', ES: 'ESP',
54 SE: 'SWE', NO: 'NOR', IS: 'ISL', LI: 'LIE', CH: 'CHE', MK: 'MKD', RS: 'SRB', ME: 'MNE', AL: 'ALB',
55 TR: 'TUR', UA: 'UKR', MD: 'MDA', GE: 'GEO', BA: 'BIH', GB: 'GBR', UK: 'GBR',
56};
57
58
59const FIELDS = [
60 'publication-number', 'notice-type', 'notice-title', 'publication-date',
61 'buyer-name', 'buyer-country', 'buyer-city',
62 'deadline-receipt-tender-date-lot', 'procedure-type', 'contract-nature',
63 'classification-cpv', 'place-of-performance',
64 'estimated-value-proc', 'estimated-value-cur-proc', 'estimated-value-lot', 'estimated-value-cur-lot',
65 'total-value', 'total-value-cur', 'description-proc', 'winner-name',
66];
67
68const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
69
70async function politePostJson(url, body, attempt = 1) {
71 const wait = lastRequestAt + MIN_INTERVAL_MS - Date.now();
72 if (wait > 0) await sleep(wait);
73 lastRequestAt = Date.now();
74
75 let res;
76 try {
77 res = await fetch(url, {
78 method: 'POST',
79 headers: { 'user-agent': USER_AGENT, accept: 'application/json', 'content-type': 'application/json' },
80 body: JSON.stringify(body),
81 signal: AbortSignal.timeout(60_000),
82 });
83 } catch (err) {
84 if (attempt >= 4) throw new Error(`Network error after ${attempt} attempts: ${url} (${err.message})`);
85 await sleep(1000 * 2 ** attempt);
86 return politePostJson(url, body, attempt + 1);
87 }
88 if (res.status === 429 || res.status >= 500) {
89 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
90 const retryAfter = Number(res.headers.get('retry-after'));
91 const delayMs = retryAfter > 0 ? Math.min(retryAfter, 60) * 1000 : 1000 * 2 ** attempt;
92 log.warning(`HTTP ${res.status} from TED; waiting ${Math.round(delayMs / 1000)} s before retrying.`);
93 await sleep(delayMs);
94 return politePostJson(url, body, attempt + 1);
95 }
96 if (res.status === 400) {
97 const text = await res.text().catch(() => '');
98 let message = text;
99 try {
100 message = JSON.parse(text).message ?? text;
101 } catch { }
102 throw new Error(`TED rejected the search (HTTP 400): ${String(message).slice(0, 300)}`);
103 }
104 if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}: ${url}`);
105 return res.json();
106}
107
108
109
110function remainingBudget() {
111 try {
112 const n = Actor.getChargingManager().calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
113 return Number.isFinite(n) ? n : Infinity;
114 } catch {
115 return Infinity;
116 }
117}
118
119
120
121function readInput(raw) {
122 const input = { ...DEFAULTS, ...Object.fromEntries(Object.entries(raw ?? {}).filter(([, v]) => v !== undefined && v !== null)) };
123
124 if (typeof input.keywords !== 'string') throw new Error('Input "keywords" must be a string (comma-separated words or phrases).');
125 const phrases = input.keywords.split(',').map((s) => s.replace(/["\\]/g, ' ').replace(/\s+/g, ' ').trim()).filter(Boolean);
126 for (const p of phrases) {
127 if (p.length > 100) throw new Error(`Input "keywords": "${p.slice(0, 40)}..." is longer than 100 characters.`);
128 }
129 if (phrases.length > 20) throw new Error('Input "keywords" accepts at most 20 comma-separated phrases.');
130
131 if (!Array.isArray(input.countries)) throw new Error('Input "countries" must be a list of country codes, e.g. ["DEU", "FRA"].');
132 const countries = [...new Set(input.countries.map((c) => {
133 const code = String(c).trim().toUpperCase();
134 return ALPHA2[code] ?? code;
135 }).filter(Boolean))];
136 for (const c of countries) {
137 if (!/^[A-Z]{3}$/.test(c)) throw new Error(`Input "countries" contains "${c}". Use ISO 3166-1 codes such as "DEU" or "DE".`);
138 }
139
140 if (!Array.isArray(input.cpvCodes)) throw new Error('Input "cpvCodes" must be a list of CPV codes, e.g. ["72000000"].');
141 const cpvCodes = [...new Set(input.cpvCodes.map((c) => String(c).trim().replace(/-\d$/, '')).filter(Boolean))];
142 for (const c of cpvCodes) {
143 if (!/^\d{2,8}$/.test(c)) throw new Error(`Input "cpvCodes" contains "${c}". Use 2 to 8 digits, e.g. "72" or "72000000".`);
144 }
145
146 const category = input.noticeCategory;
147 if (!['competition', 'planning', 'result', 'any'].includes(category)) {
148 throw new Error('Input "noticeCategory" must be one of: "competition", "planning", "result", "any".');
149 }
150
151 const days = Number(input.publishedWithinDays);
152 if (!Number.isInteger(days) || days < 1 || days > 365) throw new Error('Input "publishedWithinDays" must be a whole number from 1 to 365.');
153
154 const language = String(input.language).toLowerCase();
155 if (!LANGUAGES.includes(language)) throw new Error(`Input "language" must be one of: ${LANGUAGES.join(', ')}.`);
156
157 const maxItems = Number(input.maxItems);
158 if (!Number.isInteger(maxItems) || maxItems < 1 || maxItems > 10000) throw new Error('Input "maxItems" must be a whole number from 1 to 10000.');
159
160 return {
161 phrases,
162 countries,
163 cpvCodes: cpvCodes.map((c) => c.padEnd(8, '0')),
164 category,
165 onlyOpen: input.onlyOpen !== false,
166 days,
167 language,
168 maxItems,
169 };
170}
171
172const tedDate = (d) => d.toISOString().slice(0, 10).replace(/-/g, '');
173
174function buildQuery(opts, now) {
175 const parts = [`publication-date>=${tedDate(new Date(now - opts.days * 86_400_000))}`];
176 if (opts.phrases.length) parts.push(`(${opts.phrases.map((p) => `FT~"${p}"`).join(' OR ')})`);
177 if (opts.countries.length) parts.push(`buyer-country IN (${opts.countries.join(' ')})`);
178 if (opts.cpvCodes.length) parts.push(`classification-cpv IN (${opts.cpvCodes.join(' ')})`);
179 if (opts.category !== 'any') parts.push(`notice-type IN (${NOTICE_TYPES[opts.category].join(' ')})`);
180 if (opts.onlyOpen && opts.category === 'competition') parts.push(`deadline-receipt-tender-date-lot>=${tedDate(new Date(now))}`);
181 return `${parts.join(' AND ')} SORT BY publication-number DESC`;
182}
183
184
185
186const asList = (v) => (Array.isArray(v) ? v : v === undefined || v === null ? [] : [v]);
187const uniq = (arr) => [...new Set(arr.filter((v) => v !== undefined && v !== null && v !== ''))];
188const dateOnly = (s) => (typeof s === 'string' && /^\d{4}-\d{2}-\d{2}/.test(s) ? s.slice(0, 10) : null);
189const toNumber = (v) => {
190 const n = typeof v === 'number' ? v : Number.parseFloat(String(v ?? ''));
191 return Number.isFinite(n) ? n : null;
192};
193const cleanLine = (s) => (typeof s === 'string' ? s.replace(/\s+/g, ' ').trim() || null : null);
194
195
196
197
198const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9-]+(?:\.[A-Z0-9-]+)*\.[A-Z]{2,}/gi;
199const INTL_PHONE_RE = /\+\d{1,3}[\s.-]?(?:\(0\)[\s.-]?)?\d[\d\s.-]{6,}\d/g;
200const LABELLED_PHONE_RE = /\b(tel(?:ephone)?|phone|mobile|mob)(\.?\s*(?:no\.?|number)?\s*[:.]?\s*)(\(?0[\d\s().-]{8,}\d)/gi;
201function redactContacts(text) {
202 if (typeof text !== 'string') return text;
203 return text.replace(EMAIL_RE, '[email removed]')
204 .replace(INTL_PHONE_RE, '[phone removed]')
205 .replace(LABELLED_PHONE_RE, (m, label, sep) => `${label}${sep}[phone removed]`);
206}
207
208
209
210function pickLang(map, language) {
211 if (!map || typeof map !== 'object' || Array.isArray(map)) return { text: null, lang: null };
212 const keys = Object.keys(map);
213 const key = [language, 'eng'].find((k) => k in map) ?? keys[0];
214 if (!key) return { text: null, lang: null };
215 const value = map[key];
216 const text = Array.isArray(value) ? value.find((v) => typeof v === 'string' && v.trim()) : value;
217 return { text: typeof text === 'string' ? text.trim() : null, lang: key };
218}
219
220function pickLink(links, kind, language) {
221 const group = links?.[kind];
222 if (!group || typeof group !== 'object') return null;
223 const want = language.toUpperCase();
224 return group[want] ?? group.ENG ?? Object.values(group)[0] ?? null;
225}
226
227function mapNotice(n, language) {
228 const title = pickLang(n['notice-title'], language);
229 const description = pickLang(n['description-proc'], language);
230 const deadlines = uniq(asList(n['deadline-receipt-tender-date-lot']).map(dateOnly)).sort();
231 const lotValues = uniq(asList(n['estimated-value-lot']).map(toNumber));
232 const procValue = toNumber(n['estimated-value-proc']);
233 const estimatedValue = procValue ?? (lotValues.length === 1 ? lotValues[0] : null);
234 const estimatedCurrency = estimatedValue === null ? null
235 : (asList(n['estimated-value-cur-proc'])[0] ?? asList(n['estimated-value-cur-lot'])[0] ?? null);
236 const totalValue = toNumber(asList(n['total-value'])[0]);
237 const winners = n['winner-name'] && typeof n['winner-name'] === 'object' && !Array.isArray(n['winner-name'])
238 ? Object.values(n['winner-name']).flatMap(asList)
239 : asList(n['winner-name']);
240
241 return {
242 publicationNumber: n['publication-number'] ?? null,
243 noticeType: n['notice-type'] ?? null,
244 noticeCategory: CATEGORY_OF_TYPE[n['notice-type']] ?? 'other',
245 title: redactContacts(cleanLine(title.text)),
246 titleLanguage: title.lang,
247 buyerName: cleanLine(pickLang(n['buyer-name'], language).text),
248 buyerCountry: asList(n['buyer-country'])[0] ?? null,
249 buyerCity: cleanLine(pickLang(n['buyer-city'], language).text),
250 publishedAt: dateOnly(n['publication-date']),
251 tenderDeadline: deadlines[0] ?? null,
252 procedureType: n['procedure-type'] ?? null,
253 contractNature: uniq(asList(n['contract-nature'])),
254 cpvCodes: uniq(asList(n['classification-cpv']).map(String)),
255 placeOfPerformance: uniq(asList(n['place-of-performance']).map(String)),
256 estimatedValue,
257 estimatedValueCurrency: estimatedCurrency,
258 totalValue,
259 totalValueCurrency: totalValue === null ? null : (asList(n['total-value-cur'])[0] ?? null),
260 winnerNames: uniq(winners.map(cleanLine)),
261 description: redactContacts(description.text),
262 url: pickLink(n.links, 'html', language),
263 pdfUrl: pickLink(n.links, 'pdf', language),
264 };
265}
266
267
268
269await Actor.init();
270
271const opts = readInput(await Actor.getInput());
272const now = Date.now();
273const query = buildQuery(opts, now);
274const limit = Math.min(PAGE_LIMIT, opts.maxItems);
275log.info(`TED query: ${query}`);
276
277let saved = 0;
278let page = 1;
279let total = null;
280while (saved < opts.maxItems) {
281 const data = await politePostJson(SEARCH_URL, {
282 query,
283 fields: FIELDS,
284 page,
285 limit,
286 paginationMode: 'PAGE_NUMBER',
287 });
288 if (total === null) {
289 total = data.totalNoticeCount ?? null;
290 log.info(`TED reports ${total ?? 'an unknown number of'} matching notices.`);
291 }
292 if (data.timedOut) log.warning('TED reported that this search timed out; results may be incomplete.');
293 const notices = Array.isArray(data.notices) ? data.notices : [];
294 if (notices.length === 0) break;
295
296 const budget = remainingBudget();
297 const batch = notices.map((n) => mapNotice(n, opts.language)).slice(0, Math.min(opts.maxItems - saved, budget));
298 if (batch.length === 0) {
299 log.info('Reached the maximum charge set for this run; stopping.');
300 break;
301 }
302 await Actor.pushData(batch);
303 saved += batch.length;
304 if (batch.length < notices.length && saved < opts.maxItems) {
305 log.info('Reached the maximum charge set for this run; stopping.');
306 break;
307 }
308 if (total !== null && page * limit >= total) break;
309 page += 1;
310}
311
312log.info(`Saved ${saved} notices.`);
313if (saved === 0) {
314 await Actor.setStatusMessage('No notices matched these filters in the selected period. Try more days or fewer filters.');
315}
316await Actor.exit();
