1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18import { createHash } from 'node:crypto';
19import { Actor, log } from 'apify';
20
21const USER_AGENT = 'foundry-actor/0.1 (Apify Actor uk-public-tenders; polite client)';
22const MIN_INTERVAL_MS = 1000;
23const PUSH_BATCH = 25;
24const lastRequestAt = new Map();
25
26
27const DEFAULTS = {
28 keywords: '',
29 stages: ['tender'],
30 sources: ['contracts-finder', 'find-a-tender'],
31 publishedWithinDays: 3,
32 onlyOpen: true,
33 onlyNewSinceLastRun: false,
34 cpvPrefixes: [],
35 minValue: 0,
36 maxItems: 10,
37};
38
39
40
41
42const SOURCES = {
43 'contracts-finder': {
44 label: 'Contracts Finder',
45 endpoint: 'https://www.contractsfinder.service.gov.uk/Published/Notices/OCDS/Search',
46 fromParam: 'publishedFrom',
47 serverStages: true,
48 noticeUrl: /^https:\/\/www\.contractsfinder\.service\.gov\.uk\/Notice\/([0-9a-f-]{36})/i,
49 },
50 'find-a-tender': {
51 label: 'Find a Tender',
52 endpoint: 'https://www.find-tender.service.gov.uk/api/1.0/ocdsReleasePackages',
53 fromParam: 'updatedFrom',
54 serverStages: false,
55 noticeUrl: /^https:\/\/www\.find-tender\.service\.gov\.uk\/Notice\/(\d{6}-\d{4})/,
56 },
57};
58const CLOSED_TENDER_STATUSES = new Set(['cancelled', 'withdrawn', 'unsuccessful', 'complete']);
59const STAGES = ['planning', 'tender', 'award'];
60
61class RateLimitedError extends Error {}
62
63const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
64
65async function politeGetJson(url, attempt = 1) {
66 const { host } = new URL(url);
67 const wait = (lastRequestAt.get(host) ?? 0) + MIN_INTERVAL_MS - Date.now();
68 if (wait > 0) await sleep(wait);
69 lastRequestAt.set(host, Date.now());
70
71 let res;
72 try {
73 res = await fetch(url, {
74 headers: { 'user-agent': USER_AGENT, accept: 'application/json' },
75 signal: AbortSignal.timeout(60_000),
76 });
77 } catch (err) {
78 if (attempt >= 4) throw new Error(`Network error after ${attempt} attempts: ${url} (${err.message})`);
79 await sleep(1000 * 2 ** attempt);
80 return politeGetJson(url, attempt + 1);
81 }
82 if (res.status === 429 || res.status >= 500) {
83 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
84 const retryAfter = Number(res.headers.get('retry-after'));
85 const delayMs = retryAfter > 0 ? Math.min(retryAfter, 60) * 1000 : 1000 * 2 ** attempt;
86 log.warning(`HTTP ${res.status} from ${host}; waiting ${Math.round(delayMs / 1000)} s before retrying.`);
87 await sleep(delayMs);
88 return politeGetJson(url, attempt + 1);
89 }
90 if (res.status === 403) {
91
92
93 const body = await res.text().catch(() => '');
94 if (/rate limit/i.test(body)) {
95 throw new RateLimitedError(`${host} reported that its request rate limit was reached (HTTP 403).`);
96 }
97 throw new Error(`HTTP 403 Forbidden: ${url}`);
98 }
99 if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}: ${url}`);
100 return res.json();
101}
102
103
104
105function remainingBudget() {
106 try {
107 const n = Actor.getChargingManager().calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
108 return Number.isFinite(n) ? n : Infinity;
109 } catch {
110 return Infinity;
111 }
112}
113
114
115
116function readInput(raw) {
117 const input = { ...DEFAULTS, ...Object.fromEntries(Object.entries(raw ?? {}).filter(([, v]) => v !== undefined && v !== null)) };
118
119 if (typeof input.keywords !== 'string') throw new Error('Input "keywords" must be a string (comma-separated words or phrases).');
120 const phrases = input.keywords.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
121
122 if (!Array.isArray(input.sources) || input.sources.length === 0) {
123 throw new Error('Input "sources" must list at least one of: "contracts-finder", "find-a-tender".');
124 }
125 for (const s of input.sources) {
126 if (!SOURCES[s]) throw new Error(`Input "sources" contains "${s}". Allowed values: "contracts-finder", "find-a-tender".`);
127 }
128
129 if (!Array.isArray(input.stages) || input.stages.length === 0) {
130 throw new Error('Input "stages" must list at least one of: "planning", "tender", "award".');
131 }
132 for (const s of input.stages) {
133 if (!STAGES.includes(s)) throw new Error(`Input "stages" contains "${s}". Allowed values: "planning", "tender", "award".`);
134 }
135
136 const days = Number(input.publishedWithinDays);
137 if (!Number.isInteger(days) || days < 1 || days > 90) throw new Error('Input "publishedWithinDays" must be a whole number from 1 to 90.');
138
139 const maxItems = Number(input.maxItems);
140 if (!Number.isInteger(maxItems) || maxItems < 1 || maxItems > 5000) throw new Error('Input "maxItems" must be a whole number from 1 to 5000.');
141
142 const minValue = Number(input.minValue);
143 if (!Number.isFinite(minValue) || minValue < 0) throw new Error('Input "minValue" must be 0 or a positive number (GBP).');
144
145 if (!Array.isArray(input.cpvPrefixes)) throw new Error('Input "cpvPrefixes" must be a list of CPV code prefixes, e.g. ["72", "48"].');
146 const cpvPrefixes = input.cpvPrefixes.map((c) => String(c).trim().replace(/-\d$/, '')).filter(Boolean);
147 for (const c of cpvPrefixes) {
148 if (!/^\d{1,8}$/.test(c)) throw new Error(`Input "cpvPrefixes" contains "${c}". Use 1 to 8 digits, e.g. "72" or "45210000".`);
149 }
150
151 if (typeof input.onlyNewSinceLastRun !== 'boolean') throw new Error('Input "onlyNewSinceLastRun" must be true or false.');
152
153 return {
154 phrases,
155 sources: [...new Set(input.sources)],
156 stages: [...new Set(input.stages)],
157 days,
158 onlyOpen: input.onlyOpen !== false,
159 cpvPrefixes,
160 minValue,
161 maxItems,
162 onlyNewSinceLastRun: input.onlyNewSinceLastRun,
163 };
164}
165
166
167
168
169function filterFingerprint(opts) {
170 return JSON.stringify({
171 phrases: [...opts.phrases].sort(), sources: [...opts.sources].sort(), stages: [...opts.stages].sort(),
172 onlyOpen: opts.onlyOpen, cpvPrefixes: [...opts.cpvPrefixes].sort(), minValue: opts.minValue,
173 });
174}
175
176
177
178const cleanLine = (s) => (typeof s === 'string' ? s.replace(/\s+/g, ' ').trim() || null : null);
179const cleanText = (s) => (typeof s === 'string' ? s.replace(/\r\n?/g, '\n').trim() || null : null);
180
181
182
183
184const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9-]+(?:\.[A-Z0-9-]+)*\.[A-Z]{2,}/gi;
185const INTL_PHONE_RE = /\+\d{1,3}[\s.-]?(?:\(0\)[\s.-]?)?\d[\d\s.-]{6,}\d/g;
186const LABELLED_PHONE_RE = /\b(tel(?:ephone)?|phone|mobile|mob)(\.?\s*(?:no\.?|number)?\s*[:.]?\s*)(\(?0[\d\s().-]{8,}\d)/gi;
187function redactContacts(text) {
188 if (typeof text !== 'string') return text;
189 return text.replace(EMAIL_RE, '[email removed]')
190 .replace(INTL_PHONE_RE, '[phone removed]')
191 .replace(LABELLED_PHONE_RE, (m, label, sep) => `${label}${sep}[phone removed]`);
192}
193const uniq = (arr) => [...new Set(arr.map((v) => (typeof v === 'string' ? cleanLine(v) : v)).filter((v) => v !== undefined && v !== null && v !== ''))];
194const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
195
196
197function daysUntil(value) {
198 const t = Date.parse(value ?? '');
199 return Number.isFinite(t) ? Math.ceil((t - Date.now()) / 86_400_000) : null;
200}
201
202function stageOf(tags) {
203 const t = (tags ?? []).join(' ');
204 if (/award|contract/i.test(t)) return 'award';
205 if (/tender/i.test(t)) return 'tender';
206 if (/planning/i.test(t)) return 'planning';
207 if (/implementation/i.test(t)) return 'implementation';
208 return null;
209}
210
211function allDocuments(rel) {
212 return [
213 ...(rel.tender?.documents ?? []),
214 ...(rel.planning?.documents ?? []),
215 ...(rel.awards ?? []).flatMap((a) => a.documents ?? []),
216 ...(rel.contracts ?? []).flatMap((c) => c.documents ?? []),
217 ];
218}
219
220function collectCpv(rel) {
221 const t = rel.tender ?? {};
222 const classes = [
223 t.classification,
224 ...(t.additionalClassifications ?? []),
225 ...(t.items ?? []).flatMap((i) => [i.classification, ...(i.additionalClassifications ?? [])]),
226 ...(rel.awards ?? []).flatMap((a) => (a.items ?? []).flatMap((i) => [i.classification, ...(i.additionalClassifications ?? [])])),
227 ].filter((c) => c && String(c.scheme ?? '').toUpperCase() === 'CPV' && c.id);
228 const seen = new Map();
229 for (const c of classes) if (!seen.has(String(c.id))) seen.set(String(c.id), c.description ?? null);
230 return { codes: [...seen.keys()], labels: [...seen.values()] };
231}
232
233function mapRelease(rel, sourceKey) {
234 const src = SOURCES[sourceKey];
235 const t = rel.tender ?? {};
236 const lots = t.lots ?? [];
237 const awards = rel.awards ?? [];
238 const contracts = rel.contracts ?? [];
239 const docs = allDocuments(rel);
240
241
242 let url = null;
243 let noticeId = null;
244 for (const d of docs) {
245 const m = typeof d.url === 'string' ? d.url.match(src.noticeUrl) : null;
246 if (m) {
247 url = m[0];
248 noticeId = m[1];
249 break;
250 }
251 }
252 if (!noticeId && sourceKey === 'find-a-tender' && /^\d{6}-\d{4}$/.test(rel.id ?? '')) {
253 noticeId = rel.id;
254 url = `https://www.find-tender.service.gov.uk/Notice/${rel.id}`;
255 }
256 if (!noticeId && sourceKey === 'contracts-finder') {
257 const m = String(rel.id ?? '').match(/^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i);
258 if (m) {
259 noticeId = m[1];
260 url = `https://www.contractsfinder.service.gov.uk/Notice/${m[1]}`;
261 }
262 }
263 const noticeType = docs.find((d) => d.noticeType)?.noticeType ?? null;
264
265 const buyerId = rel.buyer?.id;
266 const parties = rel.parties ?? [];
267 const buyerParty = parties.find((p) => buyerId && p.id === buyerId) ?? parties.find((p) => (p.roles ?? []).includes('buyer'));
268
269 const cpv = collectCpv(rel);
270 const value = t.value ?? lots.find((l) => l.value)?.value ?? null;
271 const awardValue = contracts.map((c) => c.value).find((v) => v && num(v.amount) !== null)
272 ?? awards.map((a) => a.value).find((v) => v && num(v.amount) !== null)
273 ?? null;
274 const deliveryAddresses = [...(t.items ?? []), ...awards.flatMap((a) => a.items ?? [])].flatMap((i) => i.deliveryAddresses ?? []);
275 const firstLotPeriod = lots.find((l) => l.contractPeriod)?.contractPeriod;
276 const contractPeriod = t.contractPeriod ?? firstLotPeriod ?? awards.find((a) => a.contractPeriod)?.contractPeriod
277 ?? contracts.find((c) => c.period)?.period ?? null;
278 const sme = t.suitability?.sme ?? lots.find((l) => l.suitability)?.suitability?.sme;
279 const vcse = t.suitability?.vcse ?? lots.find((l) => l.suitability)?.suitability?.vcse;
280
281 return {
282 id: noticeId ?? rel.id ?? null,
283 ocid: rel.ocid ?? null,
284 source: sourceKey,
285 stage: stageOf(rel.tag),
286 noticeType,
287 title: redactContacts(cleanLine(t.title)),
288 description: redactContacts(cleanText(t.description) ?? cleanText(rel.description)),
289 buyerName: cleanLine(rel.buyer?.name) ?? cleanLine(buyerParty?.name),
290 buyerLocality: cleanLine(buyerParty?.address?.locality),
291 buyerCountry: cleanLine(buyerParty?.address?.countryName) ?? cleanLine(buyerParty?.address?.country),
292 status: t.status ?? null,
293 procurementMethod: t.procurementMethod ?? null,
294 procurementMethodDetails: t.procurementMethodDetails ?? null,
295 mainCategory: t.mainProcurementCategory ?? awards.find((a) => a.mainProcurementCategory)?.mainProcurementCategory ?? null,
296 cpvCodes: cpv.codes,
297 cpvLabels: cpv.labels,
298 valueAmount: num(value?.amount),
299 valueAmountGross: num(value?.amountGross),
300 minValueAmount: num(t.minValue?.amount),
301 valueCurrency: value?.currency ?? (num(t.minValue?.amount) !== null ? t.minValue.currency : null) ?? null,
302 awardedValue: num(awardValue?.amount),
303 awardedValueCurrency: awardValue?.currency ?? null,
304 awardedSuppliers: uniq(awards.flatMap((a) => (a.suppliers ?? []).map((s) => s.name))),
305 publishedAt: rel.date ?? t.datePublished ?? null,
306 tenderDeadline: t.tenderPeriod?.endDate ?? lots.find((l) => l.tenderPeriod?.endDate)?.tenderPeriod.endDate ?? null,
307 participationDeadline: t.expressionOfInterestDeadline ?? null,
308 daysToDeadline: daysUntil(t.tenderPeriod?.endDate ?? lots.find((l) => l.tenderPeriod?.endDate)?.tenderPeriod.endDate),
309 contractStart: contractPeriod?.startDate ?? null,
310 contractEnd: contractPeriod?.endDate ?? null,
311 deliveryRegions: uniq(deliveryAddresses.map((a) => a.region)),
312 suitableForSme: typeof sme === 'boolean' ? sme : null,
313 suitableForVcse: typeof vcse === 'boolean' ? vcse : null,
314 url,
315 };
316}
317
318
319
320function passesFilters(item, opts, nowMs) {
321 if (!opts.stages.includes(item.stage)) return false;
322 if (opts.phrases.length) {
323 const hay = `${item.title ?? ''}\n${item.description ?? ''}`.toLowerCase();
324 if (!opts.phrases.some((p) => hay.includes(p))) return false;
325 }
326 if (opts.cpvPrefixes.length) {
327 if (!item.cpvCodes.some((c) => opts.cpvPrefixes.some((p) => c.startsWith(p)))) return false;
328 }
329 if (opts.minValue > 0) {
330 const v = item.valueAmount ?? item.valueAmountGross ?? item.awardedValue;
331 if (v === null || v < opts.minValue) return false;
332 }
333 if (opts.onlyOpen && item.stage === 'tender') {
334 const status = String(item.status ?? '').toLowerCase();
335 if (CLOSED_TENDER_STATUSES.has(status)) return false;
336 const deadlines = [item.tenderDeadline, item.participationDeadline].map((d) => Date.parse(d ?? '')).filter(Number.isFinite);
337 if (deadlines.length) {
338 if (Math.max(...deadlines) < nowMs) return false;
339 } else if (status !== 'active') {
340 return false;
341 }
342 }
343 return true;
344}
345
346
347
348const MAX_PAGES_PER_SOURCE = 500;
349
350async function* releaseStream(sourceKey, fromIso, stages, pageSize) {
351 const src = SOURCES[sourceKey];
352 const params = new URLSearchParams({ [src.fromParam]: fromIso, limit: String(pageSize) });
353 if (src.serverStages) params.set('stages', stages.join(','));
354 let url = `${src.endpoint}?${params}`;
355 const host = new URL(src.endpoint).host;
356 for (let page = 1; url && page <= MAX_PAGES_PER_SOURCE; page += 1) {
357 const data = await politeGetJson(url);
358 const releases = Array.isArray(data?.releases) ? data.releases : [];
359 for (const rel of releases) yield rel;
360 const next = data?.links?.next;
361 url = releases.length && typeof next === 'string' && new URL(next).host === host ? next : null;
362 }
363}
364
365
366
367await Actor.init();
368
369const opts = readInput(await Actor.getInput());
370const nowMs = Date.now();
371
372
373
374
375
376
377const STOP_AFTER_KNOWN = 100;
378const MAX_MEMORY = 100_000;
379const monitor = opts.onlyNewSinceLastRun;
380const stateStore = monitor ? await Actor.openKeyValueStore('uk-public-tenders-seen') : null;
381const stateKey = monitor ? `seen-${createHash('sha256').update(filterFingerprint(opts)).digest('hex').slice(0, 40)}` : null;
382let delivered = {};
383let prevComplete = false;
384if (monitor) {
385 const raw = (await stateStore.getValue(stateKey)) ?? {};
386 const items = raw.v === 2 ? (raw.items ?? {}) : raw;
387 prevComplete = raw.v === 2 && raw.complete === true;
388 const keepAfterMs = nowMs - (Math.max(opts.days, 30) + 2) * 86_400_000;
389 for (const [k, v] of Object.entries(items)) {
390 if (Array.isArray(v) && (Date.parse(v[1] ?? '') || 0) >= keepAfterMs) delivered[k] = v;
391 }
392 log.info(`Monitor mode: ${Object.keys(delivered).length} notice(s) delivered by earlier runs with these filters will be skipped unless updated.`);
393}
394async function saveMemory(complete) {
395 let entries = Object.entries(delivered);
396 if (entries.length > MAX_MEMORY) {
397 entries = entries.sort((a, b) => (Date.parse(b[1][1] ?? '') || 0) - (Date.parse(a[1][1] ?? '') || 0)).slice(0, MAX_MEMORY);
398 delivered = Object.fromEntries(entries);
399 }
400 await stateStore.setValue(stateKey, { v: 2, complete, items: delivered });
401}
402
403
404const keyOf = (item) => (item.stage === 'award' ? `${item.source}:${item.id}` : `${item.source}:${item.ocid ?? item.id}:${item.stage}`);
405const versionOf = (item) => `${item.id ?? ''}|${item.publishedAt ?? ''}`;
406
407const fromIso = new Date(nowMs - opts.days * 86_400_000).toISOString().slice(0, 19);
408const filtering = opts.phrases.length > 0 || opts.cpvPrefixes.length > 0 || opts.minValue > 0;
409
410const pageSizeFor = (key) => (filtering || !SOURCES[key].serverStages ? 100 : Math.min(100, Math.max(10, opts.maxItems + 5)));
411
412log.info(`Searching ${opts.sources.join(' + ')} for ${opts.stages.join('/')} notices published or updated since ${fromIso}Z.`);
413
414const streams = opts.sources.map((key) => ({ key, it: releaseStream(key, fromIso, opts.stages, pageSizeFor(key)), head: null, done: false, scanned: 0 }));
415const sourceErrors = [];
416
417async function advance(st) {
418 try {
419 const r = await st.it.next();
420 if (r.done) {
421 st.done = true;
422 st.head = null;
423 } else {
424 st.head = r.value;
425 st.scanned += 1;
426 }
427 } catch (err) {
428 st.done = true;
429 st.head = null;
430 const label = SOURCES[st.key].label;
431 if (err instanceof RateLimitedError) {
432 log.warning(`${label}: ${err.message} Stopping this source and keeping the results saved so far.`);
433 } else {
434 log.error(`${label}: ${err.message}`);
435 }
436 sourceErrors.push(`${label}: ${err.message}`);
437 }
438}
439
440let saved = 0;
441let budgetReached = false;
442const buffer = [];
443const seen = new Set();
444
445
446async function flush() {
447 if (!buffer.length) return;
448 const budget = remainingBudget();
449 const allowed = Math.max(0, Math.min(buffer.length, opts.maxItems - saved, budget));
450 const batch = buffer.splice(0).slice(0, allowed);
451 if (batch.length) {
452 await Actor.pushData(batch);
453 saved += batch.length;
454 if (monitor) {
455
456 for (const it of batch) delivered[keyOf(it)] = [versionOf(it), it.publishedAt];
457 await saveMemory(false);
458 }
459 }
460 if (budget - batch.length <= 0) budgetReached = true;
461}
462
463for (const st of streams) await advance(st);
464
465let knownStreak = 0;
466let stoppedEarly = false;
467while (!budgetReached && saved + buffer.length < opts.maxItems) {
468 const live = streams.filter((s) => !s.done && s.head);
469 if (!live.length) break;
470
471 live.sort((a, b) => (Date.parse(b.head.date ?? '') || 0) - (Date.parse(a.head.date ?? '') || 0));
472 const st = live[0];
473 const rel = st.head;
474 await advance(st);
475
476 const item = mapRelease(rel, st.key);
477 const key = keyOf(item);
478 if (seen.has(key)) continue;
479 seen.add(key);
480 if (!passesFilters(item, opts, nowMs)) continue;
481 if (monitor && delivered[key]?.[0] === versionOf(item)) {
482 knownStreak += 1;
483 if (prevComplete && knownStreak >= STOP_AFTER_KNOWN) {
484 stoppedEarly = true;
485 break;
486 }
487 continue;
488 }
489 knownStreak = 0;
490
491 buffer.push(item);
492 if (buffer.length >= PUSH_BATCH || buffer.length >= remainingBudget()) await flush();
493}
494await flush();
495if (budgetReached) log.info('Reached the maximum charge set for this run; stopping.');
496if (stoppedEarly) log.info(`Monitor mode: reached ${STOP_AFTER_KNOWN} already-delivered notices in a row; the rest of the window was covered by earlier runs.`);
497if (monitor) {
498
499 await saveMemory(stoppedEarly || (!budgetReached && saved < opts.maxItems && sourceErrors.length === 0));
500}
501
502const scanned = streams.map((s) => `${SOURCES[s.key].label}: ${s.scanned} scanned`).join(', ');
503log.info(`Saved ${saved} notices (${scanned}).`);
504
505if (sourceErrors.length && saved === 0) {
506 throw new Error(`No notices saved because a source failed. ${sourceErrors.join(' | ')}`);
507}
508if (sourceErrors.length) {
509 await Actor.setStatusMessage(`Saved ${saved} notices; some sources stopped early: ${sourceErrors.join(' | ')}`);
510} else if (saved === 0 && monitor) {
511 await Actor.setStatusMessage('No new or updated notices since the last run with these filters.');
512} else if (saved === 0) {
513 await Actor.setStatusMessage('No notices matched these filters in the selected period. Try more days or fewer filters.');
514}
515
516await Actor.exit();
