1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18import { Actor, log } from 'apify';
19
20const USER_AGENT = 'foundry-actor/0.1 (Apify Actor uk-public-tenders; polite client)';
21const MIN_INTERVAL_MS = 1000;
22const PUSH_BATCH = 25;
23const lastRequestAt = new Map();
24
25
26const DEFAULTS = {
27 keywords: '',
28 stages: ['tender'],
29 sources: ['contracts-finder', 'find-a-tender'],
30 publishedWithinDays: 3,
31 onlyOpen: true,
32 cpvPrefixes: [],
33 minValue: 0,
34 maxItems: 10,
35};
36
37
38
39
40const SOURCES = {
41 'contracts-finder': {
42 label: 'Contracts Finder',
43 endpoint: 'https://www.contractsfinder.service.gov.uk/Published/Notices/OCDS/Search',
44 fromParam: 'publishedFrom',
45 serverStages: true,
46 noticeUrl: /^https:\/\/www\.contractsfinder\.service\.gov\.uk\/Notice\/([0-9a-f-]{36})/i,
47 },
48 'find-a-tender': {
49 label: 'Find a Tender',
50 endpoint: 'https://www.find-tender.service.gov.uk/api/1.0/ocdsReleasePackages',
51 fromParam: 'updatedFrom',
52 serverStages: false,
53 noticeUrl: /^https:\/\/www\.find-tender\.service\.gov\.uk\/Notice\/(\d{6}-\d{4})/,
54 },
55};
56const CLOSED_TENDER_STATUSES = new Set(['cancelled', 'withdrawn', 'unsuccessful', 'complete']);
57const STAGES = ['planning', 'tender', 'award'];
58
59class RateLimitedError extends Error {}
60
61const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
62
63async function politeGetJson(url, attempt = 1) {
64 const { host } = new URL(url);
65 const wait = (lastRequestAt.get(host) ?? 0) + MIN_INTERVAL_MS - Date.now();
66 if (wait > 0) await sleep(wait);
67 lastRequestAt.set(host, Date.now());
68
69 let res;
70 try {
71 res = await fetch(url, {
72 headers: { 'user-agent': USER_AGENT, accept: 'application/json' },
73 signal: AbortSignal.timeout(60_000),
74 });
75 } catch (err) {
76 if (attempt >= 4) throw new Error(`Network error after ${attempt} attempts: ${url} (${err.message})`);
77 await sleep(1000 * 2 ** attempt);
78 return politeGetJson(url, attempt + 1);
79 }
80 if (res.status === 429 || res.status >= 500) {
81 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
82 const retryAfter = Number(res.headers.get('retry-after'));
83 const delayMs = retryAfter > 0 ? Math.min(retryAfter, 60) * 1000 : 1000 * 2 ** attempt;
84 log.warning(`HTTP ${res.status} from ${host}; waiting ${Math.round(delayMs / 1000)} s before retrying.`);
85 await sleep(delayMs);
86 return politeGetJson(url, attempt + 1);
87 }
88 if (res.status === 403) {
89
90
91 const body = await res.text().catch(() => '');
92 if (/rate limit/i.test(body)) {
93 throw new RateLimitedError(`${host} reported that its request rate limit was reached (HTTP 403).`);
94 }
95 throw new Error(`HTTP 403 Forbidden: ${url}`);
96 }
97 if (!res.ok) throw new Error(`HTTP ${res.status} ${res.statusText}: ${url}`);
98 return res.json();
99}
100
101
102
103function remainingBudget() {
104 try {
105 const n = Actor.getChargingManager().calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
106 return Number.isFinite(n) ? n : Infinity;
107 } catch {
108 return Infinity;
109 }
110}
111
112
113
114function readInput(raw) {
115 const input = { ...DEFAULTS, ...Object.fromEntries(Object.entries(raw ?? {}).filter(([, v]) => v !== undefined && v !== null)) };
116
117 if (typeof input.keywords !== 'string') throw new Error('Input "keywords" must be a string (comma-separated words or phrases).');
118 const phrases = input.keywords.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
119
120 if (!Array.isArray(input.sources) || input.sources.length === 0) {
121 throw new Error('Input "sources" must list at least one of: "contracts-finder", "find-a-tender".');
122 }
123 for (const s of input.sources) {
124 if (!SOURCES[s]) throw new Error(`Input "sources" contains "${s}". Allowed values: "contracts-finder", "find-a-tender".`);
125 }
126
127 if (!Array.isArray(input.stages) || input.stages.length === 0) {
128 throw new Error('Input "stages" must list at least one of: "planning", "tender", "award".');
129 }
130 for (const s of input.stages) {
131 if (!STAGES.includes(s)) throw new Error(`Input "stages" contains "${s}". Allowed values: "planning", "tender", "award".`);
132 }
133
134 const days = Number(input.publishedWithinDays);
135 if (!Number.isInteger(days) || days < 1 || days > 90) throw new Error('Input "publishedWithinDays" must be a whole number from 1 to 90.');
136
137 const maxItems = Number(input.maxItems);
138 if (!Number.isInteger(maxItems) || maxItems < 1 || maxItems > 5000) throw new Error('Input "maxItems" must be a whole number from 1 to 5000.');
139
140 const minValue = Number(input.minValue);
141 if (!Number.isFinite(minValue) || minValue < 0) throw new Error('Input "minValue" must be 0 or a positive number (GBP).');
142
143 if (!Array.isArray(input.cpvPrefixes)) throw new Error('Input "cpvPrefixes" must be a list of CPV code prefixes, e.g. ["72", "48"].');
144 const cpvPrefixes = input.cpvPrefixes.map((c) => String(c).trim().replace(/-\d$/, '')).filter(Boolean);
145 for (const c of cpvPrefixes) {
146 if (!/^\d{1,8}$/.test(c)) throw new Error(`Input "cpvPrefixes" contains "${c}". Use 1 to 8 digits, e.g. "72" or "45210000".`);
147 }
148
149 return {
150 phrases,
151 sources: [...new Set(input.sources)],
152 stages: [...new Set(input.stages)],
153 days,
154 onlyOpen: input.onlyOpen !== false,
155 cpvPrefixes,
156 minValue,
157 maxItems,
158 };
159}
160
161
162
163const cleanLine = (s) => (typeof s === 'string' ? s.replace(/\s+/g, ' ').trim() || null : null);
164const cleanText = (s) => (typeof s === 'string' ? s.replace(/\r\n?/g, '\n').trim() || null : null);
165
166
167
168
169const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9-]+(?:\.[A-Z0-9-]+)*\.[A-Z]{2,}/gi;
170const INTL_PHONE_RE = /\+\d{1,3}[\s.-]?(?:\(0\)[\s.-]?)?\d[\d\s.-]{6,}\d/g;
171const LABELLED_PHONE_RE = /\b(tel(?:ephone)?|phone|mobile|mob)(\.?\s*(?:no\.?|number)?\s*[:.]?\s*)(\(?0[\d\s().-]{8,}\d)/gi;
172function redactContacts(text) {
173 if (typeof text !== 'string') return text;
174 return text.replace(EMAIL_RE, '[email removed]')
175 .replace(INTL_PHONE_RE, '[phone removed]')
176 .replace(LABELLED_PHONE_RE, (m, label, sep) => `${label}${sep}[phone removed]`);
177}
178const uniq = (arr) => [...new Set(arr.map((v) => (typeof v === 'string' ? cleanLine(v) : v)).filter((v) => v !== undefined && v !== null && v !== ''))];
179const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : null);
180
181function stageOf(tags) {
182 const t = (tags ?? []).join(' ');
183 if (/award|contract/i.test(t)) return 'award';
184 if (/tender/i.test(t)) return 'tender';
185 if (/planning/i.test(t)) return 'planning';
186 if (/implementation/i.test(t)) return 'implementation';
187 return null;
188}
189
190function allDocuments(rel) {
191 return [
192 ...(rel.tender?.documents ?? []),
193 ...(rel.planning?.documents ?? []),
194 ...(rel.awards ?? []).flatMap((a) => a.documents ?? []),
195 ...(rel.contracts ?? []).flatMap((c) => c.documents ?? []),
196 ];
197}
198
199function collectCpv(rel) {
200 const t = rel.tender ?? {};
201 const classes = [
202 t.classification,
203 ...(t.additionalClassifications ?? []),
204 ...(t.items ?? []).flatMap((i) => [i.classification, ...(i.additionalClassifications ?? [])]),
205 ...(rel.awards ?? []).flatMap((a) => (a.items ?? []).flatMap((i) => [i.classification, ...(i.additionalClassifications ?? [])])),
206 ].filter((c) => c && String(c.scheme ?? '').toUpperCase() === 'CPV' && c.id);
207 const seen = new Map();
208 for (const c of classes) if (!seen.has(String(c.id))) seen.set(String(c.id), c.description ?? null);
209 return { codes: [...seen.keys()], labels: [...seen.values()] };
210}
211
212function mapRelease(rel, sourceKey) {
213 const src = SOURCES[sourceKey];
214 const t = rel.tender ?? {};
215 const lots = t.lots ?? [];
216 const awards = rel.awards ?? [];
217 const contracts = rel.contracts ?? [];
218 const docs = allDocuments(rel);
219
220
221 let url = null;
222 let noticeId = null;
223 for (const d of docs) {
224 const m = typeof d.url === 'string' ? d.url.match(src.noticeUrl) : null;
225 if (m) {
226 url = m[0];
227 noticeId = m[1];
228 break;
229 }
230 }
231 if (!noticeId && sourceKey === 'find-a-tender' && /^\d{6}-\d{4}$/.test(rel.id ?? '')) {
232 noticeId = rel.id;
233 url = `https://www.find-tender.service.gov.uk/Notice/${rel.id}`;
234 }
235 if (!noticeId && sourceKey === 'contracts-finder') {
236 const m = String(rel.id ?? '').match(/^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i);
237 if (m) {
238 noticeId = m[1];
239 url = `https://www.contractsfinder.service.gov.uk/Notice/${m[1]}`;
240 }
241 }
242 const noticeType = docs.find((d) => d.noticeType)?.noticeType ?? null;
243
244 const buyerId = rel.buyer?.id;
245 const parties = rel.parties ?? [];
246 const buyerParty = parties.find((p) => buyerId && p.id === buyerId) ?? parties.find((p) => (p.roles ?? []).includes('buyer'));
247
248 const cpv = collectCpv(rel);
249 const value = t.value ?? lots.find((l) => l.value)?.value ?? null;
250 const awardValue = contracts.map((c) => c.value).find((v) => v && num(v.amount) !== null)
251 ?? awards.map((a) => a.value).find((v) => v && num(v.amount) !== null)
252 ?? null;
253 const deliveryAddresses = [...(t.items ?? []), ...awards.flatMap((a) => a.items ?? [])].flatMap((i) => i.deliveryAddresses ?? []);
254 const firstLotPeriod = lots.find((l) => l.contractPeriod)?.contractPeriod;
255 const contractPeriod = t.contractPeriod ?? firstLotPeriod ?? awards.find((a) => a.contractPeriod)?.contractPeriod
256 ?? contracts.find((c) => c.period)?.period ?? null;
257 const sme = t.suitability?.sme ?? lots.find((l) => l.suitability)?.suitability?.sme;
258 const vcse = t.suitability?.vcse ?? lots.find((l) => l.suitability)?.suitability?.vcse;
259
260 return {
261 id: noticeId ?? rel.id ?? null,
262 ocid: rel.ocid ?? null,
263 source: sourceKey,
264 stage: stageOf(rel.tag),
265 noticeType,
266 title: redactContacts(cleanLine(t.title)),
267 description: redactContacts(cleanText(t.description) ?? cleanText(rel.description)),
268 buyerName: cleanLine(rel.buyer?.name) ?? cleanLine(buyerParty?.name),
269 buyerLocality: cleanLine(buyerParty?.address?.locality),
270 buyerCountry: cleanLine(buyerParty?.address?.countryName) ?? cleanLine(buyerParty?.address?.country),
271 status: t.status ?? null,
272 procurementMethod: t.procurementMethod ?? null,
273 procurementMethodDetails: t.procurementMethodDetails ?? null,
274 mainCategory: t.mainProcurementCategory ?? awards.find((a) => a.mainProcurementCategory)?.mainProcurementCategory ?? null,
275 cpvCodes: cpv.codes,
276 cpvLabels: cpv.labels,
277 valueAmount: num(value?.amount),
278 valueAmountGross: num(value?.amountGross),
279 minValueAmount: num(t.minValue?.amount),
280 valueCurrency: value?.currency ?? (num(t.minValue?.amount) !== null ? t.minValue.currency : null) ?? null,
281 awardedValue: num(awardValue?.amount),
282 awardedValueCurrency: awardValue?.currency ?? null,
283 awardedSuppliers: uniq(awards.flatMap((a) => (a.suppliers ?? []).map((s) => s.name))),
284 publishedAt: rel.date ?? t.datePublished ?? null,
285 tenderDeadline: t.tenderPeriod?.endDate ?? lots.find((l) => l.tenderPeriod?.endDate)?.tenderPeriod.endDate ?? null,
286 participationDeadline: t.expressionOfInterestDeadline ?? null,
287 contractStart: contractPeriod?.startDate ?? null,
288 contractEnd: contractPeriod?.endDate ?? null,
289 deliveryRegions: uniq(deliveryAddresses.map((a) => a.region)),
290 suitableForSme: typeof sme === 'boolean' ? sme : null,
291 suitableForVcse: typeof vcse === 'boolean' ? vcse : null,
292 url,
293 };
294}
295
296
297
298function passesFilters(item, opts, nowMs) {
299 if (!opts.stages.includes(item.stage)) return false;
300 if (opts.phrases.length) {
301 const hay = `${item.title ?? ''}\n${item.description ?? ''}`.toLowerCase();
302 if (!opts.phrases.some((p) => hay.includes(p))) return false;
303 }
304 if (opts.cpvPrefixes.length) {
305 if (!item.cpvCodes.some((c) => opts.cpvPrefixes.some((p) => c.startsWith(p)))) return false;
306 }
307 if (opts.minValue > 0) {
308 const v = item.valueAmount ?? item.valueAmountGross ?? item.awardedValue;
309 if (v === null || v < opts.minValue) return false;
310 }
311 if (opts.onlyOpen && item.stage === 'tender') {
312 const status = String(item.status ?? '').toLowerCase();
313 if (CLOSED_TENDER_STATUSES.has(status)) return false;
314 const deadlines = [item.tenderDeadline, item.participationDeadline].map((d) => Date.parse(d ?? '')).filter(Number.isFinite);
315 if (deadlines.length) {
316 if (Math.max(...deadlines) < nowMs) return false;
317 } else if (status !== 'active') {
318 return false;
319 }
320 }
321 return true;
322}
323
324
325
326const MAX_PAGES_PER_SOURCE = 500;
327
328async function* releaseStream(sourceKey, fromIso, stages, pageSize) {
329 const src = SOURCES[sourceKey];
330 const params = new URLSearchParams({ [src.fromParam]: fromIso, limit: String(pageSize) });
331 if (src.serverStages) params.set('stages', stages.join(','));
332 let url = `${src.endpoint}?${params}`;
333 const host = new URL(src.endpoint).host;
334 for (let page = 1; url && page <= MAX_PAGES_PER_SOURCE; page += 1) {
335 const data = await politeGetJson(url);
336 const releases = Array.isArray(data?.releases) ? data.releases : [];
337 for (const rel of releases) yield rel;
338 const next = data?.links?.next;
339 url = releases.length && typeof next === 'string' && new URL(next).host === host ? next : null;
340 }
341}
342
343
344
345await Actor.init();
346
347const opts = readInput(await Actor.getInput());
348const nowMs = Date.now();
349const fromIso = new Date(nowMs - opts.days * 86_400_000).toISOString().slice(0, 19);
350const filtering = opts.phrases.length > 0 || opts.cpvPrefixes.length > 0 || opts.minValue > 0;
351
352const pageSizeFor = (key) => (filtering || !SOURCES[key].serverStages ? 100 : Math.min(100, Math.max(10, opts.maxItems + 5)));
353
354log.info(`Searching ${opts.sources.join(' + ')} for ${opts.stages.join('/')} notices published or updated since ${fromIso}Z.`);
355
356const streams = opts.sources.map((key) => ({ key, it: releaseStream(key, fromIso, opts.stages, pageSizeFor(key)), head: null, done: false, scanned: 0 }));
357const sourceErrors = [];
358
359async function advance(st) {
360 try {
361 const r = await st.it.next();
362 if (r.done) {
363 st.done = true;
364 st.head = null;
365 } else {
366 st.head = r.value;
367 st.scanned += 1;
368 }
369 } catch (err) {
370 st.done = true;
371 st.head = null;
372 const label = SOURCES[st.key].label;
373 if (err instanceof RateLimitedError) {
374 log.warning(`${label}: ${err.message} Stopping this source and keeping the results saved so far.`);
375 } else {
376 log.error(`${label}: ${err.message}`);
377 }
378 sourceErrors.push(`${label}: ${err.message}`);
379 }
380}
381
382let saved = 0;
383let budgetReached = false;
384const buffer = [];
385const seen = new Set();
386
387
388async function flush() {
389 if (!buffer.length) return;
390 const budget = remainingBudget();
391 const allowed = Math.max(0, Math.min(buffer.length, opts.maxItems - saved, budget));
392 const batch = buffer.splice(0).slice(0, allowed);
393 if (batch.length) {
394 await Actor.pushData(batch);
395 saved += batch.length;
396 }
397 if (budget - batch.length <= 0) budgetReached = true;
398}
399
400for (const st of streams) await advance(st);
401
402while (!budgetReached && saved + buffer.length < opts.maxItems) {
403 const live = streams.filter((s) => !s.done && s.head);
404 if (!live.length) break;
405
406 live.sort((a, b) => (Date.parse(b.head.date ?? '') || 0) - (Date.parse(a.head.date ?? '') || 0));
407 const st = live[0];
408 const rel = st.head;
409 await advance(st);
410
411 const item = mapRelease(rel, st.key);
412
413
414 const key = item.stage === 'award' ? `${item.source}:${item.id}` : `${item.source}:${item.ocid ?? item.id}:${item.stage}`;
415 if (seen.has(key)) continue;
416 seen.add(key);
417 if (!passesFilters(item, opts, nowMs)) continue;
418
419 buffer.push(item);
420 if (buffer.length >= PUSH_BATCH || buffer.length >= remainingBudget()) await flush();
421}
422await flush();
423if (budgetReached) log.info('Reached the maximum charge set for this run; stopping.');
424
425const scanned = streams.map((s) => `${SOURCES[s.key].label}: ${s.scanned} scanned`).join(', ');
426log.info(`Saved ${saved} notices (${scanned}).`);
427
428if (sourceErrors.length && saved === 0) {
429 throw new Error(`No notices saved because a source failed. ${sourceErrors.join(' | ')}`);
430}
431if (sourceErrors.length) {
432 await Actor.setStatusMessage(`Saved ${saved} notices; some sources stopped early: ${sourceErrors.join(' | ')}`);
433} else if (saved === 0) {
434 await Actor.setStatusMessage('No notices matched these filters in the selected period. Try more days or fewer filters.');
435}
436
437await Actor.exit();
