1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22import { createHash } from 'node:crypto';
23import { Actor, log } from 'apify';
24import { parse } from 'csv-parse/sync';
25import iconv from 'iconv-lite';
26
27const USER_AGENT = 'foundry-actor/0.1 (Apify Actor sam-gov-contract-notices; polite client)';
28const EXTRACT_URL = 'https://sam.gov/api/prod/fileextractservices/v1/api/download/Contract%20Opportunities/datagov/ContractOpportunitiesFullCSV.csv?privacy=Public';
29const MIN_INTERVAL_MS = 1000;
30const CHUNK_BYTES = 8 * 1024 * 1024;
31const HARD_CAP_BYTES = 150 * 1024 * 1024;
32const lastRequestAt = new Map();
33
34const NOTICE_TYPES = ['Solicitation', 'Presolicitation', 'Combined Synopsis/Solicitation', 'Sources Sought', 'Special Notice', 'Sale of Surplus Property', 'Award Notice', 'Justification'];
35const SET_ASIDE_CODES = ['SBA', 'SBP', '8A', '8AN', 'HZC', 'SDVOSBC', 'WOSB', 'WOSBSS', 'ISBEE', 'IEE', 'VSA', 'BICiv', 'NONE'];
36
37const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
38
39async function politeFetch(url, headers, attempt = 1) {
40 const { host } = new URL(url);
41 const wait = (lastRequestAt.get(host) ?? 0) + MIN_INTERVAL_MS - Date.now();
42 if (wait > 0) await sleep(wait);
43 lastRequestAt.set(host, Date.now());
44
45 let res;
46 try {
47 res = await fetch(url, { headers: { 'user-agent': USER_AGENT, ...headers }, redirect: 'manual', signal: AbortSignal.timeout(60_000) });
48 } catch (err) {
49 if (attempt >= 4) throw new Error(`Network error after ${attempt} attempts: ${url} (${err.message})`);
50 await sleep(1000 * 2 ** attempt);
51 return politeFetch(url, headers, attempt + 1);
52 }
53 if (res.status === 429 || res.status >= 500) {
54 if (attempt >= 4) throw new Error(`HTTP ${res.status} after ${attempt} attempts: ${url}`);
55 const retryAfter = Number(res.headers.get('retry-after'));
56 const delayMs = retryAfter > 0 ? Math.min(retryAfter, 60) * 1000 : 1000 * 2 ** attempt;
57 log.warning(`HTTP ${res.status} from ${host}; waiting ${Math.round(delayMs / 1000)} s before retrying.`);
58 await sleep(delayMs);
59 return politeFetch(url, headers, attempt + 1);
60 }
61 return res;
62}
63
64
65
66function remainingBudget() {
67 try {
68 const n = Actor.getChargingManager().calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
69 return Number.isFinite(n) ? n : Infinity;
70 } catch {
71 return Infinity;
72 }
73}
74
75
76
77const DEFAULTS = {
78 daysBack: 2,
79 keyword: '',
80 naicsPrefixes: [],
81 setAsideCodes: [],
82 agency: '',
83 noticeTypes: [],
84 onlyOpenDeadline: false,
85 onlySinceLastRun: false,
86 maxItems: 20,
87};
88
89function readInput(raw) {
90 const input = { ...DEFAULTS, ...Object.fromEntries(Object.entries(raw ?? {}).filter(([, v]) => v !== undefined && v !== null)) };
91
92 const daysBack = Number(input.daysBack);
93 if (!Number.isInteger(daysBack) || daysBack < 1 || daysBack > 30) throw new Error('Input "daysBack" must be a whole number from 1 to 30.');
94
95 if (typeof input.keyword !== 'string') throw new Error('Input "keyword" must be a string.');
96 const keyword = input.keyword.trim().toLowerCase();
97
98 if (!Array.isArray(input.naicsPrefixes)) throw new Error('Input "naicsPrefixes" must be a list of NAICS code prefixes, e.g. ["541512"].');
99 const naicsPrefixes = input.naicsPrefixes.map((c) => String(c).trim()).filter(Boolean);
100 for (const c of naicsPrefixes) {
101 if (!/^\d{1,6}$/.test(c)) throw new Error(`Input "naicsPrefixes" contains "${c}". Use 1 to 6 digits, e.g. "541512".`);
102 }
103
104 if (!Array.isArray(input.setAsideCodes)) throw new Error('Input "setAsideCodes" must be a list of set-aside codes.');
105 for (const c of input.setAsideCodes) {
106 if (!SET_ASIDE_CODES.includes(c)) throw new Error(`Input "setAsideCodes" contains "${c}". Allowed values: ${SET_ASIDE_CODES.join(', ')}.`);
107 }
108
109 if (typeof input.agency !== 'string') throw new Error('Input "agency" must be a string.');
110 const agency = input.agency.trim().toLowerCase();
111
112 if (!Array.isArray(input.noticeTypes)) throw new Error('Input "noticeTypes" must be a list of notice types.');
113 for (const t of input.noticeTypes) {
114 if (!NOTICE_TYPES.includes(t)) throw new Error(`Input "noticeTypes" contains "${t}". Allowed values: ${NOTICE_TYPES.join(', ')}.`);
115 }
116
117 const maxItems = Number(input.maxItems);
118 if (!Number.isInteger(maxItems) || maxItems < 1 || maxItems > 10000) throw new Error('Input "maxItems" must be a whole number from 1 to 10000.');
119
120 return {
121 daysBack,
122 keyword,
123 naicsPrefixes,
124 setAsideCodes: [...new Set(input.setAsideCodes)],
125 agency,
126 noticeTypes: [...new Set(input.noticeTypes)],
127 onlyOpenDeadline: input.onlyOpenDeadline === true,
128 onlySinceLastRun: input.onlySinceLastRun === true,
129 maxItems,
130 };
131}
132
133
134
135
136function filterFingerprint(opts) {
137 return JSON.stringify({
138 keyword: opts.keyword, naicsPrefixes: [...opts.naicsPrefixes].sort(),
139 setAsideCodes: [...opts.setAsideCodes].sort(), agency: opts.agency,
140 noticeTypes: [...opts.noticeTypes].sort(), onlyOpenDeadline: opts.onlyOpenDeadline,
141 });
142}
143
144
145function daysUntil(value, nowMs) {
146 const t = Date.parse(value ?? '');
147 return Number.isFinite(t) ? Math.ceil((t - nowMs) / 86_400_000) : null;
148}
149
150
151
152
153async function resolveExtractUrl() {
154 const res = await politeFetch(EXTRACT_URL, { accept: '*/*' });
155 if (res.status !== 303) throw new Error(`Expected SAM.gov's download endpoint to redirect (HTTP 303) but got HTTP ${res.status}.`);
156 const location = res.headers.get('location');
157 if (!location) throw new Error('SAM.gov\'s download endpoint redirected without a Location header.');
158 return location;
159}
160
161
162const POSTED_DATE_RE = /","(\d{4}-\d{2}-\d{2}) \d{2}:\d{2}:\d{2}","/g;
163
164const ROW_START_RE = /\n"[0-9a-f]{32}",/g;
165
166
167
168
169async function readWindow(extractUrl, cutoffMs) {
170 const chunks = [];
171 let offset = 0;
172 let totalSize = null;
173 for (;;) {
174 const res = await politeFetch(extractUrl, { range: `bytes=${offset}-${offset + CHUNK_BYTES - 1}` });
175 if (res.status !== 206 && res.status !== 200) throw new Error(`Unexpected HTTP ${res.status} reading the extract at byte ${offset}.`);
176 const buf = Buffer.from(await res.arrayBuffer());
177 if (buf.length === 0) break;
178 chunks.push(buf);
179 offset += buf.length;
180 if (totalSize === null) {
181 const cr = res.headers.get('content-range');
182 const total = cr && Number(cr.split('/')[1]);
183 if (Number.isFinite(total)) totalSize = total;
184 }
185 const reachedEnd = totalSize !== null && offset >= totalSize;
186
187 const text = iconv.decode(Buffer.concat(chunks), 'win1252');
188 const dates = [...text.matchAll(POSTED_DATE_RE)].map((m) => Date.parse(`${m[1]}T00:00:00Z`));
189 const oldestSeen = dates.length ? Math.min(...dates) : null;
190
191
192 if (reachedEnd || (oldestSeen !== null && oldestSeen < cutoffMs - 2 * 86_400_000)) {
193 if (reachedEnd) return text;
194 const starts = [...text.matchAll(ROW_START_RE)].map((m) => m.index);
195 if (starts.length < 2) return text;
196 return text.slice(0, starts[starts.length - 1]);
197 }
198 if (offset >= HARD_CAP_BYTES) {
199 log.warning(`Stopped after ${Math.round(offset / 1_000_000)} MB (safety ceiling) without reaching the full requested window.`);
200 const starts = [...text.matchAll(ROW_START_RE)].map((m) => m.index);
201 return starts.length >= 2 ? text.slice(0, starts[starts.length - 1]) : text;
202 }
203 }
204 return iconv.decode(Buffer.concat(chunks), 'win1252');
205}
206
207
208
209const EMAIL_RE = /[A-Z0-9._%+-]+@[A-Z0-9-]+(?:\.[A-Z0-9-]+)*\.[A-Z]{2,}/gi;
210const PHONE_RE = /\(?\b\d{3}\)?[\s.-]\d{3}[\s.-]\d{4}\b/g;
211const redact = (s) => (typeof s === 'string' ? s.replace(EMAIL_RE, '[email removed]').replace(PHONE_RE, '[phone removed]') : s);
212const str = (v) => (typeof v === 'string' && v.trim() !== '' ? v.trim() : null);
213const isoDate = (v) => (str(v) ? v.trim().replace(' ', 'T') : null);
214const money = (v) => {
215 const n = Number(String(v ?? '').replace(/[^0-9.-]/g, ''));
216 return str(v) && Number.isFinite(n) ? n : null;
217};
218
219function mapRecord(r) {
220 return {
221 noticeId: r.NoticeId,
222 title: redact(str(r.Title)),
223 solicitationNumber: str(r['Sol#']),
224 agency: str(r['Department/Ind.Agency']),
225 subTier: str(r['Sub-Tier']),
226 office: str(r.Office),
227 noticeType: str(r.Type),
228 baseType: str(r.BaseType),
229 postedDate: isoDate(r.PostedDate),
230 responseDeadline: str(r.ResponseDeadLine),
231 daysToDeadline: daysUntil(str(r.ResponseDeadLine), Date.now()),
232 archiveDate: str(r.ArchiveDate),
233 setAsideCode: str(r.SetASideCode),
234 setAside: str(r.SetASide),
235 naicsCode: str(r.NaicsCode),
236 classificationCode: str(r.ClassificationCode),
237 placeOfPerformanceCity: str(r.PopCity),
238 placeOfPerformanceState: str(r.PopState),
239 placeOfPerformanceZip: str(r.PopZip),
240 placeOfPerformanceCountry: str(r.PopCountry),
241 active: r.Active === 'Yes' ? true : r.Active === 'No' ? false : null,
242 awardNumber: str(r.AwardNumber),
243 awardDate: str(r.AwardDate),
244 awardAmount: money(r['Award$']),
245 awardee: redact(str(r.Awardee)),
246 url: str(r.Link),
247 additionalInfoUrl: str(r.AdditionalInfoLink),
248 description: redact(str(r.Description)),
249 };
250}
251
252function passesFilters(item, opts, cutoffMs, nowMs) {
253 const posted = Date.parse(item.postedDate ?? '');
254 if (!Number.isFinite(posted) || posted < cutoffMs) return false;
255 if (opts.keyword) {
256 const hay = `${item.title ?? ''}\n${item.description ?? ''}`.toLowerCase();
257 if (!hay.includes(opts.keyword)) return false;
258 }
259 if (opts.naicsPrefixes.length) {
260 if (!item.naicsCode || !opts.naicsPrefixes.some((p) => item.naicsCode.startsWith(p))) return false;
261 }
262 if (opts.setAsideCodes.length) {
263 const code = item.setAsideCode || 'NONE';
264 if (!opts.setAsideCodes.includes(code)) return false;
265 }
266 if (opts.agency && !(item.agency ?? '').toLowerCase().includes(opts.agency)) return false;
267 if (opts.noticeTypes.length && !opts.noticeTypes.includes(item.noticeType)) return false;
268 if (opts.onlyOpenDeadline && item.responseDeadline) {
269 const deadline = Date.parse(item.responseDeadline);
270 if (Number.isFinite(deadline) && deadline < nowMs) return false;
271 }
272 return true;
273}
274
275
276
277await Actor.init();
278
279const opts = readInput(await Actor.getInput());
280const nowMs = Date.now();
281const cutoffMs = nowMs - opts.daysBack * 86_400_000;
282
283
284
285
286
287
288const monitor = opts.onlySinceLastRun;
289const stateStore = monitor ? await Actor.openKeyValueStore('sam-gov-contract-notices-seen') : null;
290const stateKey = monitor ? `seen-${createHash('sha256').update(filterFingerprint(opts)).digest('hex').slice(0, 40)}` : null;
291const delivered = {};
292if (monitor) {
293 const keepAfterMs = nowMs - 32 * 86_400_000;
294 for (const [k, v] of Object.entries((await stateStore.getValue(stateKey)) ?? {})) {
295 if ((Date.parse(v ?? '') || 0) >= keepAfterMs) delivered[k] = v;
296 }
297 log.info(`Monitor mode: ${Object.keys(delivered).length} notice(s) delivered by earlier runs with these filters will be skipped unless re-posted.`);
298}
299
300log.info(`Resolving today's SAM.gov Contract Opportunities extract and reading notices posted in the last ${opts.daysBack} day(s)...`);
301const extractUrl = await resolveExtractUrl();
302const text = await readWindow(extractUrl, cutoffMs);
303
304const rawRecords = parse(text, { columns: true, skip_empty_lines: true, relax_quotes: true, relax_column_count: true });
305log.info(`Parsed ${rawRecords.length} raw rows from the extract.`);
306
307const buffer = [];
308let saved = 0;
309let budgetReached = false;
310
311async function flush() {
312 if (!buffer.length) return;
313 const budget = remainingBudget();
314 const allowed = Math.max(0, Math.min(buffer.length, opts.maxItems - saved, budget));
315 const batch = buffer.splice(0).slice(0, allowed);
316 if (batch.length) {
317 await Actor.pushData(batch);
318 saved += batch.length;
319 if (monitor) {
320
321 for (const it of batch) delivered[it.noticeId] = it.postedDate;
322 await stateStore.setValue(stateKey, delivered);
323 }
324 }
325 if (budget - batch.length <= 0) budgetReached = true;
326}
327
328for (const r of rawRecords) {
329 if (saved + buffer.length >= opts.maxItems || budgetReached) break;
330 const item = mapRecord(r);
331 if (!passesFilters(item, opts, cutoffMs, nowMs)) continue;
332 if (monitor && delivered[item.noticeId] === item.postedDate) continue;
333
334 buffer.push(item);
335 if (buffer.length >= 25) await flush();
336}
337await flush();
338if (budgetReached) log.info('Reached the maximum charge set for this run; stopping.');
339
340log.info(`Saved ${saved} notices.`);
341if (saved === 0 && monitor) {
342 await Actor.setStatusMessage('No new notices since the last run with these filters.');
343} else if (saved === 0) {
344 await Actor.setStatusMessage('No notices matched these filters in the selected window. Try more days or fewer filters.');
345}
346
347await Actor.exit();
