1import { Actor, log } from 'apify';
2import { ALIASES, GROUPS, FIELDS, SORT_FIELD, parseRecipients, isUei, nameMatches, period, baseFilters, mapAward, isRedacted } from './usa.js';
3
4const EVENT = 'award-record';
5const API = 'https://api.usaspending.gov/api/v2';
6const NAME_FIELDS = ['companyName', 'company', 'organization', 'organizationName', 'legalName', 'name', 'uei', 'recipient'];
7const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
8const getPath = (o, path) => path.split('.').reduce((v, k) => (v && typeof v === 'object' ? v[k] : undefined), o);
9class InputError extends Error {}
10class ServiceError extends Error {}
11
12async function post(path, body, { tries = 4 } = {}) {
13 let last;
14 for (let a = 0; a < tries; a++) {
15 try {
16 const res = await fetch(`${API}${path}`, { method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json', 'user-agent': 'apify-actor swillz/federal-awards-by-company (+https://apify.com/swillz/federal-awards-by-company)' }, body: JSON.stringify(body), signal: AbortSignal.timeout(60000) });
17 if (res.ok) return await res.json();
18 const txt = (await res.text()).slice(0, 300);
19 if (res.status === 400 || res.status === 422) throw new InputError(`USAspending rejected the filters: ${txt}`);
20 last = `HTTP ${res.status}`;
21 } catch (e) { if (e instanceof InputError) throw e; last = e.name === 'TimeoutError' ? 'timeout' : e.message; }
22 if (a < tries - 1) await sleep(1500 * 2 ** a);
23 }
24 throw new ServiceError(`USAspending.gov did not answer (${last}).`);
25}
26const listOf = (v) => (Array.isArray(v) ? v : String(v ?? '').split(/[,\n]+/)).map((s) => String(s ?? '').trim()).filter(Boolean);
27
28await Actor.init();
29try {
30 const input = (await Actor.getInput()) ?? {};
31 const recipients = parseRecipients(input.companies ?? input.recipients ?? input.company ?? []);
32 const dsId = String(input.datasetId || input?.payload?.resource?.defaultDatasetId || input?.resource?.defaultDatasetId || '').trim();
33 if (dsId) {
34 const max = Math.max(1, Math.floor(Number(input.datasetMaxItems) || 100));
35 let f = String(input.companyField || '').trim() || null; let got = 0;
36 let page;
37 try { page = await Actor.newClient().dataset(dsId).listItems({ limit: Math.min(max * 3, 5000), clean: true }); } catch (e) { throw new InputError(`Could not read dataset ${dsId}: ${e.message}`); }
38 const items = page.items ?? [];
39 f ||= NAME_FIELDS.find((k) => items.some((it) => String(getPath(it, k) ?? '').trim())) ?? null;
40 if (f) for (const it of items) { const v = String(getPath(it, f) ?? '').trim(); if (v && !recipients.some((r) => r.toLowerCase() === v.toLowerCase())) { recipients.push(v.slice(0, 200)); if (++got >= max) break; } }
41 log.info(`Read ${got} companies from dataset ${dsId} (field "${f ?? 'none found'}").`);
42 }
43 const keywords = listOf(input.keywords).filter((k) => k.length >= 3).slice(0, 20);
44 const agencies = listOf(input.agencies).slice(0, 10);
45 const naicsCodes = listOf(input.naicsCodes).filter((c) => /^\d{2,6}$/.test(c)).slice(0, 50);
46 const pscCodes = listOf(input.pscCodes).filter((c) => /^[A-Z0-9]{1,4}$/i.test(c)).map((c) => c.toUpperCase()).slice(0, 50);
47 const states = listOf(input.states).filter((s) => /^[A-Za-z]{2}$/.test(s)).slice(0, 60);
48 const minAmount = Number(input.minAmount) || 0;
49 const groups = listOf(input.awardTypes?.length ? input.awardTypes : ['contracts']).map((g) => g.toLowerCase()).filter((g) => GROUPS[g]);
50 if (!groups.length) throw new InputError('awardTypes must include at least one of: contracts, idvs, grants, loans.');
51 const per = period(input);
52 if (per.error) throw new InputError(per.error);
53 const maxPer = Math.max(1, Math.min(5000, Math.floor(Number(input.maxAwardsPerCompany ?? 100)) || 100));
54 const sortBy = input.sortBy === 'newest' ? 'newest' : 'amount';
55 const strict = input.strictNameMatch !== false;
56 const wantSummary = input.includeSummary !== false;
57 const hasFilters = keywords.length || agencies.length || naicsCodes.length || pscCodes.length || states.length;
58 if (!recipients.length && !hasFilters) throw new InputError('Add at least one company name or UEI in "companies" (e.g. "Booz Allen Hamilton" or "JCBMLGPE6Z71"), or search by keywords, agencies, NAICS, PSC or states.');
59 const maxCompanies = Math.max(1, Math.floor(Number(input.maxCompanies) || 200));
60 const overCap = recipients.length > maxCompanies ? recipients.splice(maxCompanies).length : 0;
61
62 const charging = Actor.getChargingManager();
63 const counts = {}; let charged = 0; let chargeLimitHit = false; let skipped = 0;
64 const now = () => new Date().toISOString();
65 async function free(row) { counts[row.status] = (counts[row.status] ?? 0) + 1; await Actor.pushData({ ...row, checkedAt: now() }); }
66 async function paid(row) {
67 if (chargeLimitHit) return false;
68 const allowed = charging?.calculateMaxEventChargeCountWithinLimit(EVENT);
69 if (Number.isFinite(allowed) && allowed < 1) { chargeLimitHit = true; log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.'); return false; }
70 counts[row.status] = (counts[row.status] ?? 0) + 1; charged++;
71 const r = await Actor.pushData({ ...row, checkedAt: now() }, EVENT);
72 if (r?.eventChargeLimitReached) chargeLimitHit = true;
73 return true;
74 }
75 const filt = (extra) => baseFilters({ keywords, agencies, naicsCodes, pscCodes, states, minAmount, start: per.start, end: per.end, ...extra });
76
77
78 async function fetchAwards(searchText, query, acceptName) {
79 const out = []; const counts2 = {};
80 for (const g of groups) {
81 if (out.length >= maxPer) break;
82 let page = 1;
83 while (out.length < maxPer && page <= 50) {
84 const body = { filters: { ...filt({ recipients: searchText }), award_type_codes: GROUPS[g].codes }, fields: FIELDS[g], limit: 100, page, sort: SORT_FIELD[sortBy][g], order: 'desc' };
85 const j = await post('/search/spending_by_award/', body);
86 for (const r of j.results ?? []) {
87 if (isRedacted(r['Recipient Name'])) continue;
88 if (acceptName && !acceptName(r)) continue;
89 out.push(mapAward(r, g, query)); counts2[g] = (counts2[g] ?? 0) + 1;
90 if (out.length >= maxPer) break;
91 }
92 if (!j.page_metadata?.hasNext) break;
93 page++;
94 }
95 }
96 return out;
97 }
98 async function topCategory(cat, searchText, limit = 5) {
99 const res = [];
100 for (const g of groups) {
101 try { const j = await post(`/search/spending_by_category/${cat}/`, { filters: { ...filt({ recipients: searchText }), award_type_codes: GROUPS[g].codes }, limit }); res.push(...(j.results ?? [])); } catch (e) { if (e instanceof InputError) throw e; log.warning(`${cat} breakdown failed: ${e.message}`); }
102 }
103 const m = new Map();
104 for (const r of res) { const k = r.code ?? r.name; const prev = m.get(k); m.set(k, { code: r.code ?? null, uei: r.uei ?? prev?.uei ?? null, name: r.name, amount: Math.round(((prev?.amount ?? 0) + (r.amount ?? 0)) * 100) / 100 }); }
105 return [...m.values()].sort((a, b) => b.amount - a.amount).slice(0, limit);
106 }
107
108 const t0 = Date.now();
109 let unavailable = false;
110 for (const q of recipients) {
111 if (chargeLimitHit) { skipped++; await free({ type: 'status', query: q, status: 'skipped_charge_limit', error: 'Not searched: the run reached the maximum cost you set. Raise the limit and run this company again.' }); continue; }
112 if (unavailable) { await free({ type: 'status', query: q, status: 'service_unavailable', error: 'USAspending.gov was not answering. Not charged; run again later.' }); continue; }
113 try {
114 const uei = isUei(q) ? q.toUpperCase() : null;
115 const qs = ALIASES[q.toLowerCase()] ?? q;
116 let ueis = [];
117 if (uei) ueis = [uei];
118 else {
119
120 const raw = [];
121 for (const g of groups) {
122 const j = await post('/search/spending_by_category/recipient/', { filters: { ...filt({ recipients: [qs] }), award_type_codes: GROUPS[g].codes }, limit: 50 });
123 raw.push(...(j.results ?? []));
124 }
125 const seenU = new Set();
126 for (const r of raw) if (r.uei && !isRedacted(r.name) && (!strict || nameMatches(qs, r.name)) && !seenU.has(r.uei)) { seenU.add(r.uei); ueis.push(r.uei); }
127 ueis = ueis.slice(0, 20);
128 }
129 let rows;
130 if (ueis.length) rows = await fetchAwards(ueis, q, null);
131 else rows = await fetchAwards([qs], q, (r) => !strict || nameMatches(qs, r['Recipient Name']));
132 if (!rows.length) {
133 await free({ type: 'status', query: q, status: 'not_found', error: `No ${groups.join('/')} awards for "${q}" between ${per.start} and ${per.end}${hasFilters ? ' with these filters' : ''}. Try the legal name (e.g. "Lockheed Martin Corporation"), the 12-character UEI, a longer date range (startDate), more awardTypes${strict ? ', or strictNameMatch: false' : ''}. Not charged.` });
134 continue;
135 }
136 if (wantSummary) {
137 const sText = ueis.length ? ueis : [qs];
138 const [aggs, agenciesTop, naicsTop] = await Promise.all([topCategory('recipient', sText, 20), topCategory('awarding_agency', sText, 5), topCategory('naics', sText, 5)]);
139 const keep = aggs.filter((a) => !isRedacted(a.name) && (ueis.length ? true : !strict || nameMatches(qs, a.name)));
140 const total = keep.reduce((s, a) => s + (a.amount ?? 0), 0);
141 const ok = await paid({ type: 'company_summary', query: q, status: 'found', recipientName: rows[0].recipientName, recipientUei: rows[0].recipientUei,
142 matchedRecipients: keep.map((a) => ({ name: a.name, uei: a.uei, amount: a.amount })), recipientUeis: ueis,
143 periodStart: per.start, periodEnd: per.end, awardTypes: groups, obligatedInPeriod: Math.round(total * 100) / 100,
144 awardsReturned: rows.length, topAwardingAgencies: agenciesTop.map((a) => ({ name: a.name, amount: a.amount })), topNaics: naicsTop.map((a) => ({ code: a.code, name: a.name, amount: a.amount })),
145 recipientProfileUrl: rows.find((r) => r.recipientProfileUrl)?.recipientProfileUrl ?? null });
146 if (!ok) { skipped++; continue; }
147 }
148 let rank = 0;
149 for (const r of rows) { rank++; if (!(await paid({ ...r, rank, status: 'found' }))) break; }
150 } catch (e) {
151 if (e instanceof InputError) await free({ type: 'status', query: q, status: 'invalid_input', error: `${e.message} Not charged.` });
152 else if (e instanceof ServiceError) { unavailable = true; await free({ type: 'status', query: q, status: 'service_unavailable', error: `${e.message} Not charged; run again later.` }); }
153 else throw e;
154 }
155 }
156 if (!recipients.length) {
157 try {
158 const rows = await fetchAwards(undefined, '(filters only)', null);
159 if (!rows.length) await free({ type: 'status', query: '(filters only)', status: 'not_found', error: `No awards match these filters between ${per.start} and ${per.end}. Loosen the filters or widen the dates. Not charged.` });
160 let rank = 0;
161 for (const r of rows) { rank++; if (!(await paid({ ...r, rank, status: 'found' }))) break; }
162 } catch (e) {
163 if (e instanceof InputError) await free({ type: 'status', query: '(filters only)', status: 'invalid_input', error: `${e.message} Not charged.` });
164 else if (e instanceof ServiceError) await free({ type: 'status', query: '(filters only)', status: 'service_unavailable', error: `${e.message} Not charged; run again later.` });
165 else throw e;
166 }
167 }
168
169 const dsOut = Actor.getEnv().defaultDatasetId;
170 const summary = {
171 companies: recipients.length, recordsReturned: charged, byStatus: counts, period: per, awardTypes: groups, notRunOverMaxCompanies: overCap, chargeLimitReached: chargeLimitHit, skippedCompanies: skipped,
172 seconds: Math.round((Date.now() - t0) / 1000), sourceDatasetId: dsId || null,
173 zeroResultsReason: charged ? null : `No records returned: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ') || 'nothing matched'}. See each row's error.`,
174 nextSteps: [
175 'Amounts are federal obligations reported to USAspending.gov (FPDS for contracts, FABS for assistance). Recent months can be incomplete and DoD contracts are published with a 90-day delay.',
176 'Use recipientUei from the company_summary row as a "companies" entry for an exact, faster rerun.',
177 'Add "idvs" to awardTypes to include GSA schedules, GWACs and other indefinite-delivery vehicles.',
178 `Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${dsOut}"}`,
179 ],
180 finishedAt: now(),
181 };
182 await Actor.setValue('OUTPUT', summary);
183 const freeN = Object.entries(counts).filter(([k]) => k !== 'found');
184 await Actor.exit(`${charged ? '' : '0 records: '}Returned ${charged} federal award records for ${recipients.length ? `${recipients.length} ${recipients.length === 1 ? 'company' : 'companies'}` : 'a filter search'} (${per.start} to ${per.end})${freeN.length ? `; ${freeN.map(([k, v]) => `${v} ${k}`).join(', ')} not charged` : ''}.`);
185} catch (e) {
186 if (e instanceof InputError) { log.error(e.message); await Actor.fail(`Invalid input: ${e.message}`); } else { log.exception(e, 'Run failed'); await Actor.fail(e.message); }
187}
