1import { Actor, log } from 'apify';
2import { STATUSES, ELIGIBILITIES, CATEGORIES, INSTRUMENTS, SORTS, listOf, toCode, searchBody, mapOpportunity, daysUntil } from './grants.js';
3
4const EVENT = 'opportunity';
5const API = 'https://api.grants.gov/v1/api';
6const UA = 'apify-actor swillz/grants-gov-opportunity-search (+https://apify.com/swillz/grants-gov-opportunity-search)';
7const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
8class InputError extends Error {}
9class ServiceError extends Error {}
10
11async function post(path, body, { tries = 4 } = {}) {
12 let last;
13 for (let a = 0; a < tries; a++) {
14 try {
15 const res = await fetch(`${API}${path}`, { method: 'POST', headers: { 'content-type': 'application/json', accept: 'application/json', 'user-agent': UA }, body: JSON.stringify(body), signal: AbortSignal.timeout(45000) });
16 if (res.ok) {
17 const j = await res.json();
18 if (j.errorcode && j.errorcode !== 0) { if (/invalid|not found/i.test(j.msg ?? '')) throw new InputError(`Grants.gov: ${j.msg}`); last = `error ${j.errorcode}: ${j.msg}`; } else return j.data ?? {};
19 } else last = `HTTP ${res.status}`;
20 if (res.status === 400) throw new InputError(`Grants.gov rejected the search: ${(await res.text()).slice(0, 200)}`);
21 } catch (e) { if (e instanceof InputError) throw e; last = e.name === 'TimeoutError' ? 'timeout' : e.message; }
22 if (a < tries - 1) await sleep(1500 * 2 ** a);
23 }
24 throw new ServiceError(`Grants.gov did not answer (${last}).`);
25}
26
27
28function codes(values, map, field, bad) {
29 const out = [];
30 for (const v of listOf(values)) { const c = toCode(v, map); if (c) { if (!out.includes(c)) out.push(c); } else bad.push(`${field}: "${v}"`); }
31 return out;
32}
33
34await Actor.init();
35try {
36 const input = (await Actor.getInput()) ?? {};
37 const keywords = listOf(input.keywords ?? input.keyword ?? input.query ?? input.queries).map((k) => k.slice(0, 200)).slice(0, 50);
38 const oppNums = listOf(input.opportunityNumbers).map((k) => k.slice(0, 60)).slice(0, 200);
39 const statuses = listOf(input.statuses?.length ? input.statuses : ['posted', 'forecasted']).map((s) => s.toLowerCase());
40 const badStatus = statuses.filter((s) => !STATUSES.includes(s));
41 if (badStatus.length) throw new InputError(`Unknown statuses ${badStatus.map((s) => `"${s}"`).join(', ')}. Use any of: ${STATUSES.join(', ')}.`);
42 const bad = [];
43 const eligibilities = codes(input.eligibilities, ELIGIBILITIES, 'eligibilities', bad);
44 const categories = codes(input.fundingCategories, CATEGORIES, 'fundingCategories', bad);
45 const instruments = codes(input.fundingInstruments, INSTRUMENTS, 'fundingInstruments', bad);
46 if (bad.length) throw new InputError(`Unknown filter values: ${bad.join('; ')}. Use the codes or labels listed in the input form (e.g. eligibilities "23" or "Small businesses").`);
47 let agencies = listOf(input.agencies).map((a) => a.toUpperCase()).slice(0, 50);
48 const cfdas = listOf(input.assistanceListings ?? input.cfda).filter((c) => /^\d{2}\.\d{3,4}[A-Z]?$/i.test(c)).slice(0, 20);
49 const badCfda = listOf(input.assistanceListings ?? input.cfda).filter((c) => !/^\d{2}\.\d{3,4}[A-Z]?$/i.test(c));
50 if (badCfda.length) throw new InputError(`assistanceListings must look like "47.075" or "93.866" (ALN/CFDA numbers); got ${badCfda.map((c) => `"${c}"`).join(', ')}.`);
51 const postedWithinDays = Math.max(0, Math.floor(Number(input.postedWithinDays) || 0));
52 const closingWithinDays = Math.max(0, Math.floor(Number(input.closingWithinDays) || 0));
53 const minAwardCeiling = Math.max(0, Number(input.minAwardCeiling) || 0);
54 const sortKey = Object.hasOwn(SORTS, input.sortBy) ? input.sortBy : 'closingSoonest';
55 const maxResults = Math.max(1, Math.min(5000, Math.floor(Number(input.maxResults ?? 100)) || 100));
56 const wantDetails = input.includeDetails !== false || minAwardCeiling > 0;
57 const includeDescription = input.includeDescription !== false;
58 const maxDescriptionChars = Math.max(200, Math.min(20000, Math.floor(Number(input.maxDescriptionChars) || 4000)));
59 const statusRows = input.includeStatusRows !== false;
60 const dedupe = input.dedupe !== false;
61 const hasFilters = agencies.length || eligibilities.length || categories.length || instruments.length || cfdas.length || postedWithinDays || closingWithinDays || minAwardCeiling;
62 if (!keywords.length && !oppNums.length && !hasFilters) throw new InputError('Add at least one keyword (e.g. "artificial intelligence"), an opportunityNumber (e.g. "PD-19-127Y"), or a filter such as agencies ["NSF"], eligibilities ["Small businesses"] or postedWithinDays 7.');
63
64
65 if (agencies.length) {
66 const facets = await post('/search2', { rows: 0, oppStatuses: STATUSES.join('|') });
67 const exp = [];
68 for (const a of agencies) {
69 const top = (facets.agencies ?? []).find((o) => String(o.value).toUpperCase() === a);
70 exp.push(a, ...(top?.subAgencyOptions ?? []).map((s) => String(s.value).toUpperCase()));
71 }
72 agencies = [...new Set(exp)];
73 }
74
75 const charging = Actor.getChargingManager();
76 const counts = {}; let charged = 0; let chargeLimitHit = false; let unavailable = false;
77 const seen = new Set();
78 const now = () => new Date().toISOString();
79 async function free(row) { counts[row.status] = (counts[row.status] ?? 0) + 1; if (statusRows) await Actor.pushData({ ...row, checkedAt: now() }); }
80 async function paid(row) {
81 if (chargeLimitHit) return false;
82 const allowed = charging?.calculateMaxEventChargeCountWithinLimit(EVENT);
83 if (Number.isFinite(allowed) && allowed < 1) { chargeLimitHit = true; log.warning('Reached the maximum cost you set for this run. Stopping; results so far are saved.'); return false; }
84 counts.found = (counts.found ?? 0) + 1; charged++;
85 const r = await Actor.pushData({ ...row, status: 'found', checkedAt: now() }, EVENT);
86 if (r?.eventChargeLimitReached) chargeLimitHit = true;
87 return true;
88 }
89 const detailCache = new Map();
90 async function details(id) {
91 if (!detailCache.has(id)) detailCache.set(id, post('/fetchOpportunity', { opportunityId: Number(id) }, { tries: 3 }).catch((e) => { log.warning(`Details for ${id} failed: ${e.message}`); return null; }));
92 return detailCache.get(id);
93 }
94 const passes = (row) => (!closingWithinDays || (row.daysUntilClose != null && row.daysUntilClose >= 0 && row.daysUntilClose <= closingWithinDays))
95 && (!postedWithinDays || (row.postedDate && -daysUntil(row.postedDate) <= postedWithinDays))
96 && (!minAwardCeiling || (row.awardCeiling != null && row.awardCeiling >= minAwardCeiling));
97
98
99 async function runSearch(label, base) {
100 let start = 0; let matched = 0; let hitCount = 0; let dup = 0; let filtered = 0;
101 const pageRows = wantDetails ? 50 : 250;
102 while (matched < maxResults && !chargeLimitHit) {
103 const data = await post('/search2', searchBody({ ...base, statuses, agencies, eligibilities, categories, instruments, cfda: cfdas.join('|'), sort: SORTS[sortKey], postedWithinDays, rows: pageRows, start }));
104 hitCount = data.hitCount ?? 0;
105 const hits = data.oppHits ?? [];
106 if (!hits.length) break;
107
108 const pre = hits.map((h) => mapOpportunity(h, null)).map((r, i) => ({ r, h: hits[i] })).filter(({ r }) => (!closingWithinDays || (r.daysUntilClose != null && r.daysUntilClose >= 0 && r.daysUntilClose <= closingWithinDays)) && (!postedWithinDays || (r.postedDate && -daysUntil(r.postedDate) <= postedWithinDays)));
109 filtered += hits.length - pre.length;
110 const todo = [];
111 for (const p of pre) { if (dedupe && seen.has(p.r.opportunityId)) { dup++; continue; } todo.push(p); }
112 for (let i = 0; i < todo.length && matched < maxResults && !chargeLimitHit; i += 4) {
113 const batch = todo.slice(i, i + 4);
114 const ds = wantDetails ? await Promise.all(batch.map((p) => details(p.h.id))) : batch.map(() => null);
115 for (let k = 0; k < batch.length && matched < maxResults; k++) {
116 const row = mapOpportunity(batch[k].h, ds[k], { includeDescription, maxDescriptionChars });
117 if (!passes(row)) { filtered++; continue; }
118 if (dedupe) { if (seen.has(row.opportunityId)) { dup++; continue; } seen.add(row.opportunityId); }
119 if (!(await paid({ query: label, rank: matched + 1, ...row }))) break;
120 matched++;
121 }
122 }
123 start += hits.length;
124 if (start >= hitCount) break;
125
126 if (closingWithinDays && sortKey === 'closingSoonest' && pre.length === 0 && hits.every((h) => { const d = mapOpportunity(h, null).daysUntilClose; return d != null && d > closingWithinDays; })) break;
127 }
128 return { matched, hitCount, dup, filtered };
129 }
130
131 const searches = [
132 ...oppNums.map((n) => ({ label: n, base: { oppNum: n }, kind: 'number' })),
133 ...keywords.map((k) => ({ label: k, base: { keyword: k }, kind: 'keyword' })),
134 ];
135 if (!searches.length) searches.push({ label: '(filters only)', base: {}, kind: 'filters' });
136 const t0 = Date.now();
137 const perSearch = [];
138 for (const s of searches) {
139 if (chargeLimitHit) { await free({ type: 'status', query: s.label, status: 'skipped_charge_limit', error: 'Not searched: the run reached the maximum cost you set. Raise the limit and run again.' }); continue; }
140 if (unavailable) { await free({ type: 'status', query: s.label, status: 'service_unavailable', error: 'Grants.gov was not answering. Not charged; run again later.' }); continue; }
141 try {
142 const r = await runSearch(s.label, s.base);
143 perSearch.push({ query: s.label, ...r });
144 if (!r.matched) {
145 const why = r.dup ? `all ${r.dup} matches were already returned for an earlier search in this run` : r.hitCount ? `${r.hitCount} Grants.gov results, 0 passed your filters (closingWithinDays/postedWithinDays/minAwardCeiling)` : s.kind === 'number' ? `opportunity number "${s.label}" not found with statuses ${statuses.join('/')}${hasFilters ? ' and these filters (agencies/eligibilities/categories/instruments/assistance listings also apply to opportunity numbers)' : ''}` : `no Grants.gov opportunities match with statuses ${statuses.join('/')}${hasFilters ? ' and these filters' : ''}`;
146 await free({ type: 'status', query: s.label, status: s.kind === 'number' && !r.hitCount ? 'not_found' : 'no_results', error: `${why[0].toUpperCase()}${why.slice(1)}. Try fewer filters, add "closed" or "archived" to statuses, or a broader keyword. Not charged.` });
147 }
148 } catch (e) {
149 if (e instanceof InputError) await free({ type: 'status', query: s.label, status: 'invalid_input', error: `${e.message} Not charged.` });
150 else if (e instanceof ServiceError) { unavailable = true; await free({ type: 'status', query: s.label, status: 'service_unavailable', error: `${e.message} Not charged; run again later.` }); }
151 else throw e;
152 }
153 }
154
155 const dsOut = Actor.getEnv().defaultDatasetId;
156 const summary = {
157 searches: searches.length, opportunitiesReturned: charged, byStatus: counts, perSearch, statuses, sortBy: sortKey, agencies, chargeLimitReached: chargeLimitHit,
158 seconds: Math.round((Date.now() - t0) / 1000),
159 zeroResultsReason: charged ? null : `No opportunities returned: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ') || 'nothing matched'}. See each status row's error.`,
160 nextSteps: [
161 'Data comes live from the official Grants.gov API (api.grants.gov). Always confirm deadlines and eligibility on grantsGovUrl before applying.',
162 'Schedule this actor daily with postedWithinDays: 1 or 3 to get only new opportunities.',
163 'Check past federal awards to a company or university with swillz/federal-awards-by-company.',
164 `Export to Google Sheets: run swillz/google-sheets-import-export with {"spreadsheet": "<your sheet URL>", "datasetId": "${dsOut}"}`,
165 ],
166 finishedAt: now(),
167 };
168 await Actor.setValue('OUTPUT', summary);
169 const freeN = Object.entries(counts).filter(([k]) => k !== 'found');
170 await Actor.exit(`${charged ? '' : '0 opportunities: '}Returned ${charged} Grants.gov ${charged === 1 ? "opportunity" : "opportunities"} for ${searches.length} ${searches.length === 1 ? 'search' : 'searches'}${freeN.length ? `; ${freeN.map(([k, v]) => `${v} ${k}`).join(', ')} not charged` : ''}.`);
171} catch (e) {
172 if (e instanceof InputError) { log.error(e.message); await Actor.fail(`Invalid input: ${e.message}`); } else { log.exception(e, 'Run failed'); await Actor.fail(e.message); }
173}
