1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22import { Actor, log } from "apify";
23import { mapSerperResponse } from "@/lib/serper-map";
24
25const MAX_QUERIES = 1000;
26const CHARGE_EVENT = "query-executed";
27const REQUEST_TIMEOUT_MS = 30_000;
28const RETRY_DELAY_MS = 1_500;
29
30interface Input {
31 queries?: string[] | string;
32 num?: number;
33 country?: string;
34 language?: string;
35 page?: number;
36 device?: string;
37}
38
39function withScores(results: { title: string; url: string; description: string }[]) {
40 const n = results.length || 1;
41 return results.map((r, i) => ({ ...r, score: Math.round(((n - i) / n) * 1000) / 1000 }));
42}
43
44
45function classifyUpstream(status: number, detail: string): { error: string; hint: string } {
46 const d = detail.toLowerCase();
47 if (d.includes("credit")) {
48 return { error: "upstream_credits_exhausted", hint: "The Serper account behind this Actor has no credits left. Top up at serper.dev." };
49 }
50 if (status === 401 || status === 403) {
51 return { error: "upstream_auth_error", hint: "Serper rejected the API key. Check SERPER_API_KEY under this Actor's Settings -> Environment variables." };
52 }
53 if (status === 429) return { error: "upstream_rate_limited", hint: "Serper rate limit hit. Retry the run in a minute." };
54 if (status >= 500) return { error: "upstream_unavailable", hint: "Serper returned a server error. Retry the run later." };
55 return { error: "upstream_error", hint: "Serper rejected the request. See message for its response." };
56}
57
58async function main(): Promise<void> {
59 await Actor.init();
60
61 let pushed = 0;
62 let errors = 0;
63
64
65 const notice = async (query: string, error: string, message: string, extra: { status?: number; hint?: string } = {}) => {
66 await Actor.pushData({ query, error, message, ...extra, checked_at: new Date().toISOString() });
67 pushed++;
68 errors++;
69 };
70
71
72 const finish = async (statusMessage: string): Promise<void> => {
73 if (pushed === 0) {
74 await Actor.pushData({ query: "", error: "no_output", message: "The run completed without producing any results.", checked_at: new Date().toISOString() });
75 pushed++;
76 }
77 log.info(statusMessage);
78 await Actor.exit({ statusMessage, exitCode: 0 });
79 };
80
81 const input = (await Actor.getInput<Input>()) ?? {};
82 const rawQueries = Array.isArray(input.queries) ? input.queries : typeof input.queries === "string" ? [input.queries] : [];
83 let queries = rawQueries.map((q) => String(q ?? "").trim()).filter(Boolean);
84 const num = Math.min(Math.max(Math.trunc(Number(input.num)) || 10, 1), 100);
85 const { country, language, device } = input;
86 const page = Number.isInteger(Number(input.page)) && Number(input.page) >= 1 ? Number(input.page) : undefined;
87
88 const apiKey = (process.env.SERPER_API_KEY ?? "").trim();
89 if (!apiKey) {
90 await notice("", "missing_api_key", "SERPER_API_KEY is not set for this Actor, so no search was run.", {
91 hint: "Add SERPER_API_KEY under this Actor's Settings -> Environment variables in the Apify Console, then run again.",
92 });
93 return finish("No search run: SERPER_API_KEY is not configured (see the dataset item).");
94 }
95
96 if (queries.length === 0) {
97 await notice("", "no_queries", 'Input must include a non-empty "queries" array, so no search was run.', {
98 hint: 'Example input: { "queries": ["web scraping tools"], "num": 3 }',
99 });
100 return finish('No search run: no "queries" in the input (see the dataset item).');
101 }
102
103 if (queries.length > MAX_QUERIES) {
104 const skipped = queries.length - MAX_QUERIES;
105 queries = queries.slice(0, MAX_QUERIES);
106 await notice("", "too_many_queries", `Got ${MAX_QUERIES + skipped} queries; this Actor caps at ${MAX_QUERIES} per run. The first ${MAX_QUERIES} were run; ${skipped} were skipped.`, {
107 hint: "Split the remaining queries into another run.",
108 });
109 }
110
111 let processed = 0;
112 let stopped = false;
113
114 for (const q of queries) {
115 if (stopped) break;
116 const started = Date.now();
117 const body: Record<string, unknown> = { q, num };
118 if (country) body.gl = country;
119 if (language) body.hl = language;
120 if (page) body.page = page;
121 if (device) body.device = device;
122
123
124 let res: Response | null = null;
125 let networkError: unknown = null;
126 for (let attempt = 0; attempt < 2; attempt++) {
127 try {
128 res = await fetch("https://google.serper.dev/search", {
129 method: "POST",
130 headers: { "X-API-KEY": apiKey, "Content-Type": "application/json" },
131 body: JSON.stringify(body),
132 signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
133 });
134 networkError = null;
135 if (res.status !== 429 && res.status < 500) break;
136 } catch (e) {
137 networkError = e;
138 res = null;
139 }
140 if (attempt === 0) await new Promise((r) => setTimeout(r, RETRY_DELAY_MS));
141 }
142
143 if (!res) {
144 const e = networkError as Error | null;
145 const timedOut = e?.name === "TimeoutError" || e?.name === "AbortError";
146 await notice(q, timedOut ? "request_timeout" : "request_failed", String(e?.message ?? e ?? "request failed"), {
147 hint: "Network error reaching Serper. Retry the run later.",
148 });
149 continue;
150 }
151
152 if (!res.ok) {
153 const detail = await res.text().catch(() => "");
154 const { error, hint } = classifyUpstream(res.status, detail);
155 await notice(q, error, detail.slice(0, 500) || `HTTP ${res.status}`, { status: res.status, hint });
156
157 if (error === "upstream_auth_error" || error === "upstream_credits_exhausted") {
158 stopped = true;
159 log.warning(`${error}: stopping — the same failure would repeat for every remaining query.`);
160 }
161 continue;
162 }
163
164 let mapped;
165 try {
166 const raw = await res.json();
167 mapped = mapSerperResponse(raw);
168 } catch (e) {
169 await notice(q, "invalid_upstream_response", `Serper returned a response that could not be read: ${String((e as Error)?.message ?? e)}`.slice(0, 500), {
170 status: res.status,
171 hint: "Retry the run. If it persists, the upstream response format may have changed.",
172 });
173 continue;
174 }
175
176 const results = withScores(mapped.organic.map((o) => ({ title: o.title, url: o.url, description: o.description })));
177
178 await Actor.pushData({
179 query: q,
180 results,
181 organic: mapped.organic,
182 ads: mapped.ads,
183 people_also_ask: mapped.people_also_ask,
184 related_searches: mapped.related_searches,
185 knowledge_graph: mapped.knowledge_graph,
186 answer_box: mapped.answer_box,
187 shopping: mapped.shopping,
188 ai_overview: mapped.ai_overview,
189
190
191
192 results_count: results.length,
193 organic_count: mapped.organic.length,
194 people_also_ask_count: mapped.people_also_ask.length,
195 related_searches_count: mapped.related_searches.length,
196 latency_ms: Date.now() - started,
197 checked_at: new Date().toISOString(),
198 });
199 pushed++;
200 processed++;
201
202
203 try {
204 const charge = await Actor.charge({ eventName: CHARGE_EVENT, count: 1 });
205 if (charge?.eventChargeLimitReached) {
206 await notice("", "charge_limit_reached", "The maximum charge for this run was reached, so remaining queries were not run.", {
207 hint: "Raise the run's maximum cost and run the remaining queries again.",
208 });
209 stopped = true;
210 }
211 } catch (e) {
212 log.warning(`Actor.charge failed (result kept, run continues): ${String((e as Error)?.message ?? e)}`);
213 }
214 }
215
216 await finish(`Executed ${processed}/${queries.length} quer${queries.length === 1 ? "y" : "ies"}${errors ? `; ${errors} error record${errors === 1 ? "" : "s"} in the dataset` : ""}.`);
217}
218
219
220main().catch(async (err) => {
221 log.exception(err as Error, "Unhandled error in main()");
222 await Actor.fail(err instanceof Error ? err.message : String(err));
223});