1import { Actor, log } from 'apify';
2import { gotScraping } from 'got-scraping';
3import robotsParser from 'robots-parser';
4import { htmlToMarkdown, truncateToTokens, estTokens, isProbablyHtmlUrl } from './markdown.js';
5import { normalizeSchema, validate, stripNulls, InputError } from './schema.js';
6import { extract, PRESETS, priceOf, costUsd, LlmError } from './llm.js';
7import { freeTrialCap } from './plan.js';
8const FREE_TRIAL_PAGES = 25;
9
10const EVENT_BUILTIN = 'page-extracted';
11const EVENT_BYOK = 'page-extracted-own-key';
12const PRICE_BUILTIN = 0.008;
13const UA_TOKEN = 'AIExtract';
14const MAX_BODY = 5 * 1024 * 1024;
15
16const BUILTIN = { maxIn: 16000, maxOut: 4000, maxPriceIn: 0.30, maxPriceOut: 2.50 };
17const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
18const clampInt = (v, d, lo, hi) => Math.max(lo, Math.min(hi, Math.floor(Number.isFinite(Number(v)) && v !== null && v !== '' ? Number(v) : d)));
19const normUrl = (u) => { const x = new URL(u); x.hash = ''; return x.href; };
20
21await Actor.init();
22try {
23 const input = (await Actor.getInput()) ?? {};
24 const {
25 startUrls = [], urls, instructions = '', jsonSchema, maxCrawlDepth = 0, maxPages = 20, sameDomainOnly = true,
26 includeUrlPatterns = [], excludeUrlPatterns = [], contentSelector, removeSelectors, keepHeaderFooter = false,
27 includeMarkdown = false, maxInputTokensPerPage = 8000, maxOutputTokensPerPage = 1500, maxConcurrency = 5, maxRetries = 2,
28 requestTimeoutSecs = 30, llmApiKey, llmProvider = 'openai', llmModel, llmBaseUrl, maxAiCostUsd,
29 proxyConfiguration = { useApifyProxy: false },
30 } = input;
31
32
33 const rawUrls = [...(Array.isArray(startUrls) ? startUrls : [startUrls]), ...(Array.isArray(urls) ? urls : urls ? String(urls).split(/[\n,\s]+/) : [])]
34 .map((x) => (typeof x === 'object' && x ? x.url : x)).map((x) => String(x ?? '').trim()).filter(Boolean);
35 if (!rawUrls.length) throw new InputError('Add at least one URL to "Start URLs", e.g. https://books.toscrape.com/');
36 const schema = normalizeSchema(jsonSchema);
37 const prompt = String(instructions || '').trim();
38 if (!prompt && !schema) throw new InputError('Describe what to extract in "Instructions" and/or give a "JSON schema".');
39 if (prompt.length > 8000) throw new InputError('"Instructions" is too long (max 8,000 characters).');
40 const toRegex = (arr, name) => (Array.isArray(arr) ? arr : [arr]).filter(Boolean).map((p) => {
41 try { return new RegExp(typeof p === 'object' ? p.regex ?? p.glob ?? p.url : p, 'i'); } catch (e) { throw new InputError(`Invalid pattern in "${name}": ${e.message}`); }
42 });
43 const inc = toRegex(includeUrlPatterns, 'Include URL patterns'); const exc = toRegex(excludeUrlPatterns, 'Exclude URL patterns');
44
45
46 const byok = Boolean(llmApiKey && String(llmApiKey).trim());
47 let ai;
48 if (byok) {
49 if (!PRESETS[llmProvider]) throw new InputError(`Unknown "AI provider" ${llmProvider}. Use one of: ${Object.keys(PRESETS).join(', ')}`);
50 if (llmProvider === 'openai-compatible' && !llmBaseUrl) throw new InputError('Set "AI API base URL" when the provider is "openai-compatible".');
51 const model = llmModel || PRESETS[llmProvider].model;
52 if (!model) throw new InputError('Set "AI model" for an OpenAI-compatible provider.');
53 ai = { provider: llmProvider, apiKey: String(llmApiKey).trim(), baseUrl: llmBaseUrl || null, model, mode: 'own-key',
54 maxIn: clampInt(maxInputTokensPerPage, 8000, 500, 400000), maxOut: clampInt(maxOutputTokensPerPage, 1500, 64, 64000) };
55 } else {
56 const key = process.env.LLM_API_KEY;
57 if (!key) throw new InputError('The built-in AI is not available on this deployment right now. Add your own AI API key in "Your AI API key (optional)" to run.');
58 const provider = process.env.LLM_PROVIDER || 'openai';
59 let model = process.env.LLM_MODEL || PRESETS[provider]?.model;
60 if (llmModel && llmModel !== model) {
61 const [pi, po] = priceOf(llmModel);
62 if (pi <= BUILTIN.maxPriceIn && po <= BUILTIN.maxPriceOut && PRESETS[provider]?.kind === 'openai' && /^gpt-/.test(llmModel)) model = llmModel;
63 else log.warning(`"AI model" ${llmModel} is only available with your own API key; using ${model}.`);
64 }
65 ai = { provider, apiKey: key, baseUrl: process.env.LLM_BASE_URL || null, model, mode: 'built-in',
66 maxIn: clampInt(maxInputTokensPerPage, 8000, 500, BUILTIN.maxIn), maxOut: clampInt(maxOutputTokensPerPage, 1500, 64, BUILTIN.maxOut) };
67 }
68 const aiCfg = { provider: ai.provider, apiKey: ai.apiKey, baseUrl: ai.baseUrl, model: ai.model, maxOutputTokens: ai.maxOut, timeoutMs: 120000 };
69 const EVENT = byok ? EVENT_BYOK : EVENT_BUILTIN;
70 const requestedPages = clampInt(maxPages, 20, 1, 100000);
71 const trial = freeTrialCap({ requested: requestedPages, byok, limit: FREE_TRIAL_PAGES });
72 const pageCap = trial.cap;
73 if (trial.limited) log.warning(`Apify free plan: the built-in AI processes at most ${FREE_TRIAL_PAGES} pages per run. Add your own AI API key, or use any paid Apify plan, to process up to ${requestedPages}.`);
74 const depthCap = clampInt(maxCrawlDepth, 0, 0, 10);
75 const conc = clampInt(maxConcurrency, 5, 1, 25);
76 const retries = clampInt(maxRetries, 2, 0, 5);
77 const aiCap = byok && Number(maxAiCostUsd) > 0 ? Number(maxAiCostUsd) : null;
78 log.info(`AI: ${ai.mode} key, ${ai.provider}/${ai.model}; up to ${pageCap} pages, depth ${depthCap}, ${ai.maxIn} input tokens/page.`);
79
80 const proxy = proxyConfiguration?.useApifyProxy || proxyConfiguration?.proxyUrls?.length ? await Actor.createProxyConfiguration(proxyConfiguration) : null;
81 const charging = Actor.getChargingManager();
82 const stats = { requests: 0, httpRetries: 0, aiCalls: 0, aiRetries: 0, inputTokens: 0, outputTokens: 0, aiCostUsd: 0, robotsBlocked: 0, truncatedPages: 0 };
83 const counts = {}; let charged = 0; let stop = null; let fatal = null;
84
85
86 async function get(url, accept = 'text/html,application/xhtml+xml;q=0.9,*/*;q=0.5') {
87 let lastErr;
88 for (let a = 0; a <= retries; a++) {
89 stats.requests++;
90 try {
91 const res = await gotScraping({ url, proxyUrl: proxy ? await proxy.newUrl() : undefined, timeout: { request: clampInt(requestTimeoutSecs, 30, 5, 120) * 1000 },
92 throwHttpErrors: false, retry: { limit: 0 }, followRedirect: true, maxRedirects: 8, headers: { accept },
93 headerGeneratorOptions: { browsers: ['chrome'], devices: ['desktop'] } });
94 if ((res.statusCode === 429 || res.statusCode >= 500) && a < retries) { stats.httpRetries++; await sleep(1500 * 2 ** a); continue; }
95 return res;
96 } catch (e) { lastErr = e; if (a < retries) { stats.httpRetries++; await sleep(1000 * 2 ** a); } }
97 }
98 throw lastErr;
99 }
100 const robotsCache = new Map();
101 function robotsFor(origin) {
102 if (!robotsCache.has(origin)) {
103 robotsCache.set(origin, (async () => {
104 try {
105 const res = await get(`${origin}/robots.txt`, 'text/plain,*/*;q=0.5');
106 if (res.statusCode >= 500) return { blockAll: true };
107 const body = res.statusCode >= 400 || /<html/i.test(String(res.body).slice(0, 500)) ? '' : String(res.body).slice(0, 500000);
108 return { robots: robotsParser(`${origin}/robots.txt`, body) };
109 } catch { return { unreachable: true }; }
110 })());
111 }
112 return robotsCache.get(origin);
113 }
114 async function robotsCheck(url) {
115 const r = await robotsFor(new URL(url).origin);
116 if (r.unreachable) return 'unreachable';
117 if (r.blockAll) return false;
118 return r.robots.isAllowed(url, UA_TOKEN) !== false;
119 }
120 const hostNext = new Map();
121 async function polite(url) {
122 const { host, origin } = new URL(url);
123 const r = await robotsFor(origin);
124 const delay = Math.max(250, Math.min(10, Number(r.robots?.getCrawlDelay(UA_TOKEN)) || 0) * 1000);
125 for (;;) {
126 const now = Date.now(); const t = hostNext.get(host) || 0;
127 if (now >= t) { hostNext.set(host, now + delay); return; }
128 await sleep(t - now);
129 }
130 }
131
132
133 const seen = new Set(); const queue = []; let enqueued = 0; let capHit = false;
134 const startHosts = new Set();
135 const invalid = [];
136 for (const r of rawUrls) {
137 let u; try { u = new URL(/^https?:\/\//i.test(r) ? r : `https://${r}`); } catch { invalid.push(r); continue; }
138 if (!/^https?:$/.test(u.protocol)) { invalid.push(r); continue; }
139 const n = normUrl(u.href); if (seen.has(n)) continue;
140 seen.add(n); startHosts.add(u.hostname.replace(/^www\./, '')); queue.push({ url: n, depth: 0 }); enqueued++;
141 }
142 for (const r of invalid) { counts.invalid_url = (counts.invalid_url ?? 0) + 1; await Actor.pushData({ url: r, status: 'invalid_url', error: 'Not a valid http(s) URL', data: null, extractedAt: new Date().toISOString() }); }
143 const wanted = (u, depth) => {
144 if (depth > depthCap) return false;
145 if (enqueued >= pageCap) { capHit = true; return false; }
146 let x; try { x = new URL(u); } catch { return false; }
147 if (!/^https?:$/.test(x.protocol) || !isProbablyHtmlUrl(x.pathname)) return false;
148 if (sameDomainOnly && !startHosts.has(x.hostname.replace(/^www\./, ''))) return false;
149 if (inc.length && !inc.some((re) => re.test(u))) return false;
150 if (exc.some((re) => re.test(u))) return false;
151 return true;
152 };
153
154
155 async function processPage({ url, depth }) {
156 const row = { url, loadedUrl: null, depth, status: 'ok', title: null, data: null, error: null, truncated: false,
157 inputTokens: 0, outputTokens: 0, model: ai.model, aiKey: ai.mode, extractedAt: null };
158 const ok = await robotsCheck(url);
159 if (ok === 'unreachable') return { ...row, status: 'fetch_error', error: 'Website could not be reached' };
160 if (!ok) { stats.robotsBlocked++; return { ...row, status: 'blocked_by_robots_txt', error: 'The site\'s robots.txt does not allow automated access to this page' }; }
161 await polite(url);
162 let res;
163 try { res = await get(url); } catch (e) { return { ...row, status: 'fetch_error', error: `Could not load page: ${e.code || e.message}` }; }
164 const finalUrl = res.url || url; row.loadedUrl = finalUrl;
165 if (new URL(finalUrl).origin !== new URL(url).origin && (await robotsCheck(finalUrl)) !== true) { stats.robotsBlocked++; return { ...row, status: 'blocked_by_robots_txt', error: 'Redirect target is disallowed by robots.txt' }; }
166 if (res.statusCode >= 400) return { ...row, status: 'http_error', error: `Page returned HTTP ${res.statusCode}${res.statusCode === 403 ? ' (bot protection; try enabling a proxy)' : ''}` };
167 const ct = String(res.headers['content-type'] || ''); let html = String(res.body ?? '');
168 if (!/html|xml/i.test(ct) && !/<html|<body|<div/i.test(html.slice(0, 3000))) return { ...row, status: 'not_html', error: `Not an HTML page (${ct || 'unknown type'})` };
169 if (html.length > MAX_BODY) html = html.slice(0, MAX_BODY);
170 let page;
171 try { page = htmlToMarkdown(html, finalUrl, { contentSelector, removeSelectors, keepHeaderFooter }); } catch (e) { return { ...row, status: 'parse_error', error: `Could not parse HTML: ${e.message}` }; }
172 html = null;
173 row.title = page.title;
174
175 if (depth < depthCap) for (const l of page.links) { const n = normUrl(l); if (!seen.has(n) && wanted(n, depth + 1)) { seen.add(n); queue.push({ url: n, depth: depth + 1 }); enqueued++; } }
176 if (!page.markdown || page.markdown.replace(/[\s#*_|>-]+/g, '').length < 5) return { ...row, status: 'empty_page', error: 'Page has no readable text (it may need JavaScript to render)' };
177 const overhead = 600 + estTokens(prompt) + (schema ? estTokens(JSON.stringify(schema)) : 0);
178 const budget = ai.maxIn - overhead;
179 if (budget < 200) return { ...row, status: 'input_too_large', error: 'Instructions + schema use the whole input token budget; raise "Max input tokens per page" or shorten them' };
180 const t = truncateToTokens(page.markdown, budget); row.truncated = t.truncated; if (t.truncated) stats.truncatedPages++;
181 if (includeMarkdown) row.markdown = t.text;
182
183 if (aiCap && stats.aiCostUsd >= aiCap) { stop ??= `Reached your AI cost cap of $${aiCap}`; return { ...row, status: 'skipped_cost_cap', error: stop }; }
184 if (!byok && stats.aiCostUsd > 0.05 + 0.5 * PRICE_BUILTIN * Math.max(1, charged)) { stop ??= 'Built-in AI cost guard tripped (unusually large pages); stopping'; return { ...row, status: 'skipped_cost_cap', error: stop }; }
185 let feedback = null;
186 for (let a = 0; a <= retries; a++) {
187 if (fatal) return { ...row, status: 'ai_error', error: fatal };
188 try {
189 stats.aiCalls++;
190 const r = await extract(aiCfg,
191 { instructions: prompt, schema, page: { url: finalUrl, title: page.title, markdown: t.text }, feedback });
192 row.inputTokens += r.inputTokens; row.outputTokens += r.outputTokens; row.model = r.model;
193 stats.inputTokens += r.inputTokens; stats.outputTokens += r.outputTokens; stats.aiCostUsd += costUsd(ai.model, r.inputTokens, r.outputTokens);
194 let data = schema ? stripNulls(r.data, schema) : r.data;
195 const v = validate(schema, data);
196 if (!v.ok) { feedback = `it did not match the schema (${v.errors})`; if (a < retries) { stats.aiRetries++; continue; } return { ...row, status: 'invalid_output', error: `AI output did not match the schema: ${v.errors}` }; }
197 if (schema?.['x-wrapped']) data = data.items;
198 return { ...row, data, extractedAt: new Date().toISOString() };
199 } catch (e) {
200 if (!(e instanceof LlmError)) throw e;
201 if (e.fatal) { fatal ??= byok ? e.message : `Built-in AI is temporarily unavailable (${e.status ?? 'error'}). Try again later or add your own AI API key.`; return { ...row, status: 'ai_error', error: fatal }; }
202 if (a < retries) { stats.aiRetries++; if (/not valid JSON|max output/i.test(e.message)) feedback = e.message; await sleep(e.retryable ? 2000 * 2 ** a : 500); continue; }
203 return { ...row, status: 'ai_error', error: e.message };
204 }
205 }
206 return { ...row, status: 'ai_error', error: 'AI extraction failed' };
207 }
208
209 async function save(row) {
210 counts[row.status] = (counts[row.status] ?? 0) + 1;
211 row.extractedAt ??= new Date().toISOString();
212 if (row.status !== 'ok') { await Actor.pushData(row); return; }
213 if (charging) {
214 const left = charging.calculateMaxEventChargeCountWithinLimit(EVENT);
215 if (Number.isFinite(left) && left < 1) {
216 stop ??= 'Reached the maximum cost you set for this run'; counts.ok--; counts.skipped_max_cost = (counts.skipped_max_cost ?? 0) + 1;
217 await Actor.pushData({ url: row.url, depth: row.depth, status: 'skipped_max_cost', error: 'Not saved: the run reached the maximum cost you set.', data: null, extractedAt: row.extractedAt });
218 return;
219 }
220 }
221 const r = await Actor.pushData(row, EVENT); charged++;
222 if (r?.eventChargeLimitReached) stop ??= 'Reached the maximum cost you set for this run';
223 }
224
225 let active = 0; let done = 0;
226 const worker = async () => {
227 for (;;) {
228 if (stop || fatal) return;
229 const job = queue.shift();
230 if (!job) { if (active === 0) return; await sleep(200); continue; }
231 active++;
232 let row;
233 try { row = await processPage(job); } catch (e) { row = { url: job.url, depth: job.depth, status: 'error', error: e.message, data: null }; }
234 await save(row); active--;
235 if (++done % 10 === 0) log.info(`Processed ${done} pages (${counts.ok ?? 0} extracted, ${queue.length} queued)`);
236 }
237 };
238 await Promise.all(Array.from({ length: conc }, worker));
239
240
241 const notProcessed = queue.length;
242 const freePlanNote = trial.limited && (capHit || requestedPages > pageCap && enqueued >= pageCap) ? `Free plan: processed up to ${pageCap} pages (you asked for ${requestedPages}). Add your own AI API key (llmApiKey), or use any paid Apify plan, to process more.` : null;
243 const notice = [fatal || stop ? `Stopped: ${fatal || stop}${notProcessed ? `; ${notProcessed} queued pages not processed (not charged)` : ''}.` : null,
244 freePlanNote, !trial.limited && capHit ? `Reached maxPages (${pageCap}); more links were found. Raise maxPages to crawl further.` : null].filter(Boolean).join(' ');
245 if (notice) await Actor.pushData({ url: null, status: fatal || stop ? 'stopped' : 'limited', error: notice, data: null, extractedAt: new Date().toISOString() });
246 const summary = { pagesProcessed: done, extracted: counts.ok ?? 0, charged, byStatus: counts, aiKey: ai.mode, provider: ai.provider, model: ai.model,
247 truncated: !!notice, maxPages: pageCap, freePlanLimited: !!freePlanNote, notice: notice || null,
248 zeroResultsReason: counts.ok ? null : (Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ') || 'no pages processed') + '; see the status and error fields of each row.',
249 stoppedBecause: fatal || stop || null, stats: { ...stats, aiCostUsd: Number(stats.aiCostUsd.toFixed(6)) }, finishedAt: new Date().toISOString() };
250 await Actor.setValue('OUTPUT', summary);
251 log.info(`Summary: ${JSON.stringify(summary)}`);
252 if (fatal && !(counts.ok)) throw new Error(fatal);
253 const fetchFails = (counts.fetch_error ?? 0);
254 if (done >= 3 && !(counts.ok) && fetchFails === done) throw new Error(`None of the ${done} pages could be reached. This looks like a network problem; try again or enable a proxy.`);
255 const other = Object.entries(counts).filter(([k]) => k !== 'ok').map(([k, v]) => `${v} ${k}`).join(', ');
256 await Actor.exit(`${counts.ok ? '' : '0 results: '}Extracted ${counts.ok ?? 0} of ${done + invalid.length} pages${other ? ` (${other}; not charged)` : ''}${fatal || stop ? `. Stopped: ${fatal || stop}` : ''}.${freePlanNote ? ` ${freePlanNote}` : ''}`);
257} catch (e) {
258 if (e instanceof InputError) { log.error(e.message); await Actor.fail(`Input error: ${e.message}`); }
259 else { log.exception(e, 'Run failed'); await Actor.fail(e.message); }
260}
