1
2
3import { createHash } from 'node:crypto';
4import * as cheerio from 'cheerio';
5
6export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; WebsiteChangeMonitor/1.0; +https://apify.com)';
7export const MIN_REQUEST_INTERVAL_MS = 400;
8export const MAX_RETRIES = 3;
9export const SNIPPET_LENGTH = 500;
10const RETRY_BASE_MS = 500;
11const MAX_REDIRECTS = 10;
12
13const ALWAYS_REMOVED = 'script, style, noscript, template, svg, iframe';
14
15export const sha256 = (s) => createHash('sha256').update(s).digest('hex');
16export const sha1 = (s) => createHash('sha1').update(s).digest('hex');
17export const snapshotKey = (url) => `snapshot:${sha1(url)}`;
18
19
20export const toStoreKey = (key) => key.replace(/[^a-zA-Z0-9!\-_.'()]/g, '-');
21
22export const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
23
24
25export function createMemoryStore() {
26 const map = new Map();
27 return {
28 map,
29 get: async (key) => (map.has(key) ? structuredClone(map.get(key)) : null),
30 set: async (key, value) => { map.set(key, structuredClone(value)); },
31 };
32}
33
34
35export function validateUrl(raw) {
36 if (typeof raw !== 'string' || !raw.trim()) return { error: 'URL is empty or not a string' };
37 let u;
38 try {
39 u = new URL(raw.trim());
40 } catch {
41 return { error: `Invalid URL: ${raw}` };
42 }
43 if (u.protocol !== 'http:' && u.protocol !== 'https:') return { error: `Unsupported protocol "${u.protocol}" (only http/https)` };
44 return { url: u.href };
45}
46
47
48export function createThrottle(minIntervalMs = MIN_REQUEST_INTERVAL_MS) {
49 let last = 0;
50 return async () => {
51 const wait = last + minIntervalMs - Date.now();
52 if (wait > 0) await sleep(wait);
53 last = Date.now();
54 };
55}
56
57class HttpError extends Error {
58 constructor(message, { statusCode = null, retryable = false, finalUrl = null } = {}) {
59 super(message);
60 this.statusCode = statusCode;
61 this.retryable = retryable;
62 this.finalUrl = finalUrl;
63 }
64}
65
66function describeNetworkError(err) {
67 const cause = err?.cause;
68 const code = cause?.code || err?.code;
69 if (err?.name === 'AbortError' || err?.name === 'TimeoutError') return { msg: 'Request timed out', retryable: true };
70 if (code === 'ENOTFOUND' || code === 'EAI_AGAIN') {
71
72 return { msg: `DNS lookup failed (${code}${cause?.hostname ? ` ${cause.hostname}` : ''})`, retryable: code === 'EAI_AGAIN' };
73 }
74 if (code === 'ERR_INVALID_URL') return { msg: 'Invalid URL', retryable: false };
75 return { msg: `Network error: ${cause?.message || err?.message || String(err)}${code ? ` (${code})` : ''}`, retryable: true };
76}
77
78
79async function fetchOnce(url, { userAgent, timeoutSecs, followRedirects }) {
80 const controller = new AbortController();
81 const timer = setTimeout(() => controller.abort(), timeoutSecs * 1000);
82 try {
83 let current = url;
84 for (let hop = 0; ; hop++) {
85 let res;
86 try {
87 res = await fetch(current, {
88 method: 'GET',
89 redirect: 'manual',
90 signal: controller.signal,
91 headers: {
92 'user-agent': userAgent,
93 accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
94 'accept-language': 'en-US,en;q=0.8',
95 },
96 });
97 } catch (err) {
98 const { msg, retryable } = describeNetworkError(err);
99 throw new HttpError(msg, { retryable, finalUrl: current });
100 }
101 const location = res.headers.get('location');
102 if (res.status >= 300 && res.status < 400 && location) {
103 await res.body?.cancel().catch(() => {});
104 if (!followRedirects) {
105 throw new HttpError(`HTTP ${res.status} redirect to ${new URL(location, current).href} (followRedirects=false)`, { statusCode: res.status, finalUrl: current });
106 }
107 if (hop >= MAX_REDIRECTS) throw new HttpError(`Too many redirects (>${MAX_REDIRECTS})`, { statusCode: res.status, finalUrl: current });
108 current = new URL(location, current).href;
109 continue;
110 }
111 if (res.status < 200 || res.status > 299) {
112 await res.body?.cancel().catch(() => {});
113 const retryable = res.status === 429 || res.status >= 500;
114 const retryAfter = Number(res.headers.get('retry-after'));
115 const e = new HttpError(`HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ''}`, { statusCode: res.status, retryable, finalUrl: current });
116 if (retryable && Number.isFinite(retryAfter) && retryAfter > 0) e.retryAfterMs = Math.min(retryAfter, 30) * 1000;
117 throw e;
118 }
119 let body;
120 try {
121 body = await res.text();
122 } catch (err) {
123 const { msg, retryable } = describeNetworkError(err);
124 throw new HttpError(msg, { retryable, statusCode: res.status, finalUrl: current });
125 }
126 return { statusCode: res.status, finalUrl: current, body, contentType: res.headers.get('content-type') || '' };
127 }
128 } finally {
129 clearTimeout(timer);
130 }
131}
132
133
134export async function fetchWithRetry(url, opts) {
135 const { throttle, log, maxRetries = MAX_RETRIES, retryBaseMs = RETRY_BASE_MS } = opts;
136 for (let attempt = 0; ; attempt++) {
137 if (throttle) await throttle();
138 try {
139 return { ...(await fetchOnce(url, opts)), attempts: attempt + 1 };
140 } catch (err) {
141 if (!(err instanceof HttpError) || !err.retryable || attempt >= maxRetries) {
142 err.attempts = attempt + 1;
143 throw err;
144 }
145 const delay = err.retryAfterMs ?? retryBaseMs * 2 ** attempt;
146 log?.warning?.(`${url}: ${err.message} — retry ${attempt + 1}/${maxRetries} in ${delay}ms`);
147 await sleep(delay);
148 }
149 }
150}
151
152
153export function extractText(html, { selector, ignoreSelectors = [] } = {}) {
154 const $ = cheerio.load(html);
155 $(ALWAYS_REMOVED).remove();
156 for (const sel of ignoreSelectors) {
157 if (!sel || !String(sel).trim()) continue;
158 try {
159 $(sel).remove();
160 } catch (err) {
161 throw new Error(`Invalid ignoreSelector "${sel}": ${err.message}`);
162 }
163 }
164 let parts;
165 if (selector) {
166 let nodes;
167 try {
168 nodes = $(selector);
169 } catch (err) {
170 throw new Error(`Invalid selector "${selector}": ${err.message}`);
171 }
172 parts = nodes.map((_, el) => $(el).text()).get();
173 return { text: parts.map(normalizeWhitespace).filter(Boolean).join('\n'), matched: nodes.length };
174 }
175 const root = $('body').length ? $('body') : $.root();
176 return { text: normalizeWhitespace(root.text()), matched: null };
177}
178
179export const normalizeWhitespace = (s) => String(s ?? '').replace(/\s+/g, ' ').trim();
180
181
182
183
184
185
186
187
188export async function checkUrl(rawUrl, options, store) {
189 const {
190 selector = null,
191 ignoreSelectors = [],
192 userAgent = DEFAULT_USER_AGENT,
193 timeoutSecs = 30,
194 followRedirects = true,
195 saveHtml = false,
196 log,
197 } = options;
198 const selectorUsed = selector || null;
199 const row = {
200 url: typeof rawUrl === 'string' ? rawUrl.trim() : String(rawUrl),
201 finalUrl: null,
202 statusCode: null,
203 checkedAt: new Date().toISOString(),
204 contentHash: null,
205 previousHash: null,
206 changed: false,
207 changeType: 'error',
208 textSnippet: null,
209 error: null,
210 httpElapsedMs: null,
211 selectorUsed,
212 htmlKey: null,
213 };
214
215 const v = validateUrl(rawUrl);
216 if (v.error) {
217 row.error = v.error;
218 return row;
219 }
220 row.url = v.url;
221
222 const key = snapshotKey(v.url);
223 const started = Date.now();
224 let res;
225 try {
226 res = await fetchWithRetry(v.url, { ...options, userAgent, timeoutSecs, followRedirects });
227 } catch (err) {
228 row.httpElapsedMs = Date.now() - started;
229 row.statusCode = err.statusCode ?? null;
230 row.finalUrl = err.finalUrl ?? null;
231 row.error = err.attempts > 1 ? `${err.message} (after ${err.attempts} attempts)` : err.message;
232 return row;
233 } finally {
234 row.checkedAt = new Date().toISOString();
235 }
236 row.httpElapsedMs = Date.now() - started;
237 row.statusCode = res.statusCode;
238 row.finalUrl = res.finalUrl;
239
240 let extracted;
241 try {
242 extracted = extractText(res.body, { selector: selectorUsed, ignoreSelectors });
243 } catch (err) {
244 row.error = err.message;
245 return row;
246 }
247 if (selectorUsed && extracted.matched === 0) {
248
249 log?.warning?.(`${v.url}: selector "${selectorUsed}" matched no elements`);
250 }
251
252 const hash = sha256(extracted.text);
253 const previous = await store.get(key);
254 row.contentHash = hash;
255 row.previousHash = previous?.hash ?? null;
256 row.textSnippet = extracted.text.slice(0, SNIPPET_LENGTH);
257 if (!previous?.hash) row.changeType = 'new';
258 else row.changeType = previous.hash === hash ? 'unchanged' : 'changed';
259
260 row.changed = row.changeType === 'changed';
261
262 if (saveHtml) {
263
264 const logical = `html:${hash}`;
265 row.htmlKey = (await store.setHtml?.(logical, res.body)) ?? logical;
266 }
267 await store.set(key, { hash, checkedAt: row.checkedAt, textSnippet: row.textSnippet, url: v.url, selector: selectorUsed });
268 return row;
269}
270
271
272
273
274
275export async function runChecks(input, store, { log, onResult } = {}) {
276 const maxUrls = clampInt(input.maxUrls, 50, 1, 200);
277 const timeoutSecs = clampInt(input.timeoutSecs, 30, 1, 300);
278 const seen = new Set();
279 const urls = [];
280 for (const raw of input.urls ?? []) {
281 const k = validateUrl(raw).url ?? String(raw).trim();
282 if (seen.has(k)) continue;
283 seen.add(k);
284 urls.push(raw);
285 }
286 if (urls.length > maxUrls) log?.warning?.(`Got ${urls.length} unique URLs, checking only the first ${maxUrls} (maxUrls).`);
287 const throttle = createThrottle(input.minIntervalMs ?? MIN_REQUEST_INTERVAL_MS);
288 const opts = {
289 selector: input.selector?.trim() || null,
290 ignoreSelectors: Array.isArray(input.ignoreSelectors) ? input.ignoreSelectors.filter((s) => typeof s === 'string' && s.trim()) : [],
291 userAgent: input.userAgent?.trim() || DEFAULT_USER_AGENT,
292 timeoutSecs,
293 followRedirects: input.followRedirects !== false,
294 saveHtml: input.saveHtml === true,
295 throttle,
296 log,
297 retryBaseMs: input.retryBaseMs,
298 };
299 const rows = [];
300 for (const raw of urls.slice(0, maxUrls)) {
301 const row = await checkUrl(raw, opts, store);
302 rows.push(row);
303 log?.info?.(`${row.changeType.toUpperCase().padEnd(9)} ${row.url}${row.error ? ` — ${row.error}` : ''}`);
304 if (onResult) await onResult(row);
305 }
306 return rows;
307}
308
309function clampInt(value, def, min, max) {
310 const n = Number.isFinite(Number(value)) && value !== null && value !== '' ? Math.trunc(Number(value)) : def;
311 return Math.min(max, Math.max(min, n));
312}
