1import { Actor, log } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4
5await Actor.init();
6
7const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
8const USER_AGENT = 'Mozilla/5.0 (compatible; LahcenAI-Bot/1.0; +https://apify.com/lahcen_ai)';
9
10
11async function fetchRetry(url, options = {}, { retries = 2, timeoutMs = 20000 } = {}) {
12 let lastError;
13 for (let attempt = 0; attempt <= retries; attempt++) {
14 try {
15 const res = await fetch(url, {
16 redirect: 'follow',
17 ...options,
18 headers: { 'user-agent': USER_AGENT, ...(options.headers || {}) },
19 signal: AbortSignal.timeout(timeoutMs),
20 });
21 if ((res.status === 429 || res.status >= 500) && attempt < retries) {
22 const retryAfter = Number(res.headers.get('retry-after'));
23 await res.body?.cancel().catch(() => {});
24 await sleep(Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter, 30) * 1000 : 1500 * 2 ** attempt);
25 continue;
26 }
27 return res;
28 } catch (e) {
29 lastError = e;
30 if (attempt < retries) await sleep(1500 * 2 ** attempt);
31 }
32 }
33 throw lastError;
34}
35
36
37async function readText(res, maxBytes = 5_000_000) {
38 if (!res.body) return '';
39 const chunks = [];
40 let size = 0;
41 for await (const chunk of res.body) {
42 size += chunk.length;
43 if (size > maxBytes) {
44 chunks.push(Buffer.from(chunk).subarray(0, Math.max(0, chunk.length - (size - maxBytes))));
45 break;
46 }
47 chunks.push(Buffer.from(chunk));
48 }
49 return Buffer.concat(chunks).toString('utf8');
50}
51
52
53async function mapPool(items, concurrency, fn) {
54 const results = new Array(items.length);
55 let next = 0;
56 const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
57 while (next < items.length) {
58 const i = next++;
59 results[i] = await fn(items[i], i);
60 }
61 });
62 await Promise.all(workers);
63 return results;
64}
65
66
67function datasetBudget() {
68 try {
69 const charging = Actor.getChargingManager();
70 if (charging.getPricingInfo().isPayPerEvent) {
71 return charging.calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
72 }
73 } catch { }
74 return Infinity;
75}
76
77
78class Output {
79 constructor() {
80 this.budget = datasetBudget();
81 this.count = 0;
82 }
83
84 get full() { return this.count >= this.budget; }
85
86 async push(item) {
87 if (this.full) return false;
88 this.count++;
89 await Actor.pushData(item);
90 return true;
91 }
92
93
94 fail(item) {
95 this.failed = this.failed || [];
96 this.failed.push(item);
97 log.warning(`Failed: ${JSON.stringify(item).slice(0, 300)}`);
98 }
99
100 async finish(label) {
101 if (this.failed?.length) await Actor.setValue('FAILED_INPUTS', this.failed);
102 const msg = `Done: ${this.count} ${label}${this.failed?.length ? `, ${this.failed.length} failed (free, see FAILED_INPUTS)` : ''}.`;
103 if (!this.count && this.failed?.length) {
104 await Actor.fail(`No results: all ${this.failed.length} input(s) failed. First error: ${this.failed[0].error}`);
105 return;
106 }
107 if (this.full) log.warning('Stopped early: the maximum charge limit for this run was reached.');
108 await Actor.setStatusMessage(msg, { isStatusMessageTerminal: true });
109 await Actor.exit();
110 }
111}
112
113
114function toHostname(value, { stripWww = false } = {}) {
115 let s = String(value ?? '').trim().toLowerCase();
116 if (!s) return null;
117 if (s.includes('@') && !s.includes('/')) s = s.split('@').pop();
118 if (!/^[a-z][a-z0-9+.-]*:\/\//.test(s)) s = `http://${s}`;
119 try {
120 let host = new URL(s).hostname.replace(/\.$/, '');
121 if (stripWww) host = host.replace(/^www\./, '');
122 if (!/^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9-]{2,63}$/.test(host)) return null;
123 return host;
124 } catch {
125 return null;
126 }
127}
128
129
130function toUrl(value) {
131 let s = String(typeof value === 'object' && value ? value.url : value ?? '').trim();
132 if (!s) return null;
133 if (!/^https?:\/\//i.test(s)) s = `https://${s}`;
134 try {
135 const u = new URL(s);
136 if (!['http:', 'https:'].includes(u.protocol)) return null;
137 u.hash = '';
138 return u.toString();
139 } catch {
140 return null;
141 }
142}
143
144function isPrivateIp(ip) {
145 if (net.isIPv4(ip)) {
146 const [a, b] = ip.split('.').map(Number);
147 return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31)
148 || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224;
149 }
150 const v = ip.toLowerCase();
151 if (v.startsWith('::ffff:')) return isPrivateIp(v.slice(7));
152 return v === '::1' || v === '::' || v.startsWith('fc') || v.startsWith('fd') || v.startsWith('fe80');
153}
154
155
156async function assertPublicHost(hostname) {
157 if (net.isIP(hostname)) {
158 if (isPrivateIp(hostname)) throw new Error('Private or internal addresses are not allowed.');
159 return;
160 }
161 const records = await dns.lookup(hostname, { all: true });
162 if (!records.length || records.some((r) => isPrivateIp(r.address))) {
163 throw new Error('Host resolves to a private or internal address.');
164 }
165}
166
167
168function collectList(...values) {
169 const out = [];
170 for (const v of values) {
171 if (!v) continue;
172 const arr = Array.isArray(v) ? v : String(v).split(/[\n,;]+/);
173 for (const item of arr) {
174 const s = typeof item === 'object' && item ? (item.url ?? item.value ?? '') : String(item);
175 if (s.trim()) out.push(s.trim());
176 }
177 }
178 return [...new Set(out)];
179}
180
181function clampInt(value, fallback, min, max) {
182 const n = Number.isFinite(Number(value)) ? Math.trunc(Number(value)) : fallback;
183 return Math.min(max, Math.max(min, n));
184}
185
186
187function decodeEntities(s) {
188 return String(s ?? '')
189 .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(Number(d)))
190 .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16)))
191 .replace(/"/g, '"').replace(/'|'/g, "'").replace(/</g, '<').replace(/>/g, '>')
192 .replace(/ /g, ' ').replace(/&/g, '&');
193}
194
195function parseAttrs(tag) {
196 const attrs = {};
197 const re = /([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*(?:=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
198 const inner = tag.replace(/^<\s*[a-zA-Z0-9-]+/, '').replace(/\/?>$/, '');
199 let m;
200 while ((m = re.exec(inner))) attrs[m[1].toLowerCase()] = decodeEntities(m[2] ?? m[3] ?? m[4] ?? '');
201 return attrs;
202}
203
204function findTags(html, name) {
205 return [...String(html).matchAll(new RegExp(`<${name}\\b[^>]*>`, 'gi'))].map((m) => parseAttrs(m[0]));
206}
207
208function textOf(html) {
209 return decodeEntities(String(html)
210 .replace(/<script[\s\S]*?<\/script>/gi, ' ')
211 .replace(/<style[\s\S]*?<\/style>/gi, ' ')
212 .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
213 .replace(/<[^>]+>/g, ' '))
214 .replace(/\s+/g, ' ').trim();
215}
216
217function firstMatch(html, re) {
218 const m = String(html).match(re);
219 return m ? textOf(m[1]).trim() : null;
220}
221
222
223function extractMeta(html, baseUrl) {
224 const metas = findTags(html, 'meta');
225 const links = findTags(html, 'link');
226 const byName = (n) => metas.find((m) => (m.name || '').toLowerCase() === n)?.content ?? null;
227 const abs = (u) => { try { return u ? new URL(u, baseUrl).toString() : null; } catch { return null; } };
228 const og = {};
229 const twitter = {};
230 for (const m of metas) {
231 const key = (m.property || m.name || '').toLowerCase();
232 if (key.startsWith('og:') && m.content != null && !(key in og)) og[key.slice(3)] = m.content;
233 if (key.startsWith('twitter:') && m.content != null && !(key in twitter)) twitter[key.slice(8)] = m.content;
234 }
235 const jsonLd = [];
236 for (const m of String(html).matchAll(/<script\b[^>]*type\s*=\s*["']?application\/ld\+json["']?[^>]*>([\s\S]*?)<\/script>/gi)) {
237 try { jsonLd.push(JSON.parse(m[1].trim())); } catch { jsonLd.push({ parseError: true }); }
238 }
239 const schemaTypes = new Set();
240 const walk = (node) => {
241 if (Array.isArray(node)) return node.forEach(walk);
242 if (node && typeof node === 'object') {
243 const t = node['@type'];
244 if (t) (Array.isArray(t) ? t : [t]).forEach((x) => schemaTypes.add(String(x)));
245 for (const v of Object.values(node)) if (v && typeof v === 'object') walk(v);
246 }
247 };
248 walk(jsonLd);
249 const htmlTag = (String(html).match(/<html\b[^>]*>/i) || [''])[0];
250 const relHas = (l, r) => (l.rel || '').toLowerCase().split(/\s+/).includes(r);
251 return {
252 title: firstMatch(html, /<title[^>]*>([\s\S]*?)<\/title>/i),
253 metaDescription: byName('description'),
254 metaRobots: byName('robots'),
255 canonical: abs(links.find((l) => relHas(l, 'canonical'))?.href),
256 lang: htmlTag ? parseAttrs(htmlTag).lang ?? null : null,
257 h1: [...String(html).matchAll(/<h1\b[^>]*>([\s\S]*?)<\/h1>/gi)].map((m) => textOf(m[1])).filter(Boolean),
258 favicon: abs(links.find((l) => relHas(l, 'icon'))?.href) ?? abs('/favicon.ico'),
259 openGraph: og,
260 twitterCard: twitter,
261 jsonLd,
262 schemaTypes: [...schemaTypes],
263 alternateFeeds: links.filter((l) => relHas(l, 'alternate') && /rss|atom/i.test(l.type || '')).map((l) => abs(l.href)).filter(Boolean),
264 hreflang: links.filter((l) => relHas(l, 'alternate') && l.hreflang).map((l) => ({ hreflang: l.hreflang, href: abs(l.href) })),
265 };
266}
267
268
269function parseRobots(text) {
270 const groups = [];
271 let current = null;
272 let lastWasAgent = false;
273 const sitemaps = [];
274 for (const raw of String(text || '').split(/\r?\n/)) {
275 const line = raw.replace(/#.*$/, '').trim();
276 const m = line.match(/^([a-zA-Z-]+)\s*:\s*(.*)$/);
277 if (!m) continue;
278 const key = m[1].toLowerCase();
279 const value = m[2].trim();
280 if (key === 'user-agent') {
281 if (!lastWasAgent || !current) { current = { agents: [], rules: [] }; groups.push(current); }
282 current.agents.push(value.toLowerCase());
283 lastWasAgent = true;
284 } else if (key === 'allow' || key === 'disallow') {
285 if (current) current.rules.push({ allow: key === 'allow', path: value });
286 lastWasAgent = false;
287 } else if (key === 'sitemap') {
288 sitemaps.push(value);
289 } else {
290 lastWasAgent = false;
291 }
292 }
293 return { groups, sitemaps };
294}
295
296function robotsAllows(robots, agent, path = '/') {
297 const ua = agent.toLowerCase();
298 let matched = robots.groups.filter((g) => g.agents.some((a) => a !== '*' && ua.includes(a)));
299 let usedGroup = 'specific';
300 if (!matched.length) { matched = robots.groups.filter((g) => g.agents.includes('*')); usedGroup = matched.length ? 'wildcard' : 'none'; }
301 const rules = matched.flatMap((g) => g.rules).filter((r) => r.path !== '' || r.allow);
302 let best = null;
303 for (const r of rules) {
304 if (r.path === '') continue;
305 const pattern = r.path.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
306 const re = new RegExp(`^${pattern.endsWith('\\$') ? `${pattern.slice(0, -2)}$` : pattern}`);
307 if (re.test(path)) {
308 if (!best || r.path.length > best.path.length || (r.path.length === best.path.length && r.allow)) best = r;
309 }
310 }
311 return { allowed: best ? best.allow : true, matchedGroup: usedGroup, rule: best ? `${best.allow ? 'Allow' : 'Disallow'}: ${best.path}` : null };
312}
313
314import tls from 'node:tls';
315
316function getCertificate(host, port, timeoutMs) {
317 return new Promise((resolve, reject) => {
318 const started = Date.now();
319 const socket = tls.connect({ host, port, servername: net.isIP(host) ? undefined : host, rejectUnauthorized: false, ALPNProtocols: ['h2', 'http/1.1'] });
320 const timer = setTimeout(() => { socket.destroy(); reject(new Error(`TLS handshake timed out after ${timeoutMs} ms`)); }, timeoutMs);
321 socket.once('secureConnect', () => {
322 clearTimeout(timer);
323 const cert = socket.getPeerCertificate(true);
324 const result = {
325 cert,
326 authorized: socket.authorized,
327 authorizationError: socket.authorizationError ? String(socket.authorizationError) : null,
328 protocol: socket.getProtocol(),
329 cipher: socket.getCipher()?.name ?? null,
330 alpn: socket.alpnProtocol || null,
331 handshakeMs: Date.now() - started,
332 remoteAddress: socket.remoteAddress,
333 };
334 socket.end();
335 resolve(result);
336 });
337 socket.once('error', (e) => { clearTimeout(timer); reject(e); });
338 });
339}
340
341function chainOf(cert) {
342 const chain = [];
343 let c = cert;
344 const seen = new Set();
345 while (c && c.fingerprint256 && !seen.has(c.fingerprint256)) {
346 seen.add(c.fingerprint256);
347 chain.push({ subject: c.subject?.CN ?? c.subject?.O ?? null, issuer: c.issuer?.CN ?? c.issuer?.O ?? null, validTo: c.valid_to ? new Date(c.valid_to).toISOString() : null });
348 c = c.issuerCertificate;
349 }
350 return chain;
351}
352
353const HEADER_CHECKS = [
354 ['strict-transport-security', 20, 'HSTS missing: browsers may be downgraded to HTTP.', 'Add "Strict-Transport-Security: max-age=31536000; includeSubDomains".'],
355 ['content-security-policy', 20, 'Content-Security-Policy missing: weaker XSS protection.', 'Define a Content-Security-Policy.'],
356 ['x-content-type-options', 10, 'X-Content-Type-Options missing.', 'Add "X-Content-Type-Options: nosniff".'],
357 ['x-frame-options', 10, 'Clickjacking protection missing (X-Frame-Options or CSP frame-ancestors).', 'Add "X-Frame-Options: SAMEORIGIN" or CSP frame-ancestors.'],
358 ['referrer-policy', 10, 'Referrer-Policy missing.', 'Add "Referrer-Policy: strict-origin-when-cross-origin".'],
359 ['permissions-policy', 10, 'Permissions-Policy missing.', 'Add a Permissions-Policy restricting camera, microphone, geolocation.'],
360 ['cross-origin-opener-policy', 5, 'Cross-Origin-Opener-Policy missing.', 'Add "Cross-Origin-Opener-Policy: same-origin".'],
361];
362
363async function checkHeaders(host, port) {
364 const base = `https://${host}${port === 443 ? '' : `:${port}`}/`;
365 let res = await fetchRetry(base, { redirect: 'follow' }, { retries: 1, timeoutMs: 15000 });
366 const h = res.headers;
367 await res.body?.cancel().catch(() => {});
368 const present = {};
369 const missing = [];
370 let score = 5;
371 const issues = [];
372 for (const [name, points, message, fix] of HEADER_CHECKS) {
373 let value = h.get(name);
374 if (name === 'x-frame-options' && !value && /frame-ancestors/i.test(h.get('content-security-policy') || '')) value = 'CSP frame-ancestors';
375 if (value) { present[name] = value; score += points; } else { missing.push(name); issues.push({ severity: points >= 20 ? 'high' : 'medium', code: `MISSING_${name.toUpperCase().replace(/-/g, '_')}`, message, fix }); }
376 }
377 const hsts = h.get('strict-transport-security');
378 const maxAge = Number(hsts?.match(/max-age=(\d+)/i)?.[1] ?? 0);
379 if (hsts && maxAge < 15552000) issues.push({ severity: 'low', code: 'HSTS_SHORT_MAX_AGE', message: `HSTS max-age is ${maxAge}s (< 180 days).`, fix: 'Use max-age=31536000.' });
380 const leaks = ['server', 'x-powered-by', 'x-aspnet-version', 'x-aspnetmvc-version'].filter((n) => h.get(n) && /\d/.test(h.get(n))).map((n) => `${n}: ${h.get(n)}`);
381 if (leaks.length) issues.push({ severity: 'low', code: 'VERSION_DISCLOSURE', message: `Software versions disclosed (${leaks.join('; ')}).`, fix: 'Hide version numbers in response headers.' });
382
383 let httpRedirectsToHttps = null;
384 if (port === 443) {
385 try {
386 const r = await fetchRetry(`http://${host}/`, { redirect: 'manual' }, { retries: 0, timeoutMs: 10000 });
387 await r.body?.cancel().catch(() => {});
388 const loc = r.headers.get('location') || '';
389 httpRedirectsToHttps = r.status >= 300 && r.status < 400 && /^https:/i.test(new URL(loc, `http://${host}/`).toString());
390 if (!httpRedirectsToHttps) issues.push({ severity: 'high', code: 'NO_HTTPS_REDIRECT', message: 'HTTP is not redirected to HTTPS.', fix: 'Redirect all http:// traffic to https:// with a 301.' });
391 } catch { httpRedirectsToHttps = null; }
392 }
393 const grade = score >= 90 ? 'A' : score >= 75 ? 'B' : score >= 55 ? 'C' : score >= 35 ? 'D' : 'F';
394 return { finalUrl: res.url, statusCode: res.status, headersScore: Math.min(100, score), headersGrade: grade, presentHeaders: present, missingHeaders: missing, httpRedirectsToHttps, headerIssues: issues };
395}
396
397async function checkHost(target, { port, warnDays, includeHeaders, timeoutMs }) {
398 const host = target;
399 await assertPublicHost(host);
400 const t = await getCertificate(host, port, timeoutMs);
401 const c = t.cert || {};
402 if (!c.valid_to) throw new Error('No certificate presented by the server.');
403 const validTo = new Date(c.valid_to);
404 const validFrom = new Date(c.valid_from);
405 const daysRemaining = Math.floor((validTo - Date.now()) / 86400000);
406 const sans = (c.subjectaltname || '').split(',').map((s) => s.trim().replace(/^DNS:/, '')).filter(Boolean);
407 const hostnameMatch = !tls.checkServerIdentity(host, c);
408 const selfSigned = c.issuer && c.subject && JSON.stringify(c.issuer) === JSON.stringify(c.subject);
409 const issues = [];
410 if (daysRemaining < 0) issues.push({ severity: 'critical', code: 'CERT_EXPIRED', message: `Certificate expired ${-daysRemaining} day(s) ago.`, fix: 'Renew the certificate now.' });
411 else if (daysRemaining <= warnDays) issues.push({ severity: 'high', code: 'CERT_EXPIRING_SOON', message: `Certificate expires in ${daysRemaining} day(s).`, fix: 'Renew or verify auto-renewal.' });
412 if (!hostnameMatch) issues.push({ severity: 'critical', code: 'HOSTNAME_MISMATCH', message: `Certificate is not valid for ${host}.`, fix: 'Issue a certificate that covers this hostname.' });
413 if (!t.authorized) issues.push({ severity: selfSigned ? 'critical' : 'high', code: 'CHAIN_NOT_TRUSTED', message: `Certificate chain not trusted: ${t.authorizationError}.`, fix: 'Install the full intermediate chain or use a publicly trusted CA.' });
414 if (['TLSv1', 'TLSv1.1'].includes(t.protocol)) issues.push({ severity: 'high', code: 'OLD_TLS', message: `Negotiated deprecated ${t.protocol}.`, fix: 'Enable TLS 1.2/1.3 only.' });
415 if (c.bits && c.bits < 2048 && !c.asn1Curve) issues.push({ severity: 'medium', code: 'WEAK_KEY', message: `Key size ${c.bits} bits.`, fix: 'Use RSA 2048+ or ECDSA.' });
416 const result = {
417 host,
418 port,
419 sslValid: t.authorized && hostnameMatch && daysRemaining >= 0,
420 daysRemaining,
421 validFrom: validFrom.toISOString(),
422 validTo: validTo.toISOString(),
423 expiryStatus: daysRemaining < 0 ? 'expired' : daysRemaining <= warnDays ? 'expiring_soon' : 'ok',
424 issuer: c.issuer?.O ?? c.issuer?.CN ?? null,
425 issuerCommonName: c.issuer?.CN ?? null,
426 subjectCommonName: c.subject?.CN ?? null,
427 subjectAltNames: sans,
428 sanCount: sans.length,
429 wildcard: sans.some((s) => s.startsWith('*.')),
430 hostnameMatch,
431 chainTrusted: t.authorized,
432 chainError: t.authorizationError,
433 selfSigned: Boolean(selfSigned),
434 tlsProtocol: t.protocol,
435 cipher: t.cipher,
436 http2: t.alpn === 'h2',
437 keyBits: c.bits ?? null,
438 keyCurve: c.asn1Curve ?? null,
439 serialNumber: c.serialNumber ?? null,
440 fingerprintSha256: c.fingerprint256 ?? null,
441 chain: chainOf(c),
442 ipAddress: t.remoteAddress,
443 handshakeMs: t.handshakeMs,
444 };
445 if (includeHeaders) {
446 try { Object.assign(result, await checkHeaders(host, port)); issues.push(...result.headerIssues); delete result.headerIssues; } catch (e) { result.headersError = e.message; }
447 }
448 const order = { critical: 0, high: 1, medium: 2, low: 3, info: 4 };
449 result.issues = issues.sort((a, b) => order[a.severity] - order[b.severity]);
450 result.issueCount = issues.length;
451 result.checkedAt = new Date().toISOString();
452 return result;
453}
454
455const input = (await Actor.getInput()) ?? {};
456const raw = collectList(input.domains);
457const hosts = [...new Set(raw.map((v) => toHostname(v)).filter(Boolean))];
458if (!hosts.length) throw new Error('Please provide at least one valid domain or URL in "domains" (for example "apify.com").');
459const opts = {
460 port: clampInt(input.port, 443, 1, 65535),
461 warnDays: clampInt(input.expiryWarningDays, 30, 1, 365),
462 includeHeaders: input.checkSecurityHeaders !== false,
463 timeoutMs: clampInt(input.timeoutSecs, 15, 3, 60) * 1000,
464};
465const out = new Output();
466log.info(`Checking ${hosts.length} host(s) on port ${opts.port}.`);
467await mapPool(hosts, clampInt(input.maxConcurrency, 10, 1, 50), async (host) => {
468 if (out.full) return;
469 let item;
470 try { item = await checkHost(host, opts); } catch (e) {
471 item = { host, port: opts.port, sslValid: false, expiryStatus: 'unreachable', error: e.code ? `${e.code}: ${e.message}` : e.message, checkedAt: new Date().toISOString() };
472 }
473 await out.push(item);
474});
475await out.finish('host(s) checked');
