1import { Actor, log } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4
5await Actor.init();
6
7const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
8const USER_AGENT = 'Mozilla/5.0 (compatible; LahcenAI-Bot/1.0; +https://apify.com/lahcen_ai)';
9
10
11async function fetchRetry(url, options = {}, { retries = 2, timeoutMs = 20000 } = {}) {
12 let lastError;
13 for (let attempt = 0; attempt <= retries; attempt++) {
14 try {
15 const res = await fetch(url, {
16 redirect: 'follow',
17 ...options,
18 headers: { 'user-agent': USER_AGENT, ...(options.headers || {}) },
19 signal: AbortSignal.timeout(timeoutMs),
20 });
21 if ((res.status === 429 || res.status >= 500) && attempt < retries) {
22 const retryAfter = Number(res.headers.get('retry-after'));
23 await res.body?.cancel().catch(() => {});
24 await sleep(Number.isFinite(retryAfter) && retryAfter > 0 ? Math.min(retryAfter, 30) * 1000 : 1500 * 2 ** attempt);
25 continue;
26 }
27 return res;
28 } catch (e) {
29 lastError = e;
30 if (attempt < retries) await sleep(1500 * 2 ** attempt);
31 }
32 }
33 throw lastError;
34}
35
36
37async function readText(res, maxBytes = 5_000_000) {
38 if (!res.body) return '';
39 const chunks = [];
40 let size = 0;
41 for await (const chunk of res.body) {
42 size += chunk.length;
43 if (size > maxBytes) {
44 chunks.push(Buffer.from(chunk).subarray(0, Math.max(0, chunk.length - (size - maxBytes))));
45 break;
46 }
47 chunks.push(Buffer.from(chunk));
48 }
49 return Buffer.concat(chunks).toString('utf8');
50}
51
52
53async function mapPool(items, concurrency, fn) {
54 const results = new Array(items.length);
55 let next = 0;
56 const workers = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
57 while (next < items.length) {
58 const i = next++;
59 results[i] = await fn(items[i], i);
60 }
61 });
62 await Promise.all(workers);
63 return results;
64}
65
66
67function datasetBudget() {
68 try {
69 const charging = Actor.getChargingManager();
70 if (charging.getPricingInfo().isPayPerEvent) {
71 return charging.calculateMaxEventChargeCountWithinLimit('apify-default-dataset-item');
72 }
73 } catch { }
74 return Infinity;
75}
76
77
78class Output {
79 constructor() {
80 this.budget = datasetBudget();
81 this.count = 0;
82 }
83
84 get full() { return this.count >= this.budget; }
85
86 async push(item) {
87 if (this.full) return false;
88 this.count++;
89 await Actor.pushData(item);
90 return true;
91 }
92
93
94 fail(item) {
95 this.failed = this.failed || [];
96 this.failed.push(item);
97 log.warning(`Failed: ${JSON.stringify(item).slice(0, 300)}`);
98 }
99
100 async finish(label) {
101 if (this.failed?.length) await Actor.setValue('FAILED_INPUTS', this.failed);
102 const msg = `Done: ${this.count} ${label}${this.failed?.length ? `, ${this.failed.length} failed (free, see FAILED_INPUTS)` : ''}.`;
103 if (!this.count && this.failed?.length) {
104 await Actor.fail(`No results: all ${this.failed.length} input(s) failed. First error: ${this.failed[0].error}`);
105 return;
106 }
107 if (this.full) log.warning('Stopped early: the maximum charge limit for this run was reached.');
108 await Actor.setStatusMessage(msg, { isStatusMessageTerminal: true });
109 await Actor.exit();
110 }
111}
112
113
114function toHostname(value, { stripWww = false } = {}) {
115 let s = String(value ?? '').trim().toLowerCase();
116 if (!s) return null;
117 if (s.includes('@') && !s.includes('/')) s = s.split('@').pop();
118 if (!/^[a-z][a-z0-9+.-]*:\/\//.test(s)) s = `http://${s}`;
119 try {
120 let host = new URL(s).hostname.replace(/\.$/, '');
121 if (stripWww) host = host.replace(/^www\./, '');
122 if (!/^(?=.{1,253}$)(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z0-9-]{2,63}$/.test(host)) return null;
123 return host;
124 } catch {
125 return null;
126 }
127}
128
129
130function toUrl(value) {
131 let s = String(typeof value === 'object' && value ? value.url : value ?? '').trim();
132 if (!s) return null;
133 if (!/^https?:\/\//i.test(s)) s = `https://${s}`;
134 try {
135 const u = new URL(s);
136 if (!['http:', 'https:'].includes(u.protocol)) return null;
137 u.hash = '';
138 return u.toString();
139 } catch {
140 return null;
141 }
142}
143
144function isPrivateIp(ip) {
145 if (net.isIPv4(ip)) {
146 const [a, b] = ip.split('.').map(Number);
147 return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31)
148 || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224;
149 }
150 const v = ip.toLowerCase();
151 if (v.startsWith('::ffff:')) return isPrivateIp(v.slice(7));
152 return v === '::1' || v === '::' || v.startsWith('fc') || v.startsWith('fd') || v.startsWith('fe80');
153}
154
155
156async function assertPublicHost(hostname) {
157 if (net.isIP(hostname)) {
158 if (isPrivateIp(hostname)) throw new Error('Private or internal addresses are not allowed.');
159 return;
160 }
161 const records = await dns.lookup(hostname, { all: true });
162 if (!records.length || records.some((r) => isPrivateIp(r.address))) {
163 throw new Error('Host resolves to a private or internal address.');
164 }
165}
166
167
168function collectList(...values) {
169 const out = [];
170 for (const v of values) {
171 if (!v) continue;
172 const arr = Array.isArray(v) ? v : String(v).split(/[\n,;]+/);
173 for (const item of arr) {
174 const s = typeof item === 'object' && item ? (item.url ?? item.value ?? '') : String(item);
175 if (s.trim()) out.push(s.trim());
176 }
177 }
178 return [...new Set(out)];
179}
180
181function clampInt(value, fallback, min, max) {
182 const n = Number.isFinite(Number(value)) ? Math.trunc(Number(value)) : fallback;
183 return Math.min(max, Math.max(min, n));
184}
185
186
187function decodeEntities(s) {
188 return String(s ?? '')
189 .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(Number(d)))
190 .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16)))
191 .replace(/"/g, '"').replace(/'|'/g, "'").replace(/</g, '<').replace(/>/g, '>')
192 .replace(/ /g, ' ').replace(/&/g, '&');
193}
194
195function parseAttrs(tag) {
196 const attrs = {};
197 const re = /([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*(?:=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
198 const inner = tag.replace(/^<\s*[a-zA-Z0-9-]+/, '').replace(/\/?>$/, '');
199 let m;
200 while ((m = re.exec(inner))) attrs[m[1].toLowerCase()] = decodeEntities(m[2] ?? m[3] ?? m[4] ?? '');
201 return attrs;
202}
203
204function findTags(html, name) {
205 return [...String(html).matchAll(new RegExp(`<${name}\\b[^>]*>`, 'gi'))].map((m) => parseAttrs(m[0]));
206}
207
208function textOf(html) {
209 return decodeEntities(String(html)
210 .replace(/<script[\s\S]*?<\/script>/gi, ' ')
211 .replace(/<style[\s\S]*?<\/style>/gi, ' ')
212 .replace(/<noscript[\s\S]*?<\/noscript>/gi, ' ')
213 .replace(/<[^>]+>/g, ' '))
214 .replace(/\s+/g, ' ').trim();
215}
216
217function firstMatch(html, re) {
218 const m = String(html).match(re);
219 return m ? textOf(m[1]).trim() : null;
220}
221
222
223function extractMeta(html, baseUrl) {
224 const metas = findTags(html, 'meta');
225 const links = findTags(html, 'link');
226 const byName = (n) => metas.find((m) => (m.name || '').toLowerCase() === n)?.content ?? null;
227 const abs = (u) => { try { return u ? new URL(u, baseUrl).toString() : null; } catch { return null; } };
228 const og = {};
229 const twitter = {};
230 for (const m of metas) {
231 const key = (m.property || m.name || '').toLowerCase();
232 if (key.startsWith('og:') && m.content != null && !(key in og)) og[key.slice(3)] = m.content;
233 if (key.startsWith('twitter:') && m.content != null && !(key in twitter)) twitter[key.slice(8)] = m.content;
234 }
235 const jsonLd = [];
236 for (const m of String(html).matchAll(/<script\b[^>]*type\s*=\s*["']?application\/ld\+json["']?[^>]*>([\s\S]*?)<\/script>/gi)) {
237 try { jsonLd.push(JSON.parse(m[1].trim())); } catch { jsonLd.push({ parseError: true }); }
238 }
239 const schemaTypes = new Set();
240 const walk = (node) => {
241 if (Array.isArray(node)) return node.forEach(walk);
242 if (node && typeof node === 'object') {
243 const t = node['@type'];
244 if (t) (Array.isArray(t) ? t : [t]).forEach((x) => schemaTypes.add(String(x)));
245 for (const v of Object.values(node)) if (v && typeof v === 'object') walk(v);
246 }
247 };
248 walk(jsonLd);
249 const htmlTag = (String(html).match(/<html\b[^>]*>/i) || [''])[0];
250 const relHas = (l, r) => (l.rel || '').toLowerCase().split(/\s+/).includes(r);
251 return {
252 title: firstMatch(html, /<title[^>]*>([\s\S]*?)<\/title>/i),
253 metaDescription: byName('description'),
254 metaRobots: byName('robots'),
255 canonical: abs(links.find((l) => relHas(l, 'canonical'))?.href),
256 lang: htmlTag ? parseAttrs(htmlTag).lang ?? null : null,
257 h1: [...String(html).matchAll(/<h1\b[^>]*>([\s\S]*?)<\/h1>/gi)].map((m) => textOf(m[1])).filter(Boolean),
258 favicon: abs(links.find((l) => relHas(l, 'icon'))?.href) ?? abs('/favicon.ico'),
259 openGraph: og,
260 twitterCard: twitter,
261 jsonLd,
262 schemaTypes: [...schemaTypes],
263 alternateFeeds: links.filter((l) => relHas(l, 'alternate') && /rss|atom/i.test(l.type || '')).map((l) => abs(l.href)).filter(Boolean),
264 hreflang: links.filter((l) => relHas(l, 'alternate') && l.hreflang).map((l) => ({ hreflang: l.hreflang, href: abs(l.href) })),
265 };
266}
267
268
269function parseRobots(text) {
270 const groups = [];
271 let current = null;
272 let lastWasAgent = false;
273 const sitemaps = [];
274 for (const raw of String(text || '').split(/\r?\n/)) {
275 const line = raw.replace(/#.*$/, '').trim();
276 const m = line.match(/^([a-zA-Z-]+)\s*:\s*(.*)$/);
277 if (!m) continue;
278 const key = m[1].toLowerCase();
279 const value = m[2].trim();
280 if (key === 'user-agent') {
281 if (!lastWasAgent || !current) { current = { agents: [], rules: [] }; groups.push(current); }
282 current.agents.push(value.toLowerCase());
283 lastWasAgent = true;
284 } else if (key === 'allow' || key === 'disallow') {
285 if (current) current.rules.push({ allow: key === 'allow', path: value });
286 lastWasAgent = false;
287 } else if (key === 'sitemap') {
288 sitemaps.push(value);
289 } else {
290 lastWasAgent = false;
291 }
292 }
293 return { groups, sitemaps };
294}
295
296function robotsAllows(robots, agent, path = '/') {
297 const ua = agent.toLowerCase();
298 let matched = robots.groups.filter((g) => g.agents.some((a) => a !== '*' && ua.includes(a)));
299 let usedGroup = 'specific';
300 if (!matched.length) { matched = robots.groups.filter((g) => g.agents.includes('*')); usedGroup = matched.length ? 'wildcard' : 'none'; }
301 const rules = matched.flatMap((g) => g.rules).filter((r) => r.path !== '' || r.allow);
302 let best = null;
303 for (const r of rules) {
304 if (r.path === '') continue;
305 const pattern = r.path.replace(/[.+?^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*');
306 const re = new RegExp(`^${pattern.endsWith('\\$') ? `${pattern.slice(0, -2)}$` : pattern}`);
307 if (re.test(path)) {
308 if (!best || r.path.length > best.path.length || (r.path.length === best.path.length && r.allow)) best = r;
309 }
310 }
311 return { allowed: best ? best.allow : true, matchedGroup: usedGroup, rule: best ? `${best.allow ? 'Allow' : 'Disallow'}: ${best.path}` : null };
312}
313
314
315
316let bootstrap = null;
317async function rdapBaseFor(domain) {
318 if (!bootstrap) {
319 const res = await fetchRetry('https://data.iana.org/rdap/dns.json', { headers: { accept: 'application/json' } });
320 if (!res.ok) throw new Error(`IANA RDAP bootstrap unavailable (HTTP ${res.status})`);
321 const json = await res.json();
322 bootstrap = new Map();
323 for (const [tlds, urls] of json.services) {
324 const url = urls.find((u) => u.startsWith('https://')) ?? urls[0];
325 for (const t of tlds) bootstrap.set(t.toLowerCase(), url.endsWith('/') ? url : `${url}/`);
326 }
327 }
328 const labels = domain.split('.');
329 for (let i = 1; i < labels.length; i++) {
330 const suffix = labels.slice(i).join('.');
331 if (bootstrap.has(suffix)) return bootstrap.get(suffix);
332 }
333 return null;
334}
335
336const lastHit = new Map();
337async function politeGet(url) {
338 const host = new URL(url).host;
339 const wait = 400 - (Date.now() - (lastHit.get(host) || 0));
340 lastHit.set(host, Date.now() + Math.max(0, wait));
341 if (wait > 0) await sleep(wait);
342 return fetchRetry(url, { headers: { accept: 'application/rdap+json, application/json' } }, { retries: 3, timeoutMs: 20000 });
343}
344
345function vcardValue(entity, key) {
346 const arr = entity?.vcardArray?.[1];
347 if (!Array.isArray(arr)) return null;
348 const row = arr.find((r) => r[0] === key);
349 if (!row) return null;
350 return Array.isArray(row[3]) ? row[3].filter(Boolean).join(' ') : row[3];
351}
352
353function findEntities(entities, role, acc = []) {
354 for (const e of entities || []) {
355 if ((e.roles || []).includes(role)) acc.push(e);
356 if (e.entities) findEntities(e.entities, role, acc);
357 }
358 return acc;
359}
360
361function parseRdap(domain, json, source) {
362 const events = {};
363 for (const ev of json.events || []) if (ev.eventAction && ev.eventDate && !events[ev.eventAction]) events[ev.eventAction] = ev.eventDate;
364 const registrar = findEntities(json.entities, 'registrar')[0];
365 const abuse = findEntities(registrar?.entities || json.entities, 'abuse')[0];
366 const created = events.registration ? new Date(events.registration) : null;
367 const expires = events.expiration ? new Date(events.expiration) : null;
368 const now = Date.now();
369 const status = (json.status || []).map((s) => String(s).toLowerCase());
370 const daysUntilExpiry = expires ? Math.floor((expires - now) / 86400000) : null;
371 return {
372 domain,
373 registered: true,
374 availability: 'registered',
375 domainAgeDays: created ? Math.floor((now - created) / 86400000) : null,
376 domainAgeYears: created ? Math.round(((now - created) / 31557600000) * 10) / 10 : null,
377 createdDate: created?.toISOString() ?? null,
378 updatedDate: events['last changed'] ? new Date(events['last changed']).toISOString() : null,
379 expiryDate: expires?.toISOString() ?? null,
380 daysUntilExpiry,
381 expiringWithin30Days: daysUntilExpiry !== null && daysUntilExpiry <= 30,
382 registrar: vcardValue(registrar, 'fn') ?? registrar?.handle ?? null,
383 registrarIanaId: registrar?.publicIds?.find((p) => /iana/i.test(p.type))?.identifier ?? null,
384 registrarUrl: registrar?.links?.find((l) => l.rel === 'about' || l.rel === 'related')?.href ?? registrar?.url ?? null,
385 registrarAbuseEmail: vcardValue(abuse, 'email'),
386 registrarAbusePhone: vcardValue(abuse, 'tel'),
387 statuses: status,
388 transferLocked: status.some((s) => s.includes('transfer prohibited')),
389 pendingDelete: status.some((s) => s.includes('pending delete') || s.includes('redemption period')),
390 nameservers: (json.nameservers || []).map((n) => String(n.ldhName || '').toLowerCase()).filter(Boolean),
391 dnssec: json.secureDNS ? Boolean(json.secureDNS.delegationSigned) : null,
392 handle: json.handle ?? null,
393 rdapSource: source,
394 checkedAt: new Date().toISOString(),
395 };
396}
397
398async function lookup(domain) {
399 const base = await rdapBaseFor(domain);
400 const url = base ? `${base}domain/${encodeURIComponent(domain)}` : `https://rdap.org/domain/${encodeURIComponent(domain)}`;
401 const res = await politeGet(url);
402 if (res.status === 404) {
403 await res.body?.cancel().catch(() => {});
404 return { domain, registered: false, availability: 'not_found_possibly_available', rdapSource: url, checkedAt: new Date().toISOString(), note: 'The registry returned "not found". The domain is likely available (confirm with a registrar; reserved/premium names may differ).' };
405 }
406 if (!res.ok) {
407 await res.body?.cancel().catch(() => {});
408 if (!base) throw new Error(`No RDAP service is published for this TLD (HTTP ${res.status}). Some ccTLDs still only offer legacy WHOIS.`);
409 throw new Error(`RDAP server answered HTTP ${res.status}`);
410 }
411 const json = await res.json();
412
413 return parseRdap(domain, json, res.url || url);
414}
415
416function registrableDomain(host) {
417
418 return host.replace(/^www\./, '');
419}
420
421const input = (await Actor.getInput()) ?? {};
422const raw = collectList(input.domains);
423const domains = [...new Set(raw.map((v) => toHostname(v, { stripWww: true })).filter(Boolean).map(registrableDomain))];
424if (!domains.length) throw new Error('Please provide at least one valid domain in "domains" (for example "apify.com").');
425const out = new Output();
426log.info(`Looking up ${domains.length} domain(s) via RDAP.`);
427let done = 0;
428await mapPool(domains, clampInt(input.maxConcurrency, 5, 1, 20), async (domain) => {
429 if (out.full) return;
430 let item;
431 try { item = await lookup(domain); } catch (e) {
432 out.fail({ domain, error: e.message });
433 return;
434 }
435 if (input.onlyExpiringWithinDays && item.daysUntilExpiry != null && item.daysUntilExpiry > Number(input.onlyExpiringWithinDays)) return;
436 await out.push(item);
437 if (++done % 25 === 0) await Actor.setStatusMessage(`Looked up ${done}/${domains.length}`);
438});
439await out.finish('domain(s) looked up');
