1import { Actor } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4import { fileURLToPath } from 'node:url';
5
6const USER_AGENT = 'IndieWebEndpointAuditor/0.1 (+https://apify.com)';
7const DEFAULT_TIMEOUT_SECONDS = 10;
8const DEFAULT_MAX_HTML_BYTES = 1024 * 1024;
9const MAX_HTML_BYTES = 2 * 1024 * 1024;
10
11
12
13
14
15
16
17
18function isPrivateIPv4(ip) {
19 const parts = ip.split('.').map(Number);
20 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;
21 const [a, b] = parts;
22 return a === 10
23 || (a === 172 && b >= 16 && b <= 31)
24 || (a === 192 && b === 168)
25 || a === 127
26 || a === 0
27 || (a === 169 && b === 254);
28}
29
30function isPrivateIPv6(ip) {
31 const normalized = ip.toLowerCase();
32 return normalized === '::1'
33 || normalized.startsWith('fc')
34 || normalized.startsWith('fd')
35 || normalized.startsWith('fe80:');
36}
37
38export async function normalizeAndValidateUrl(rawUrl) {
39 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');
40 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {
41 throw new Error('Only HTTP and HTTPS URLs are supported');
42 }
43
44 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;
45 const url = new URL(withScheme);
46 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');
47 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');
48
49
50 const hostLiteral = url.hostname.startsWith('[') && url.hostname.endsWith(']')
51 ? url.hostname.slice(1, -1)
52 : url.hostname;
53
54 const literalType = net.isIP(hostLiteral);
55 if (literalType === 4 && isPrivateIPv4(hostLiteral)) throw new Error('Private IPv4 targets are blocked');
56 if (literalType === 6 && isPrivateIPv6(hostLiteral)) throw new Error('Private IPv6 targets are blocked');
57
58 const records = literalType ? [{ address: hostLiteral, family: literalType }] : await dns.lookup(url.hostname, { all: true });
59 for (const record of records) {
60 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');
61 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');
62 }
63 return url;
64}
65
66
67
68
69
70async function fetchHtml(initialUrl, timeoutSeconds, maxBytes, redirectsRemaining = 3) {
71 await normalizeAndValidateUrl(initialUrl.href);
72 const controller = new AbortController();
73 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);
74 try {
75 const response = await fetch(initialUrl, {
76 redirect: 'manual',
77 signal: controller.signal,
78 headers: {
79 'user-agent': USER_AGENT,
80 accept: 'text/html,application/xhtml+xml,*/*;q=0.1',
81 },
82 });
83
84 if ([301, 302, 303, 307, 308].includes(response.status)) {
85 if (redirectsRemaining <= 0) throw new Error('Too many redirects');
86 const location = response.headers.get('location');
87 if (!location) throw new Error('Redirect without Location header');
88 const nextUrl = new URL(location, initialUrl.href);
89 await normalizeAndValidateUrl(nextUrl.href);
90 return fetchHtml(nextUrl, timeoutSeconds, maxBytes, redirectsRemaining - 1);
91 }
92
93
94 const reader = response.body.getReader();
95 const chunks = [];
96 let total = 0;
97 let truncated = false;
98 while (true) {
99 const { done, value } = await reader.read();
100 if (done) break;
101 if (total + value.length > maxBytes) {
102 chunks.push(value.slice(0, Math.max(0, maxBytes - total)));
103 truncated = true;
104 break;
105 }
106 chunks.push(value);
107 total += value.length;
108 }
109 try { await reader.cancel(); } catch { }
110
111 const html = new TextDecoder('utf-8', { fatal: false }).decode(Buffer.concat(chunks));
112 const headerMap = {};
113 response.headers.forEach((value, key) => { headerMap[key.toLowerCase()] = value; });
114
115 return {
116 ok: response.ok,
117 status: response.status,
118 finalUrl: response.url || initialUrl.href,
119 https: (response.url || initialUrl.href).startsWith('https://'),
120 html,
121 headers: headerMap,
122 truncated,
123 error: null,
124 };
125 } catch (error) {
126 return {
127 ok: false,
128 status: null,
129 finalUrl: initialUrl.href,
130 https: initialUrl.protocol === 'https:',
131 html: '',
132 headers: {},
133 truncated: false,
134 error: error.message,
135 };
136 } finally {
137 clearTimeout(timeout);
138 }
139}
140
141
142
143
144
145function decodeHtmlEntities(value) {
146 if (!value) return '';
147 return value
148 .replace(/&/gi, '&')
149 .replace(/"/gi, '"')
150 .replace(/'|'/gi, "'")
151 .replace(/</gi, '<')
152 .replace(/>/gi, '>')
153 .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
154 .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)))
155 .trim();
156}
157
158function parseAttributes(tag) {
159 const attrs = {};
160 const attrPattern = /([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*(?:=\s*("[^"]*"|'[^']*'|[^\s"'>/]+))?/g;
161 for (const match of tag.matchAll(attrPattern)) {
162 if (match[2] === undefined) {
163 attrs[match[1].toLowerCase()] = '';
164 } else {
165 const raw = match[2];
166 const unquoted = raw.startsWith('"') || raw.startsWith("'") ? raw.slice(1, -1) : raw;
167 attrs[match[1].toLowerCase()] = decodeHtmlEntities(unquoted);
168 }
169 }
170 return attrs;
171}
172
173function allLinkTags(html) {
174 return [...html.matchAll(/<link\b[^>]*>/gi)].map((m) => ({ raw: m[0], attrs: parseAttributes(m[0]) }));
175}
176
177function allAnchorTags(html) {
178 return [...html.matchAll(/<a\b[^>]*>/gi)].map((m) => ({ raw: m[0], attrs: parseAttributes(m[0]) }));
179}
180
181
182
183
184
185
186
187export function parseLinkHeader(headerValue) {
188 if (!headerValue || typeof headerValue !== 'string') return [];
189 const results = [];
190
191 const parts = headerValue.split(/,(?=\s*<)/);
192 for (const part of parts) {
193 const urlMatch = part.match(/<([^>]*)>/);
194 if (!urlMatch) continue;
195 const url = urlMatch[1].trim();
196 const relMatch = part.match(/rel\s*=\s*"?([^";]+)"?/i);
197 const rel = relMatch ? relMatch[1].trim().toLowerCase() : '';
198 results.push({ url, rel });
199 }
200 return results;
201}
202
203
204
205
206
207const ENDPOINT_RELS = [
208 { key: 'authorizationEndpoint', rel: 'authorization_endpoint', label: 'IndieAuth authorization endpoint' },
209 { key: 'tokenEndpoint', rel: 'token_endpoint', label: 'IndieAuth token endpoint' },
210 { key: 'indieauthMetadata', rel: 'indieauth-metadata', label: 'IndieAuth server metadata document' },
211 { key: 'micropubEndpoint', rel: 'micropub', label: 'Micropub endpoint' },
212 { key: 'webmentionEndpoint', rel: 'webmention', label: 'Webmention endpoint' },
213 { key: 'microsubEndpoint', rel: 'microsub', label: 'Microsub endpoint' },
214];
215
216function resolveAbsolute(href, baseUrl) {
217 try {
218 return new URL(href, baseUrl).href;
219 } catch {
220 return null;
221 }
222}
223
224export function discoverEndpoints(html, headers, baseUrl) {
225 const linkTags = allLinkTags(html);
226 const headerLinks = parseLinkHeader(headers.link || headers.Link || '');
227
228 const found = {};
229 for (const def of ENDPOINT_RELS) {
230 found[def.key] = null;
231 }
232
233
234 for (const tag of linkTags) {
235 const rels = (tag.attrs.rel || '').toLowerCase().split(/\s+/).filter(Boolean);
236 for (const def of ENDPOINT_RELS) {
237 if (found[def.key]) continue;
238 if (rels.includes(def.rel) && tag.attrs.href) {
239 found[def.key] = {
240 url: tag.attrs.href,
241 resolvedUrl: resolveAbsolute(tag.attrs.href, baseUrl),
242 source: 'html',
243 };
244 }
245 }
246 }
247
248
249 for (const entry of headerLinks) {
250 for (const def of ENDPOINT_RELS) {
251 if (found[def.key]) continue;
252 if (entry.rel === def.rel && entry.url) {
253 found[def.key] = {
254 url: entry.url,
255 resolvedUrl: resolveAbsolute(entry.url, baseUrl),
256 source: 'header',
257 };
258 }
259 }
260 }
261
262
263 const relMeLinks = [];
264 const seenRelMe = new Set();
265 for (const tag of [...linkTags, ...allAnchorTags(html)]) {
266 const rels = (tag.attrs.rel || '').toLowerCase().split(/\s+/).filter(Boolean);
267 if (rels.includes('me') && tag.attrs.href) {
268 const resolvedUrl = resolveAbsolute(tag.attrs.href, baseUrl);
269 const key = resolvedUrl || tag.attrs.href;
270 if (seenRelMe.has(key)) continue;
271 seenRelMe.add(key);
272 relMeLinks.push({ href: tag.attrs.href, resolvedUrl });
273 }
274 }
275
276 return { endpoints: found, relMeLinks };
277}
278
279export function analyzeEndpoints(endpoints, relMeLinks) {
280 const issues = [];
281 const indieAuthReady = Boolean(endpoints.authorizationEndpoint) && Boolean(endpoints.tokenEndpoint);
282
283 if (!endpoints.authorizationEndpoint) {
284 issues.push('No authorization_endpoint link found; IndieAuth sign-in is unavailable.');
285 }
286 if (endpoints.authorizationEndpoint && !endpoints.tokenEndpoint) {
287 issues.push('authorization_endpoint is present but token_endpoint is missing; full IndieAuth flows (non-profile scopes) cannot complete.');
288 }
289 if (!endpoints.micropubEndpoint && endpoints.tokenEndpoint) {
290 issues.push('token_endpoint is present but no micropub endpoint was found; IndieAuth clients cannot publish content.');
291 }
292 if (!endpoints.webmentionEndpoint) {
293 issues.push('No webmention endpoint found; the site cannot receive IndieWeb webmention notifications.');
294 }
295 if (relMeLinks.length === 0) {
296 issues.push('No rel="me" identity links found; IndieAuth identity verification (RelMeAuth) is unavailable.');
297 }
298
299 return { indieAuthReady, issues };
300}
301
302export function scoreAudit(endpoints, relMeLinks, issues) {
303 let score = 0;
304 if (endpoints.authorizationEndpoint) score += 25;
305 if (endpoints.tokenEndpoint) score += 20;
306 if (endpoints.micropubEndpoint) score += 20;
307 if (endpoints.webmentionEndpoint) score += 20;
308 if (relMeLinks.length > 0) score += 15;
309 score = Math.max(0, Math.min(100, score));
310 return score;
311}
312
313export function gradeFromScore(score) {
314 if (score >= 95) return 'A+';
315 if (score >= 85) return 'A';
316 if (score >= 75) return 'B';
317 if (score >= 65) return 'C';
318 if (score >= 50) return 'D';
319 if (score >= 30) return 'E';
320 return 'F';
321}
322
323export function buildRecommendations(endpoints, relMeLinks) {
324 const recs = new Set();
325 if (!endpoints.authorizationEndpoint) {
326 recs.add('Add <link rel="authorization_endpoint" href="..."> (or an equivalent HTTP Link header) so IndieAuth clients can discover where to send sign-in requests.');
327 }
328 if (!endpoints.tokenEndpoint) {
329 recs.add('Add <link rel="token_endpoint" href="..."> so IndieAuth clients can exchange authorization codes for access tokens.');
330 }
331 if (!endpoints.indieauthMetadata) {
332 recs.add('Consider publishing an IndieAuth server metadata document (rel="indieauth-metadata") per the current IndieAuth spec for richer client discovery.');
333 }
334 if (!endpoints.micropubEndpoint) {
335 recs.add('Add <link rel="micropub" href="..."> if the site supports remote content publishing via the Micropub protocol.');
336 }
337 if (!endpoints.webmentionEndpoint) {
338 recs.add('Add <link rel="webmention" href="..."> so other sites can notify this page about mentions, replies, and likes.');
339 }
340 if (relMeLinks.length === 0) {
341 recs.add('Add rel="me" links to social profiles or an email mailto: link to support RelMeAuth-based identity verification.');
342 }
343 if (recs.size === 0) {
344 recs.add('IndieWeb endpoint discovery looks complete. Schedule this audit periodically to catch regressions.');
345 }
346 return [...recs];
347}
348
349
350
351
352
353export async function auditIndiewebEndpoints(input) {
354 const startUrl = await normalizeAndValidateUrl(input.startUrl);
355 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);
356 const maxHtmlBytes = Math.min(Math.max(Number(input.maxHtmlBytes || DEFAULT_MAX_HTML_BYTES), 16384), MAX_HTML_BYTES);
357
358 const fetchResult = await fetchHtml(startUrl, timeoutSeconds, maxHtmlBytes);
359
360 if (fetchResult.error) {
361 return {
362 inputUrl: input.startUrl,
363 finalUrl: fetchResult.finalUrl,
364 https: fetchResult.https,
365 endpoints: {},
366 relMeLinks: [],
367 indieAuthReady: false,
368 score: 0,
369 grade: 'F',
370 issues: [],
371 checkedAt: new Date().toISOString(),
372 recommendations: ['The request failed before HTML could be inspected. Verify the URL is reachable and try again.'],
373 error: fetchResult.error,
374 };
375 }
376
377 const contentType = fetchResult.headers['content-type'] || '';
378 const isHtml = /text\/html|application\/xhtml\+xml/i.test(contentType);
379
380 const { endpoints, relMeLinks } = isHtml
381 ? discoverEndpoints(fetchResult.html, fetchResult.headers, fetchResult.finalUrl)
382 : discoverEndpoints('', fetchResult.headers, fetchResult.finalUrl);
383
384 const { indieAuthReady, issues } = analyzeEndpoints(endpoints, relMeLinks);
385 const score = scoreAudit(endpoints, relMeLinks, issues);
386 const grade = gradeFromScore(score);
387 const recommendations = buildRecommendations(endpoints, relMeLinks);
388
389 return {
390 inputUrl: input.startUrl,
391 finalUrl: fetchResult.finalUrl,
392 https: fetchResult.https,
393 endpoints,
394 relMeLinks,
395 indieAuthReady,
396 score,
397 grade,
398 issues,
399 checkedAt: new Date().toISOString(),
400 recommendations,
401 error: null,
402 };
403}
404
405
406
407
408
409const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
410
411if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {
412 await Actor.init();
413 try {
414 const input = await Actor.getInput();
415 const result = await auditIndiewebEndpoints(input || {});
416 await Actor.pushData(result);
417 await Actor.setValue('OUTPUT', result);
418 Actor.log.info('IndieWeb endpoint audit complete', { finalUrl: result.finalUrl, indieAuthReady: result.indieAuthReady, score: result.score, grade: result.grade });
419 } finally {
420 await Actor.exit();
421 }
422}