1import { Actor } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4import { fileURLToPath } from 'node:url';
5
6const USER_AGENT = 'OEmbedDiscoveryAuditor/0.1 (+https://apify.com)';
7const DEFAULT_TIMEOUT_SECONDS = 10;
8const DEFAULT_MAX_BYTES = 1024 * 1024;
9const MAX_BYTES_CAP = 2 * 1024 * 1024;
10const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
11
12
13
14
15
16
17
18
19function isPrivateIPv4(ip) {
20 const parts = ip.split('.').map(Number);
21 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;
22 const [a, b] = parts;
23 return a === 10
24 || (a === 172 && b >= 16 && b <= 31)
25 || (a === 192 && b === 168)
26 || a === 127
27 || a === 0
28 || (a === 169 && b === 254);
29}
30
31function isPrivateIPv6(ip) {
32 const normalized = ip.toLowerCase();
33 return normalized === '::1'
34 || normalized.startsWith('fc')
35 || normalized.startsWith('fd')
36 || normalized.startsWith('fe80:');
37}
38
39export async function normalizeAndValidateUrl(rawUrl) {
40 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('URL is required');
41 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {
42 throw new Error('Only HTTP and HTTPS URLs are supported');
43 }
44
45 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;
46 const url = new URL(withScheme);
47 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');
48 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');
49
50 const hostname = url.hostname.replace(/^\[|\]$/g, '');
51 const literalType = net.isIP(hostname);
52 if (literalType === 4 && isPrivateIPv4(hostname)) throw new Error('Private IPv4 targets are blocked');
53 if (literalType === 6 && isPrivateIPv6(hostname)) throw new Error('Private IPv6 targets are blocked');
54
55 const records = literalType ? [{ address: hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });
56 for (const record of records) {
57 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');
58 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');
59 }
60 return url;
61}
62
63function clampInteger(value, fallback, min, max) {
64 const parsed = Number(value);
65 if (!Number.isFinite(parsed)) return fallback;
66 return Math.min(Math.max(Math.trunc(parsed), min), max);
67}
68
69
70
71
72
73function decodeBasicEntities(value) {
74 return (value || '').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"').replace(/'/g, "'");
75}
76
77function getAttr(tag, name) {
78 const match = tag.match(new RegExp(`\\s${name}\\s*=\\s*("([^"]*)"|'([^']*)'|([^\\s>]+))`, 'i'));
79 return match ? decodeBasicEntities((match[2] ?? match[3] ?? match[4] ?? '').trim()) : null;
80}
81
82const OEMBED_TYPES = new Set(['application/json+oembed', 'text/xml+oembed']);
83
84
85
86
87
88export function extractDiscoveryLinks(html, baseUrl) {
89 const base = new URL(baseUrl);
90 const links = [];
91 for (const match of html.matchAll(/<link\b[^>]*>/gi)) {
92 const tag = match[0];
93 const rel = (getAttr(tag, 'rel') || '').toLowerCase();
94 if (!rel.split(/\s+/).includes('alternate')) continue;
95 const type = (getAttr(tag, 'type') || '').toLowerCase();
96 if (!OEMBED_TYPES.has(type)) continue;
97 const href = getAttr(tag, 'href');
98 if (!href) continue;
99 let resolved;
100 try { resolved = new URL(href, base).href; } catch { continue; }
101 if (!/^https?:\/\//i.test(resolved)) continue;
102 links.push({ type, href, resolvedUrl: resolved, title: getAttr(tag, 'title') || null });
103 }
104 return links;
105}
106
107
108
109
110
111async function boundedFetch(initialUrl, timeoutSeconds, maxBytes, accept, redirectsRemaining = 3) {
112 await normalizeAndValidateUrl(initialUrl.href);
113 const controller = new AbortController();
114 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);
115 try {
116 const response = await fetch(initialUrl, {
117 method: 'GET',
118 redirect: 'manual',
119 signal: controller.signal,
120 headers: { 'user-agent': USER_AGENT, accept },
121 });
122
123 if (REDIRECT_STATUSES.has(response.status)) {
124 if (redirectsRemaining <= 0) throw new Error('Too many redirects');
125 const location = response.headers.get('location');
126 if (!location) throw new Error('Redirect without Location header');
127 const nextUrl = new URL(location, initialUrl.href);
128 await normalizeAndValidateUrl(nextUrl.href);
129 return boundedFetch(nextUrl, timeoutSeconds, maxBytes, accept, redirectsRemaining - 1);
130 }
131
132 const reader = response.body?.getReader();
133 const chunks = [];
134 let received = 0;
135 if (reader) {
136 while (true) {
137 const { done, value } = await reader.read();
138 if (done) break;
139 received += value.byteLength;
140 if (received > maxBytes) {
141 try { reader.cancel(); } catch { }
142 break;
143 }
144 chunks.push(value);
145 }
146 }
147 const body = Buffer.concat(chunks).toString('utf8');
148
149 return {
150 ok: response.ok,
151 status: response.status,
152 finalUrl: response.url || initialUrl.href,
153 contentType: response.headers.get('content-type') || '',
154 https: (response.url || initialUrl.href).startsWith('https://'),
155 body,
156 error: null,
157 };
158 } catch (error) {
159 return {
160 ok: false,
161 status: null,
162 finalUrl: initialUrl.href,
163 contentType: '',
164 https: initialUrl.protocol === 'https:',
165 body: '',
166 error: error.message,
167 };
168 } finally {
169 clearTimeout(timeout);
170 }
171}
172
173
174
175
176
177const VALID_TYPES = new Set(['photo', 'video', 'link', 'rich']);
178
179
180
181
182
183export function validateOembedResponse(doc) {
184 const issues = [];
185 if (doc === null || typeof doc !== 'object' || Array.isArray(doc)) {
186 return { valid: false, type: null, version: null, issues: ['oEmbed response is not a JSON object.'] };
187 }
188
189 const type = typeof doc.type === 'string' ? doc.type : null;
190 if (!type) {
191 issues.push("'type' member is required and must be a string.");
192 } else if (!VALID_TYPES.has(type)) {
193 issues.push(`'type' value '${type}' is not one of the four defined types: photo, video, link, rich.`);
194 }
195
196 const version = typeof doc.version === 'string' ? doc.version : null;
197 if (!version) {
198 issues.push("'version' member is required and must be a string.");
199 } else if (version !== '1.0') {
200 issues.push(`'version' is '${version}'; the current oEmbed spec version is '1.0'.`);
201 }
202
203
204 if (type === 'photo') {
205 if (typeof doc.url !== 'string') issues.push("type 'photo' requires a string 'url' member (source of the image).");
206 if (typeof doc.width !== 'number') issues.push("type 'photo' requires a numeric 'width' member.");
207 if (typeof doc.height !== 'number') issues.push("type 'photo' requires a numeric 'height' member.");
208 }
209 if (type === 'video' || type === 'rich') {
210 if (typeof doc.html !== 'string') issues.push(`type '${type}' requires a string 'html' member (embed markup).`);
211 if (typeof doc.width !== 'number') issues.push(`type '${type}' requires a numeric 'width' member.`);
212 if (typeof doc.height !== 'number') issues.push(`type '${type}' requires a numeric 'height' member.`);
213 }
214
215
216 const optionalStringFields = ['title', 'author_name', 'author_url', 'provider_name', 'provider_url', 'cache_age', 'thumbnail_url'];
217 for (const field of optionalStringFields) {
218 if (field in doc && field !== 'cache_age' && typeof doc[field] !== 'string') {
219 issues.push(`'${field}' is present but is not a string.`);
220 }
221 }
222 if ('thumbnail_width' in doc && typeof doc.thumbnail_width !== 'number') issues.push("'thumbnail_width' is present but is not a number.");
223 if ('thumbnail_height' in doc && typeof doc.thumbnail_height !== 'number') issues.push("'thumbnail_height' is present but is not a number.");
224
225 return { valid: issues.length === 0, type, version, issues };
226}
227
228
229
230
231
232function evalDiscovery({ discoveryLinks, discoveryFetchError }) {
233 if (discoveryFetchError) {
234 return { status: 'missing', note: `Could not fetch the page to look for discovery links: ${discoveryFetchError}`, recommendation: 'Verify the page URL is public and reachable.' };
235 }
236 if (discoveryLinks.length === 0) {
237 return { status: 'missing', note: 'No <link rel="alternate" type="application/json+oembed|text/xml+oembed"> discovery tag found.', recommendation: 'Publish a <link rel="alternate" type="application/json+oembed" href="..."> discovery tag so embedding clients can auto-discover the oEmbed endpoint (oEmbed spec section 2.2).' };
238 }
239 return { status: 'good', note: `${discoveryLinks.length} oEmbed discovery link(s) found.` };
240}
241
242function evalJsonDiscoveryPresent({ discoveryLinks }) {
243 const hasJson = discoveryLinks.some((l) => l.type === 'application/json+oembed');
244 if (discoveryLinks.length === 0) return { status: 'info', note: 'No discovery links to evaluate.' };
245 if (!hasJson) {
246 return { status: 'warn', note: 'Only an XML (text/xml+oembed) discovery link was found; no JSON discovery link.', recommendation: 'Publish a JSON discovery link (type="application/json+oembed") since most modern embedding clients prefer JSON.' };
247 }
248 return { status: 'good', note: 'A JSON (application/json+oembed) discovery link is present.' };
249}
250
251function evalEndpointFetch({ endpointResult, endpointTested }) {
252 if (!endpointTested) return { status: 'info', note: 'No oEmbed endpoint was tested.' };
253 if (endpointResult.error) {
254 return { status: 'missing', note: `oEmbed endpoint request failed: ${endpointResult.error}`, recommendation: 'Verify the discovered oEmbed endpoint URL is reachable.' };
255 }
256 if (endpointResult.status !== 200) {
257 return { status: 'missing', note: `oEmbed endpoint returned HTTP ${endpointResult.status} instead of 200.`, recommendation: 'Serve HTTP 200 with a valid oEmbed response from the discovered endpoint URL.' };
258 }
259 return { status: 'good', note: 'oEmbed endpoint responded 200.' };
260}
261
262function evalEndpointHttps({ endpointResult, endpointTested }) {
263 if (!endpointTested || endpointResult.error) return { status: 'info', note: 'No response to evaluate.' };
264 if (!endpointResult.https) {
265 return { status: 'warn', note: 'oEmbed endpoint was not served over HTTPS.', recommendation: 'Serve the oEmbed endpoint over HTTPS.' };
266 }
267 return { status: 'good', note: 'oEmbed endpoint served over HTTPS.' };
268}
269
270function evalContentType({ endpointResult, endpointTested }) {
271 if (!endpointTested || endpointResult.error || endpointResult.status !== 200) return { status: 'info', note: 'No response to evaluate.' };
272 const ct = endpointResult.contentType.toLowerCase();
273 if (ct.includes('application/json')) {
274 return { status: 'good', note: `Content-Type is '${endpointResult.contentType}'.` };
275 }
276 if (ct.includes('text/xml') || ct.includes('application/xml')) {
277 return { status: 'warn', note: `Content-Type is '${endpointResult.contentType}' (XML); this auditor validates JSON responses most thoroughly.`, recommendation: 'Prefer serving the JSON oEmbed format (application/json) alongside or instead of XML.' };
278 }
279 return { status: 'missing', note: `Content-Type is '${endpointResult.contentType || 'none'}'; expected application/json or text/xml.`, recommendation: 'Serve the oEmbed response with Content-Type: application/json (or text/xml for the XML format).' };
280}
281
282function evalJsonValidity({ doc, parseError, endpointTested, isXml }) {
283 if (!endpointTested || isXml) return { status: 'info', note: 'No JSON body to evaluate.' };
284 if (doc === null) {
285 return { status: 'missing', note: `oEmbed response is not valid JSON${parseError ? `: ${parseError}` : ''}.`, recommendation: 'Serve a valid JSON object from the oEmbed endpoint.' };
286 }
287 return { status: 'good', note: 'oEmbed response is valid JSON.' };
288}
289
290function evalRequiredFields({ validation, endpointTested, isXml }) {
291 if (!endpointTested || isXml) return { status: 'info', note: 'No JSON body to evaluate.' };
292 if (!validation || validation.type === null) return { status: 'info', note: 'No JSON body to evaluate.' };
293 if (!validation.valid) {
294 return { status: 'missing', note: `oEmbed response does not conform to the spec: ${validation.issues.join('; ')}`, recommendation: "Include all required members for the declared 'type' per oEmbed spec section 2.3.4 (photo: url/width/height; video/rich: html/width/height), and ensure optional members have correct types." };
295 }
296 return { status: 'good', note: `oEmbed response declares type '${validation.type}' and includes all required fields for that type.` };
297}
298
299const CHECKS = [
300 { name: 'discovery', title: 'Discovery link present', weight: 30, fn: evalDiscovery },
301 { name: 'json-discovery', title: 'JSON discovery link', weight: 10, fn: evalJsonDiscoveryPresent },
302 { name: 'endpoint', title: 'Endpoint response', weight: 20, fn: evalEndpointFetch },
303 { name: 'https', title: 'HTTPS transport', weight: 5, fn: evalEndpointHttps },
304 { name: 'content-type', title: 'Content-Type', weight: 10, fn: evalContentType },
305 { name: 'json-valid', title: 'JSON validity', weight: 10, fn: evalJsonValidity },
306 { name: 'required-fields', title: 'Required fields for declared type', weight: 15, fn: evalRequiredFields },
307];
308
309export function buildCheckReports(context) {
310 const reports = [];
311 let earned = 0;
312 let possible = 0;
313 for (const check of CHECKS) {
314 const evaluation = check.fn(context);
315 if (evaluation.status !== 'info') possible += check.weight;
316 if (evaluation.status === 'good') earned += check.weight;
317 else if (evaluation.status === 'warn') earned += Math.round(check.weight * 0.5);
318 reports.push({
319 name: check.title,
320 check: check.name,
321 status: evaluation.status,
322 note: evaluation.note,
323 weight: check.weight,
324 recommendation: evaluation.recommendation || null,
325 });
326 }
327 return { reports, earned, possible };
328}
329
330export function scoreAudit(earned, possible) {
331 if (possible === 0) return 0;
332 return Math.min(100, Math.max(0, Math.round((earned / possible) * 100)));
333}
334
335export function gradeFromScore(score) {
336 if (score >= 95) return 'A+';
337 if (score >= 85) return 'A';
338 if (score >= 75) return 'B';
339 if (score >= 65) return 'C';
340 if (score >= 50) return 'D';
341 if (score >= 30) return 'E';
342 return 'F';
343}
344
345export function buildRecommendations(reports) {
346 const recs = [];
347 for (const r of reports) {
348 if (r.recommendation && !recs.includes(r.recommendation)) recs.push(r.recommendation);
349 }
350 if (recs.length === 0) {
351 recs.push('oEmbed discovery and response look spec-conformant. Re-run this audit after template/provider changes to catch regressions.');
352 }
353 return recs;
354}
355
356
357
358
359
360export async function auditOembed(input) {
361 const pageUrlRaw = input.startUrl;
362 const endpointOverride = typeof input.endpointOverride === 'string' && input.endpointOverride.trim() ? input.endpointOverride.trim() : null;
363 const timeoutSeconds = clampInteger(input.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, 3, 30);
364 const maxBytes = clampInteger(input.maxResponseBytes, DEFAULT_MAX_BYTES, 10000, MAX_BYTES_CAP);
365 const checkedAt = new Date().toISOString();
366
367 let discoveryLinks = [];
368 let discoveryFetchError = null;
369 let pageFinalUrl = null;
370
371 if (!endpointOverride) {
372 const pageUrl = await normalizeAndValidateUrl(pageUrlRaw);
373 const pageResult = await boundedFetch(pageUrl, timeoutSeconds, maxBytes, 'text/html,application/xhtml+xml,*/*;q=0.1');
374 pageFinalUrl = pageResult.finalUrl;
375 if (pageResult.error) {
376 discoveryFetchError = pageResult.error;
377 } else {
378 discoveryLinks = extractDiscoveryLinks(pageResult.body, pageResult.finalUrl);
379 }
380 } else {
381 await normalizeAndValidateUrl(pageUrlRaw);
382 }
383
384 const jsonLink = discoveryLinks.find((l) => l.type === 'application/json+oembed');
385 const anyLink = jsonLink || discoveryLinks[0] || null;
386 const endpointUrlString = endpointOverride || (anyLink ? anyLink.resolvedUrl : null);
387 const endpointTested = !!endpointUrlString;
388
389 let endpointResult = { ok: false, status: null, finalUrl: null, contentType: '', https: false, body: '', error: null };
390 let doc = null;
391 let parseError = null;
392 let isXml = false;
393
394 if (endpointTested) {
395 const endpointUrl = await normalizeAndValidateUrl(endpointUrlString);
396 endpointResult = await boundedFetch(endpointUrl, timeoutSeconds, maxBytes, 'application/json, text/xml;q=0.5, */*;q=0.1');
397 const ct = (endpointResult.contentType || '').toLowerCase();
398 isXml = ct.includes('xml') && !ct.includes('json');
399 if (!isXml && endpointResult.body) {
400 try {
401 doc = JSON.parse(endpointResult.body);
402 } catch (err) {
403 parseError = err.message;
404 }
405 }
406 }
407
408 const validation = doc !== null ? validateOembedResponse(doc) : { valid: false, type: null, version: null, issues: [] };
409
410 const context = { discoveryLinks, discoveryFetchError, endpointResult, endpointTested, doc, parseError, isXml, validation };
411 const { reports, earned, possible } = buildCheckReports(context);
412 const score = scoreAudit(earned, possible);
413 const grade = gradeFromScore(score);
414 const recommendations = buildRecommendations(reports);
415
416 const issues = reports
417 .filter((r) => r.status === 'warn' || r.status === 'missing')
418 .map((r) => `${r.name}: ${r.note}`);
419
420 return {
421 inputUrl: pageUrlRaw,
422 pageFinalUrl,
423 discoveryLinks,
424 endpointTested,
425 endpointUrl: endpointUrlString,
426 endpointStatus: endpointResult.status,
427 endpointHttps: endpointResult.https,
428 endpointContentType: endpointResult.contentType || null,
429 isXmlResponse: isXml,
430 jsonValid: doc !== null,
431 parseError,
432 oembedType: validation.type,
433 oembedVersion: validation.version,
434 fieldValidationIssues: validation.issues,
435 checks: reports,
436 issues,
437 score,
438 grade,
439 checkedAt,
440 recommendations,
441 error: discoveryFetchError,
442 };
443}
444
445
446
447
448
449const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
450
451if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {
452 await Actor.init();
453 try {
454 const input = await Actor.getInput();
455 const result = await auditOembed(input || {});
456 await Actor.pushData(result);
457 await Actor.setValue('OUTPUT', result);
458 Actor.log.info('oEmbed audit complete', { inputUrl: result.inputUrl, score: result.score, grade: result.grade });
459 } finally {
460 await Actor.exit();
461 }
462}
