1import { Actor } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4import { fileURLToPath } from 'node:url';
5
6const USER_AGENT = 'JsonLdSchemaAuditor/0.1 (+https://apify.com)';
7const DEFAULT_TIMEOUT_SECONDS = 10;
8const DEFAULT_MAX_HTML_BYTES = 1024 * 1024;
9const MAX_HTML_BYTES = 2 * 1024 * 1024;
10const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
11
12function isPrivateIPv4(ip) {
13 const parts = ip.split('.').map(Number);
14 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;
15 const [a, b] = parts;
16 return a === 10 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || a === 127 || a === 0 || (a === 169 && b === 254);
17}
18
19function isPrivateIPv6(ip) {
20 const normalized = ip.toLowerCase();
21 return normalized === '::1' || normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe80:');
22}
23
24export async function normalizeAndValidateUrl(rawUrl) {
25 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');
26 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) throw new Error('Only HTTP and HTTPS URLs are supported');
27 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;
28 const url = new URL(withScheme);
29 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');
30 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');
31
32 const literalType = net.isIP(url.hostname);
33 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');
34 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');
35
36 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });
37 for (const record of records) {
38 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');
39 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');
40 }
41 return url;
42}
43
44function clampInteger(value, fallback, min, max) {
45 const parsed = Number(value);
46 if (!Number.isFinite(parsed)) return fallback;
47 return Math.min(Math.max(Math.trunc(parsed), min), max);
48}
49
50function decodeHtmlEntities(value) {
51 return (value || '').replace(/&/gi, '&').replace(/"/gi, '"').replace(/'|'/gi, "'").replace(/</gi, '<').replace(/>/gi, '>').trim();
52}
53
54function scriptBodies(html) {
55 return [...html.matchAll(/<script\b[^>]*type\s*=\s*(?:"application\/ld\+json"|'application\/ld\+json'|application\/ld\+json)[^>]*>([\s\S]*?)<\/script>/gi)]
56 .map((match) => decodeHtmlEntities(match[1]));
57}
58
59function flattenJsonLd(value) {
60 if (Array.isArray(value)) return value.flatMap(flattenJsonLd);
61 if (value && typeof value === 'object') return [value, ...flattenJsonLd(value['@graph'] || [])];
62 return [];
63}
64
65function typeValues(item) {
66 const raw = item && item['@type'];
67 if (Array.isArray(raw)) return raw.filter((value) => typeof value === 'string');
68 return typeof raw === 'string' ? [raw] : [];
69}
70
71export function parseJsonLdScripts(html) {
72 const bodies = scriptBodies(html);
73 const parseErrors = [];
74 const items = [];
75 bodies.forEach((body, index) => {
76 try {
77 items.push(...flattenJsonLd(JSON.parse(body)));
78 } catch (error) {
79 parseErrors.push({ scriptIndex: index, message: error.message });
80 }
81 });
82 const schemaTypes = [...new Set(items.flatMap(typeValues))].sort();
83 const itemsMissingType = items.filter((item) => typeValues(item).length === 0).length;
84 return { scriptCount: bodies.length, validItemCount: items.length, schemaTypes, itemsMissingType, parseErrors };
85}
86
87function scoreSchema(fetchResult, parsed) {
88 if (fetchResult.error) return { score: 0, grade: 'F', issues: [fetchResult.error], recommendations: ['Verify the URL is public, reachable, and returns HTML.'] };
89 const issues = [];
90 const recommendations = [];
91 let score = 100;
92
93 if (!fetchResult.status || fetchResult.status < 200 || fetchResult.status >= 300) {
94 score -= 40;
95 issues.push(`HTTP status is ${fetchResult.status || 'unknown'}`);
96 recommendations.push('Audit a canonical page that returns a 2xx HTTP status.');
97 }
98 if (parsed.scriptCount === 0) {
99 score -= 70;
100 issues.push('no JSON-LD scripts found');
101 recommendations.push('Add application/ld+json structured data for Organization, WebSite, Article, Product, or another relevant schema type.');
102 }
103 if (parsed.parseErrors.length > 0) {
104 score -= Math.min(40, parsed.parseErrors.length * 20);
105 issues.push('one or more JSON-LD scripts are invalid JSON');
106 recommendations.push('Fix JSON syntax errors in application/ld+json scripts.');
107 }
108 if (parsed.validItemCount > 0 && parsed.schemaTypes.length === 0) {
109 score -= 25;
110 issues.push('JSON-LD items missing @type');
111 recommendations.push('Add @type to each top-level JSON-LD entity.');
112 }
113 if (parsed.itemsMissingType > 0) {
114 score -= Math.min(15, parsed.itemsMissingType * 5);
115 issues.push(`${parsed.itemsMissingType} JSON-LD item(s) missing @type`);
116 recommendations.push('Add explicit @type values to graph nodes where practical.');
117 }
118 if (!parsed.schemaTypes.some((type) => ['Organization', 'WebSite', 'LocalBusiness', 'Product', 'Article', 'BreadcrumbList', 'FAQPage'].includes(type))) {
119 score -= 10;
120 issues.push('no common SEO schema types detected');
121 recommendations.push('Consider adding common schema types that match the page purpose.');
122 }
123
124 const bounded = Math.max(0, score);
125 const grade = bounded >= 90 ? 'A' : bounded >= 75 ? 'B' : bounded >= 60 ? 'C' : bounded >= 45 ? 'D' : 'F';
126 return { score: bounded, grade, issues, recommendations };
127}
128
129async function fetchUrl(initialUrl, timeoutSeconds, maxHtmlBytes, redirectsRemaining = 3) {
130 await normalizeAndValidateUrl(initialUrl.href);
131 const controller = new AbortController();
132 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);
133 try {
134 const response = await fetch(initialUrl, { redirect: 'manual', signal: controller.signal, headers: { 'user-agent': USER_AGENT, accept: 'text/html,application/xhtml+xml,*/*;q=0.1' } });
135 if (REDIRECT_STATUSES.has(response.status)) {
136 if (redirectsRemaining <= 0) throw new Error('Too many redirects');
137 const location = response.headers.get('location');
138 if (!location) throw new Error('Redirect without Location header');
139 return fetchUrl(new URL(location, initialUrl.href), timeoutSeconds, maxHtmlBytes, redirectsRemaining - 1);
140 }
141 const reader = response.body?.getReader();
142 if (!reader) throw new Error('Response body is not readable');
143 const chunks = [];
144 let received = 0;
145 while (true) {
146 const { done, value } = await reader.read();
147 if (done) break;
148 received += value.byteLength;
149 if (received > maxHtmlBytes) throw new Error(`HTML response exceeds ${maxHtmlBytes} byte limit`);
150 chunks.push(value);
151 }
152 return { ok: response.ok, status: response.status, finalUrl: response.url || initialUrl.href, html: Buffer.concat(chunks).toString('utf8'), error: null };
153 } catch (error) {
154 return { ok: false, status: null, finalUrl: initialUrl.href, html: '', error: error.message };
155 } finally {
156 clearTimeout(timeout);
157 }
158}
159
160export async function auditJsonLdSchema(input) {
161 const startUrl = await normalizeAndValidateUrl(input.startUrl);
162 const timeoutSeconds = clampInteger(input.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, 3, 30);
163 const maxHtmlBytes = clampInteger(input.maxHtmlBytes, DEFAULT_MAX_HTML_BYTES, 100000, MAX_HTML_BYTES);
164 const fetchResult = await fetchUrl(startUrl, timeoutSeconds, maxHtmlBytes);
165 const parsed = fetchResult.error ? { scriptCount: 0, validItemCount: 0, schemaTypes: [], itemsMissingType: 0, parseErrors: [] } : parseJsonLdScripts(fetchResult.html);
166 const scored = scoreSchema(fetchResult, parsed);
167 return { inputUrl: input.startUrl, normalizedInputUrl: startUrl.href, finalUrl: fetchResult.finalUrl, ok: !fetchResult.error && fetchResult.ok, status: fetchResult.status, checkedAt: new Date().toISOString(), score: scored.score, grade: scored.grade, ...parsed, issues: scored.issues, recommendations: scored.recommendations, error: fetchResult.error };
168}
169
170const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
171
172if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {
173 await Actor.init();
174 try {
175 const result = await auditJsonLdSchema(await Actor.getInput() || {});
176 await Actor.pushData(result);
177 await Actor.setValue('OUTPUT', result);
178 Actor.log.info('JSON-LD schema audit complete', { finalUrl: result.finalUrl, score: result.score, grade: result.grade });
179 } finally {
180 await Actor.exit();
181 }
182}