1import { Actor } from 'apify';
2import dns from 'node:dns/promises';
3import net from 'node:net';
4import { fileURLToPath } from 'node:url';
5
6const USER_AGENT = 'SeoMetaTagsAuditor/0.1 (+https://apify.com)';
7const DEFAULT_TIMEOUT_SECONDS = 10;
8const DEFAULT_MAX_HTML_BYTES = 1024 * 1024;
9const MAX_HTML_BYTES = 2 * 1024 * 1024;
10
11
12
13
14
15
16
17
18
19function isPrivateIPv4(ip) {
20 const parts = ip.split('.').map(Number);
21 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;
22 const [a, b] = parts;
23 return a === 10
24 || (a === 172 && b >= 16 && b <= 31)
25 || (a === 192 && b === 168)
26 || a === 127
27 || a === 0
28 || (a === 169 && b === 254);
29}
30
31function isPrivateIPv6(ip) {
32 const normalized = ip.toLowerCase();
33 return normalized === '::1'
34 || normalized.startsWith('fc')
35 || normalized.startsWith('fd')
36 || normalized.startsWith('fe80:');
37}
38
39export async function normalizeAndValidateUrl(rawUrl) {
40 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');
41 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {
42 throw new Error('Only HTTP and HTTPS URLs are supported');
43 }
44
45 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;
46 const url = new URL(withScheme);
47 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');
48 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');
49
50 const literalType = net.isIP(url.hostname);
51 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');
52 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');
53
54 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });
55 for (const record of records) {
56 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');
57 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');
58 }
59 return url;
60}
61
62function decodeHtmlEntities(value) {
63 if (!value) return '';
64 return value
65 .replace(/&/gi, '&')
66 .replace(/"/gi, '"')
67 .replace(/'|'/gi, "'")
68 .replace(/</gi, '<')
69 .replace(/>/gi, '>')
70 .replace(/&#(\d+);/g, (_, code) => String.fromCodePoint(Number(code)))
71 .replace(/&#x([0-9a-f]+);/gi, (_, code) => String.fromCodePoint(parseInt(code, 16)))
72 .trim();
73}
74
75function parseAttributes(tag) {
76 const attrs = {};
77 const attrPattern = /([a-zA-Z_:][-a-zA-Z0-9_:.]*)\s*=\s*("([^"]*)"|'([^']*)'|([^\s"'>/]+))/g;
78 for (const match of tag.matchAll(attrPattern)) {
79 attrs[match[1].toLowerCase()] = decodeHtmlEntities(match[3] ?? match[4] ?? match[5] ?? '');
80 }
81 return attrs;
82}
83
84function allTags(html, tagName) {
85 const escaped = tagName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
86 return [...html.matchAll(new RegExp(`<${escaped}\\b[^>]*>`, 'gi'))].map((m) => ({ raw: m[0], attrs: parseAttributes(m[0]) }));
87}
88
89function firstTitle(html) {
90 const match = /<title\b[^>]*>([\s\S]*?)<\/title>/i.exec(html);
91 return decodeHtmlEntities(match?.[1]?.replace(/\s+/g, ' ') || '');
92}
93
94function firstMetaByKey(metaTags, key, value) {
95 const lowerValue = value.toLowerCase();
96 const tag = metaTags.find((item) => (item.attrs[key] || '').toLowerCase() === lowerValue);
97 return tag?.attrs.content || '';
98}
99
100function linkValues(linkTags, relName) {
101 const wanted = relName.toLowerCase();
102 return linkTags
103 .filter((item) => (item.attrs.rel || '').toLowerCase().split(/\s+/).includes(wanted))
104 .map((item) => item.attrs.href)
105 .filter(Boolean);
106}
107
108function propertyMap(metaTags, key) {
109 const result = {};
110 for (const item of metaTags) {
111 const name = item.attrs[key];
112 const content = item.attrs.content;
113 if (name && content) result[name] = content;
114 }
115 return result;
116}
117
118export function parseSeoMetadata(html, finalUrl) {
119 const metaTags = allTags(html, 'meta');
120 const linkTags = allTags(html, 'link');
121 const scriptTags = [...html.matchAll(/<script\b[^>]*>[\s\S]*?<\/script>/gi)].map((m) => ({ raw: m[0], attrs: parseAttributes(m[0]) }));
122 const og = propertyMap(metaTags, 'property');
123 const named = propertyMap(metaTags, 'name');
124 const twitter = Object.fromEntries(Object.entries(named).filter(([key]) => key.toLowerCase().startsWith('twitter:')));
125 const canonicals = linkValues(linkTags, 'canonical');
126 const hreflangLinks = linkTags
127 .filter((item) => (item.attrs.rel || '').toLowerCase().split(/\s+/).includes('alternate') && item.attrs.hreflang && item.attrs.href)
128 .map((item) => ({ hreflang: item.attrs.hreflang, href: item.attrs.href }));
129 const jsonLdCount = scriptTags.filter((item) => (item.attrs.type || '').toLowerCase() === 'application/ld+json').length;
130
131 return {
132 url: finalUrl,
133 title: firstTitle(html),
134 metaDescription: firstMetaByKey(metaTags, 'name', 'description'),
135 robots: firstMetaByKey(metaTags, 'name', 'robots'),
136 viewport: firstMetaByKey(metaTags, 'name', 'viewport'),
137 canonical: canonicals[0] || '',
138 canonicalCount: canonicals.length,
139 openGraph: {
140 title: og['og:title'] || '',
141 description: og['og:description'] || '',
142 type: og['og:type'] || '',
143 image: og['og:image'] || '',
144 url: og['og:url'] || '',
145 },
146 twitterCard: {
147 card: twitter['twitter:card'] || '',
148 title: twitter['twitter:title'] || '',
149 description: twitter['twitter:description'] || '',
150 image: twitter['twitter:image'] || '',
151 },
152 hreflang: hreflangLinks,
153 jsonLdCount,
154 };
155}
156
157function addCheck(checks, name, status, weight, points, recommendation) {
158 checks.push({ name, status, weight, points, recommendation: status === 'pass' ? null : recommendation });
159}
160
161export function scoreSeoMetadata(metadata) {
162 const checks = [];
163 const titleLength = metadata.title.length;
164 if (titleLength >= 20 && titleLength <= 65) addCheck(checks, 'Title length', 'pass', 15, 15, null);
165 else if (titleLength > 0) addCheck(checks, 'Title length', 'warn', 15, 7, 'Keep the title between 20 and 65 characters for search snippets.');
166 else addCheck(checks, 'Title length', 'fail', 15, 0, 'Add a descriptive title tag.');
167
168 const descriptionLength = metadata.metaDescription.length;
169 if (descriptionLength >= 70 && descriptionLength <= 160) addCheck(checks, 'Meta description length', 'pass', 15, 15, null);
170 else if (descriptionLength > 0) addCheck(checks, 'Meta description length', 'warn', 15, 7, 'Keep the meta description between 70 and 160 characters.');
171 else addCheck(checks, 'Meta description length', 'fail', 15, 0, 'Add a meta description for search snippets and sharing previews.');
172
173 if (metadata.canonical && metadata.canonicalCount === 1) addCheck(checks, 'Canonical tag', 'pass', 15, 15, null);
174 else if (metadata.canonicalCount > 1) addCheck(checks, 'Canonical tag', 'warn', 15, 7, 'Use exactly one canonical link tag.');
175 else addCheck(checks, 'Canonical tag', 'fail', 15, 0, 'Add a canonical link tag to reduce duplicate URL ambiguity.');
176
177 const ogRequired = ['title', 'description', 'image', 'url'];
178 const ogPresent = ogRequired.filter((key) => metadata.openGraph[key]).length;
179 addCheck(checks, 'Open Graph completeness', ogPresent === 4 ? 'pass' : ogPresent >= 2 ? 'warn' : 'fail', 15, ogPresent === 4 ? 15 : ogPresent >= 2 ? 7 : 0, 'Set og:title, og:description, og:image, and og:url for social sharing.');
180
181 const twitterRequired = ['card', 'title', 'description', 'image'];
182 const twitterPresent = twitterRequired.filter((key) => metadata.twitterCard[key]).length;
183 addCheck(checks, 'Twitter Card completeness', twitterPresent >= 3 ? 'pass' : twitterPresent >= 1 ? 'warn' : 'fail', 10, twitterPresent >= 3 ? 10 : twitterPresent >= 1 ? 5 : 0, 'Set twitter:card, twitter:title, twitter:description, and twitter:image.');
184
185 addCheck(checks, 'Robots meta', metadata.robots && /noindex/i.test(metadata.robots) ? 'warn' : 'pass', 10, metadata.robots && /noindex/i.test(metadata.robots) ? 5 : 10, 'Remove noindex from pages that should appear in search results.');
186 addCheck(checks, 'Viewport meta', metadata.viewport ? 'pass' : 'fail', 5, metadata.viewport ? 5 : 0, 'Add a viewport meta tag for mobile-friendly rendering.');
187 addCheck(checks, 'JSON-LD structured data', metadata.jsonLdCount > 0 ? 'pass' : 'warn', 10, metadata.jsonLdCount > 0 ? 10 : 5, 'Add JSON-LD structured data where relevant for rich result eligibility.');
188 addCheck(checks, 'Hreflang alternates', metadata.hreflang.length > 0 ? 'pass' : 'warn', 5, metadata.hreflang.length > 0 ? 5 : 3, 'For multilingual sites, add hreflang alternate links. This warning can be ignored for single-language pages.');
189
190 const possible = checks.reduce((sum, check) => sum + check.weight, 0);
191 const earned = checks.reduce((sum, check) => sum + check.points, 0);
192 const score = Math.round((earned / possible) * 100);
193 const grade = score >= 90 ? 'A' : score >= 75 ? 'B' : score >= 60 ? 'C' : score >= 45 ? 'D' : 'F';
194 const recommendations = checks.filter((check) => check.recommendation).map((check) => check.recommendation);
195 return { score, grade, checks, recommendations };
196}
197
198async function fetchHtml(initialUrl, timeoutSeconds, maxHtmlBytes, redirectsRemaining = 3) {
199 await normalizeAndValidateUrl(initialUrl.href);
200 const controller = new AbortController();
201 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);
202 try {
203 const response = await fetch(initialUrl, {
204 redirect: 'manual',
205 signal: controller.signal,
206 headers: { 'user-agent': USER_AGENT, accept: 'text/html,application/xhtml+xml,*/*;q=0.1' },
207 });
208
209 if ([301, 302, 303, 307, 308].includes(response.status)) {
210 if (redirectsRemaining <= 0) throw new Error('Too many redirects');
211 const location = response.headers.get('location');
212 if (!location) throw new Error('Redirect without Location header');
213 const nextUrl = new URL(location, initialUrl.href);
214 await normalizeAndValidateUrl(nextUrl.href);
215 return fetchHtml(nextUrl, timeoutSeconds, maxHtmlBytes, redirectsRemaining - 1);
216 }
217
218 const contentType = response.headers.get('content-type') || '';
219 const reader = response.body?.getReader();
220 if (!reader) throw new Error('Response body is not readable');
221 const chunks = [];
222 let received = 0;
223 while (true) {
224 const { done, value } = await reader.read();
225 if (done) break;
226 received += value.byteLength;
227 if (received > maxHtmlBytes) throw new Error(`HTML response exceeds ${maxHtmlBytes} byte limit`);
228 chunks.push(value);
229 }
230 const body = Buffer.concat(chunks).toString('utf8');
231 return { ok: response.ok, status: response.status, finalUrl: response.url || initialUrl.href, contentType, html: body, error: null };
232 } catch (error) {
233 return { ok: false, status: null, finalUrl: initialUrl.href, contentType: '', html: '', error: error.message };
234 } finally {
235 clearTimeout(timeout);
236 }
237}
238
239export async function auditSeoMetadata(input) {
240 const startUrl = await normalizeAndValidateUrl(input.startUrl);
241 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);
242 const maxHtmlBytes = Math.min(Math.max(Number(input.maxHtmlBytes || DEFAULT_MAX_HTML_BYTES), 100000), MAX_HTML_BYTES);
243 const fetchResult = await fetchHtml(startUrl, timeoutSeconds, maxHtmlBytes);
244 const checkedAt = new Date().toISOString();
245
246 if (fetchResult.error) {
247 return {
248 inputUrl: input.startUrl,
249 finalUrl: fetchResult.finalUrl,
250 ok: false,
251 status: fetchResult.status,
252 checkedAt,
253 score: 0,
254 grade: 'F',
255 metadata: null,
256 checks: [],
257 recommendations: ['The request failed before metadata could be inspected. Verify the public URL is reachable and returns HTML.'],
258 error: fetchResult.error,
259 };
260 }
261
262 const metadata = parseSeoMetadata(fetchResult.html, fetchResult.finalUrl);
263 const scored = scoreSeoMetadata(metadata);
264 return {
265 inputUrl: input.startUrl,
266 finalUrl: fetchResult.finalUrl,
267 ok: true,
268 status: fetchResult.status,
269 contentType: fetchResult.contentType,
270 checkedAt,
271 score: scored.score,
272 grade: scored.grade,
273 metadata,
274 checks: scored.checks,
275 recommendations: scored.recommendations,
276 };
277}
278
279const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];
280
281if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {
282 await Actor.init();
283 try {
284 const input = await Actor.getInput();
285 const result = await auditSeoMetadata(input || {});
286 await Actor.pushData(result);
287 await Actor.setValue('OUTPUT', result);
288 Actor.log.info('SEO metadata audit complete', { finalUrl: result.finalUrl, score: result.score, grade: result.grade });
289 } finally {
290 await Actor.exit();
291 }
292}