Onpage Seo Bulk
Pricing
Pay per usage
Pricing
Pay per usage
Rating
0.0
(0)
Developer
A Man
Maintained by CommunityActor stats
0
Bookmarked
2
Total users
1
Monthly active users
3 months ago
Last modified
Categories
Security
Limited permissionsShare
FROM apify/actor-node:20COPY package*.json ./RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optionalCOPY . ./CMD npm start{ "urls": [ "https://example.com/", "http://apify.com/", "https://www.wikipedia.org/", "https://httpbin.org/status/404", "https://httpbin.org/json", "not-a-url", "http://127.0.0.1/", "ftp://example.com/" ], "maxConcurrency": 4, "maxRetries": 1, "delayMs": 0}{ "name": "onpage-seo-bulk", "version": "0.2.5", "type": "module", "description": "Bulk-audit URLs: on-page SEO fields, 0–100 score, and prioritised issues.", "scripts": { "start": "node src/main.mjs", "test": "node --test tests/*.mjs", "battle": "node scripts/battle-live.mjs" }, "dependencies": { "apify": "^3.2.6", "cheerio": "^1.0.0", "undici": "^6.21.0" }, "engines": { "node": ">=18.0.0" }}{ "actorSpecification": 1, "name": "onpage-seo-bulk", "title": "On-Page SEO Audit — Score & Issues (Bulk URLs)", "description": "Audit any URL list in seconds: 0–100 on-page score, prioritised issues, title/meta/H1/canonical/OG/Twitter/JSON-LD/hreflang, link counts, image alts, and redirect chains. HTTP-only, failed URLs not billed when monetised. Pair with Sitemap URL Harvester for full-site audits. Built for SEO agencies, in-house SEO, and n8n/Make pipelines.", "version": "0.3", "buildTag": "latest", "categories": ["SEO", "DEVELOPER_TOOLS"], "dockerfile": "./Dockerfile", "input": "./input_schema.json", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "requestedUrl": { "type": "string", "title": "Requested URL", "description": "Original URL from the input list.", "example": "http://example.com/" }, "finalUrl": { "type": "string", "title": "Final URL", "description": "URL after following redirects (successful rows only).", "example": "https://example.com/" }, "ok": { "type": "boolean", "title": "Success", "description": "True when the page was fetched and parsed; false for invalid URLs, HTTP errors, or timeouts.", "example": true }, "contentType": { "type": ["string", "null"], "title": "Content-Type", "description": "Response Content-Type header (successful rows).", "example": "text/html; charset=utf-8" }, "score": { "type": "integer", "title": "On-page score", "description": "0–100 score from deterministic SEO checks (successful rows only).", "example": 72 }, "issues": { "type": "array", "title": "SEO issues", "description": "Prioritised problems with code, severity, and message.", "items": { "type": "object", "properties": { "code": { "type": "string" }, "severity": { "type": "string" }, "message": { "type": "string" } } } }, "error": { "type": ["string", "null"], "title": "Error", "description": "Failure reason when ok is false.", "example": "HTTP 404" }, "title": { "type": ["string", "null"], "title": "Title tag", "example": "Example Domain" }, "metaDescription": { "type": ["string", "null"], "title": "Meta description", "example": "Example meta description" }, "canonical": { "type": ["string", "null"], "title": "Canonical URL", "example": "https://example.com/" }, "robotsMeta": { "type": ["string", "null"], "title": "Robots meta", "example": "index,follow" }, "h1": { "type": ["string", "null"], "title": "First H1", "example": "Example Domain" }, "h1Count": { "type": "integer", "title": "H1 count" }, "headings": { "type": "object", "title": "Heading counts", "description": "Counts for h1–h6." }, "ogTitle": { "type": ["string", "null"], "title": "Open Graph title" }, "ogDescription": { "type": ["string", "null"], "title": "Open Graph description" }, "ogImage": { "type": ["string", "null"], "title": "Open Graph image" }, "twitterCard": { "type": ["string", "null"], "title": "Twitter card type" }, "twitterTitle": { "type": ["string", "null"], "title": "Twitter title" }, "twitterDescription": { "type": ["string", "null"], "title": "Twitter description" }, "twitterImage": { "type": ["string", "null"], "title": "Twitter image" }, "wordCount": { "type": "integer", "title": "Word count", "description": "Approximate visible body word count.", "example": 12 }, "viewport": { "type": ["string", "null"], "title": "Viewport meta" }, "htmlLang": { "type": ["string", "null"], "title": "HTML lang attribute" }, "favicon": { "type": ["string", "null"], "title": "Favicon href" }, "appleTouchIcon": { "type": ["string", "null"], "title": "Apple touch icon href" }, "jsonLdCount": { "type": "integer", "title": "JSON-LD block count", "example": 1 }, "jsonLdSchemas": { "type": "array", "title": "JSON-LD @type values", "items": { "type": "string" }, "example": ["Organization"] }, "hreflang": { "type": ["object", "null"], "title": "Hreflang map", "description": "Language code to alternate URL.", "additionalProperties": { "type": "string" } }, "links": { "type": "object", "title": "Link counts", "properties": { "internal": { "type": "integer" }, "external": { "type": "integer" }, "total": { "type": "integer" } } }, "images": { "type": "object", "title": "Image alt coverage", "properties": { "total": { "type": "integer" }, "missingAlt": { "type": "integer" } } }, "redirectChain": { "type": "array", "title": "Redirect chain", "description": "Ordered hops with HTTP status codes.", "items": { "type": "object", "properties": { "url": { "type": "string" }, "status": { "type": "integer" } } }, "example": [ { "url": "http://example.com/", "status": 301 }, { "url": "https://example.com/", "status": 200 } ] } }, "required": ["requestedUrl", "ok"] }, "views": { "overview": { "title": "Overview", "description": "Score and core SEO fields for successful audits.", "transformation": { "fields": [ "requestedUrl", "finalUrl", "ok", "score", "contentType", "title", "metaDescription", "canonical", "h1", "h1Count", "wordCount", "images" ] }, "display": { "component": "table", "properties": { "requestedUrl": { "label": "Requested", "format": "link" }, "finalUrl": { "label": "Final", "format": "link" }, "ok": { "label": "OK", "format": "boolean" }, "score": { "label": "Score" } } } }, "issues": { "title": "Issues", "description": "Successful rows with score and issue list.", "transformation": { "fields": ["requestedUrl", "score", "issues", "ok"] }, "display": { "component": "table", "properties": { "requestedUrl": { "label": "URL", "format": "link" }, "score": { "label": "Score" } } } }, "errors": { "title": "Errors", "description": "Failed URLs with error messages.", "transformation": { "fields": ["requestedUrl", "ok", "error", "redirectChain"] }, "display": { "component": "table", "properties": { "requestedUrl": { "label": "URL", "format": "link" }, "ok": { "label": "OK", "format": "boolean" } } } } }}{ "title": "On-page SEO bulk extractor", "type": "object", "schemaVersion": 1, "properties": { "urls": { "title": "Page URLs", "type": "array", "description": "HTTP(S) page URLs to fetch and parse. HTML/static pages only — not suited for client-rendered SPAs (React/Vue/Angular).", "editor": "stringList", "prefill": ["https://example.com/"] }, "maxConcurrency": { "title": "Max concurrent fetches", "type": "integer", "description": "How many pages to fetch in parallel.", "default": 5, "minimum": 1, "maximum": 50 }, "requestTimeoutSecs": { "title": "Per-request timeout (seconds)", "type": "integer", "description": "Seconds before a single request is aborted.", "default": 30, "minimum": 5, "maximum": 120 }, "maxRedirects": { "title": "Max redirects to follow", "type": "integer", "description": "Maximum number of redirect hops before stopping.", "default": 10, "minimum": 0, "maximum": 20 }, "maxRetries": { "title": "Retries on transient errors", "type": "integer", "description": "Extra attempts after network failures, HTTP 429, or 5xx (0 = no retry).", "default": 1, "minimum": 0, "maximum": 5 }, "delayMs": { "title": "Pause between requests (ms)", "type": "integer", "description": "Optional delay per URL fetch to reduce pressure on target servers.", "default": 0, "minimum": 0, "maximum": 10000 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional Apify Proxy (or custom proxies). Helps when targets rate-limit datacentre IPs.", "editor": "proxy", "default": { "useApifyProxy": false } } }, "required": ["urls"]}{ "actorOutputSchemaVersion": 1, "title": "On-page SEO audit results", "description": "One dataset row per input URL — successful audits include SEO fields; failures include ok=false and an error message.", "properties": { "seoResults": { "type": "string", "title": "SEO audit rows", "description": "Default dataset: title, meta, canonical, H1, OG tags, redirect chain, and related on-page fields.", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1import { readFileSync, existsSync } from 'node:fs';2import { dirname, join } from 'node:path';3import { fileURLToPath } from 'node:url';4import { Actor } from 'apify';5import { ProxyAgent } from 'undici';6import { chargePageAudited } from './lib/charge.mjs';7import { extractOnPageSeo } from './lib/extract.mjs';8import { fetchPageWithRedirects } from './lib/fetchPage.mjs';9import { runPool } from './lib/runPool.mjs';10import { scoreOnPageSeo } from './lib/score.mjs';11import { validateHttpUrl } from './lib/validateUrl.mjs';12
13const __dirname = dirname(fileURLToPath(import.meta.url));14const INPUT_PATH = join(__dirname, '..', 'INPUT.json');15
16await Actor.init();17
18function mergeFromInputJson(input, key) {19 if (input[key]?.length) return input;20 if (!existsSync(INPUT_PATH)) return input;21 const fromFile = JSON.parse(readFileSync(INPUT_PATH, 'utf8'));22 return { ...input, ...fromFile };23}24
25let input = (await Actor.getInput()) ?? {};26input = mergeFromInputJson(input, 'urls');27const urls = input.urls ?? [];28if (!Array.isArray(urls) || urls.length === 0) {29 throw new Error('Input must include non-empty urls array.');30}31
32const maxConcurrency = Math.min(Math.max(Number(input.maxConcurrency ?? 5), 1), 50);33const requestTimeoutSecs = Math.min(34 Math.max(Number(input.requestTimeoutSecs ?? 30), 5),35 120,36);37const maxRedirects = Math.min(Math.max(Number(input.maxRedirects ?? 10), 0), 20);38const delayMs = Math.max(Number(input.delayMs ?? 0), 0);39const maxRetries = Math.min(Math.max(Number(input.maxRetries ?? 1), 0), 5);40
41// Only create proxy when explicitly enabled — avoids local "no password" noise.42const proxyInput = input.proxyConfiguration;43const wantProxy =44 Boolean(proxyInput?.useApifyProxy) ||45 (Array.isArray(proxyInput?.proxyUrls) && proxyInput.proxyUrls.length > 0);46const proxyConfiguration = wantProxy47 ? await Actor.createProxyConfiguration(proxyInput)48 : undefined;49
50const dataset = await Actor.openDataset();51const cleaned = urls.map((u) => String(u).trim()).filter(Boolean);52
53await runPool(cleaned, maxConcurrency, async (url) => {54 const validated = validateHttpUrl(url);55 if (!validated.valid) {56 const row = {57 requestedUrl: url,58 ok: false,59 error: validated.error,60 };61 await dataset.pushData(row);62 return row;63 }64
65 const timeoutMs = requestTimeoutSecs * 1000;66 try {67 /** @type {import('undici').Dispatcher | undefined} */68 let dispatcher;69 if (proxyConfiguration) {70 const proxyUrl = await proxyConfiguration.newUrl();71 if (proxyUrl) dispatcher = new ProxyAgent(proxyUrl);72 }73
74 const fetched = await fetchPageWithRedirects(validated.url, {75 maxRedirects,76 timeoutMs,77 maxRetries,78 dispatcher,79 });80
81 if (dispatcher && typeof dispatcher.close === 'function') {82 await dispatcher.close().catch(() => {});83 }84
85 if (!fetched.ok) {86 const row = {87 requestedUrl: url,88 ok: false,89 error: fetched.error,90 redirectChain: fetched.chain,91 };92 await dataset.pushData(row);93 return row;94 }95
96 const seo = extractOnPageSeo(fetched.html, fetched.finalUrl, url);97 const contentType = fetched.contentType ?? null;98 const { score, issues } = scoreOnPageSeo({ ...seo, contentType });99 await chargePageAudited();100 if (delayMs > 0) {101 await new Promise((r) => setTimeout(r, delayMs));102 }103
104 const row = {105 ...seo,106 contentType,107 score,108 issues,109 ok: true,110 redirectChain: fetched.chain,111 };112 // Push as we go — why: large batches should not hold all rows in RAM.113 await dataset.pushData(row);114 return row;115 } catch (e) {116 const row = {117 requestedUrl: url,118 ok: false,119 error: e instanceof Error ? e.message : String(e),120 };121 await dataset.pushData(row);122 return row;123 }124});125
126await Actor.exit();1/**2 * Live battlefield: mixed real URLs → assert ok/fail shapes + score presence.3 * Run: node scripts/battle-live.mjs4 */5import { extractOnPageSeo } from '../src/lib/extract.mjs';6import { fetchPageWithRedirects } from '../src/lib/fetchPage.mjs';7import { runPool } from '../src/lib/runPool.mjs';8import { scoreOnPageSeo } from '../src/lib/score.mjs';9import { validateHttpUrl } from '../src/lib/validateUrl.mjs';10
11/** @type {{ url: string, expectOk: boolean, note: string }[]} */12const CASES = [13 { url: 'https://example.com/', expectOk: true, note: 'static HTML baseline' },14 { url: 'http://apify.com/', expectOk: true, note: 'http→https redirect chain' },15 { url: 'https://www.wikipedia.org/', expectOk: true, note: 'heavy real HTML + hreflang-ish' },16 { url: 'https://httpbin.org/status/404', expectOk: false, note: 'hard 404' },17 { url: 'https://httpbin.org/status/500', expectOk: false, note: '5xx after retries' },18 { url: 'https://httpbin.org/json', expectOk: true, note: 'JSON content-type (ok but flagged)' },19 { url: 'not-a-url', expectOk: false, note: 'invalid URL' },20 { url: 'http://127.0.0.1/', expectOk: false, note: 'SSRF loopback' },21 { url: 'ftp://example.com/', expectOk: false, note: 'bad scheme' },22 { url: 'https://httpbin.org/delay/8', expectOk: false, note: 'timeout (3s budget)' },23];24
25async function auditOne(c) {26 const validated = validateHttpUrl(c.url);27 if (!validated.valid) {28 return {29 ...c,30 ok: false,31 error: validated.error,32 pass: c.expectOk === false,33 };34 }35
36 const timeoutMs = c.url.includes('/delay/') ? 3000 : 25000;37 const fetched = await fetchPageWithRedirects(validated.url, {38 maxRedirects: 10,39 timeoutMs,40 maxRetries: c.url.includes('/status/500') ? 1 : 0,41 });42
43 if (!fetched.ok) {44 return {45 ...c,46 ok: false,47 error: fetched.error,48 chainLen: fetched.chain?.length ?? 0,49 pass: c.expectOk === false,50 };51 }52
53 const seo = extractOnPageSeo(fetched.html, fetched.finalUrl, c.url);54 const contentType = fetched.contentType ?? null;55 const { score, issues } = scoreOnPageSeo({ ...seo, contentType });56 const isJson = (contentType ?? '').toLowerCase().includes('json');57 const jsonFlagOk = !isJson || issues.some((i) => i.code === 'content_type_not_html');58
59 return {60 ...c,61 ok: true,62 score,63 issueCount: issues.length,64 title: seo.title,65 wordCount: seo.wordCount,66 contentType,67 chainLen: fetched.chain?.length ?? 0,68 pass: c.expectOk === true && typeof score === 'number' && jsonFlagOk,69 };70}71
72const started = Date.now();73const rows = await runPool(CASES, 4, auditOne);74const failed = rows.filter((r) => !r.pass);75
76for (const r of rows) {77 const mark = r.pass ? 'PASS' : 'FAIL';78 const detail = r.ok79 ? `score=${r.score} words=${r.wordCount} issues=${r.issueCount} ct=${r.contentType ?? '-'}`80 : `error=${r.error}`;81 console.log(`[${mark}] ${r.note} | ${r.url} | ${detail}`);82}83
84console.log(85 `\n${rows.length - failed.length}/${rows.length} passed in ${Date.now() - started}ms`,86);87if (failed.length) {88 console.error('Battle failures:', failed.map((f) => f.note));89 process.exit(1);90}1/**2 * Battle-test edge cases buyers hit: bad HTML, SSRF hops, 404s, SPA shells, JSON APIs.3 */4import { test } from 'node:test';5import assert from 'node:assert/strict';6import { extractOnPageSeo } from '../src/lib/extract.mjs';7import { fetchPageWithRedirects } from '../src/lib/fetchPage.mjs';8import { scoreOnPageSeo } from '../src/lib/score.mjs';9import { validateHttpUrl } from '../src/lib/validateUrl.mjs';10
11test('JSON-LD top-level array flattens @type values', () => {12 const html = `<!doctype html><html><head>13 <script type="application/ld+json">14 [{"@type":"Organization","name":"A"},{"@type":"WebSite","url":"https://ex.com/"}]15 </script>16 </head><body><h1>Hi</h1></body></html>`;17 const r = extractOnPageSeo(html, 'https://ex.com/', 'https://ex.com/');18 assert.equal(r.jsonLdCount, 1);19 assert.deepEqual(r.jsonLdSchemas.sort(), ['Organization', 'WebSite']);20});21
22test('JSON-LD @graph nested inside array is walked', () => {23 const html = `<!doctype html><html><head>24 <script type="application/ld+json">25 {"@graph":[{"@type":"BreadcrumbList"},{"@type":["Product","Thing"]}]}26 </script>27 </head><body></body></html>`;28 const r = extractOnPageSeo(html, 'https://ex.com/', 'https://ex.com/');29 assert.ok(r.jsonLdSchemas.includes('BreadcrumbList'));30 assert.ok(r.jsonLdSchemas.includes('Product'));31 assert.ok(r.jsonLdSchemas.includes('Thing'));32});33
34test('malformed JSON-LD does not crash extract', () => {35 const html = `<!doctype html><html><head>36 <script type="application/ld+json">{not-json</script>37 <title>Ok Title With Enough Length For Scoring Test</title>38 </head><body><h1>Ok</h1><p>${'word '.repeat(120)}</p></body></html>`;39 const r = extractOnPageSeo(html, 'https://ex.com/', 'https://ex.com/');40 assert.equal(r.jsonLdCount, 0);41 assert.equal(r.title?.startsWith('Ok Title'), true);42});43
44test('empty HTML yields nulls and empty_body score issue', () => {45 const r = extractOnPageSeo('', 'https://ex.com/', 'https://ex.com/');46 assert.equal(r.title, null);47 assert.equal(r.wordCount, 0);48 const scored = scoreOnPageSeo(r);49 assert.ok(scored.issues.some((i) => i.code === 'empty_body'));50 assert.ok(scored.score < 50);51});52
53test('decorative alt="" is not missing alt', () => {54 const html = `<!doctype html><html><body>55 <img src="/a.png" alt="" /><img src="/b.png" />56 </body></html>`;57 const r = extractOnPageSeo(html, 'https://ex.com/', 'https://ex.com/');58 assert.equal(r.images.total, 2);59 assert.equal(r.images.missingAlt, 1);60});61
62test('score flags noindex and non-HTML content-type', () => {63 const scored = scoreOnPageSeo({64 title: 'A solid page title that is long enough here',65 metaDescription:66 'A meta description that is comfortably between seventy and one hundred sixty characters for SEO.',67 h1: 'Main',68 h1Count: 1,69 canonical: 'https://ex.com/',70 viewport: 'width=device-width',71 robotsMeta: 'noindex,nofollow',72 ogTitle: 'OG',73 ogImage: 'https://ex.com/og.png',74 twitterCard: 'summary',75 wordCount: 250,76 htmlLang: 'en',77 jsonLdCount: 1,78 images: { total: 0, missingAlt: 0 },79 contentType: 'application/json; charset=utf-8',80 });81 const codes = scored.issues.map((i) => i.code);82 assert.ok(codes.includes('noindex'));83 assert.ok(codes.includes('content_type_not_html'));84});85
86test('mailto/tel/hash links ignored in counts', () => {87 const html = `<!doctype html><html><body>88 <a href="#top">top</a>89 <a href="mailto:a@b.com">mail</a>90 <a href="tel:+44123">phone</a>91 <a href="/ok">ok</a>92 </body></html>`;93 const r = extractOnPageSeo(html, 'https://ex.com/', 'https://ex.com/');94 assert.equal(r.links.total, 1);95 assert.equal(r.links.internal, 1);96});97
98test('SSRF: IPv6 loopback and .internal blocked', () => {99 assert.equal(validateHttpUrl('http://[::1]/').valid, false);100 assert.equal(validateHttpUrl('http://foo.internal/').valid, false);101 assert.equal(validateHttpUrl('http://169.254.169.254/').valid, false);102});103
104test('HTTP 404 returns ok:false (network)', async () => {105 const fetched = await fetchPageWithRedirects('https://httpbin.org/status/404', {106 maxRedirects: 5,107 timeoutMs: 20000,108 maxRetries: 0,109 });110 assert.equal(fetched.ok, false);111 assert.match(fetched.error ?? '', /HTTP 404/);112});113
114test('maxRedirects=0 stops on first redirect (network)', async () => {115 const fetched = await fetchPageWithRedirects('http://apify.com/', {116 maxRedirects: 0,117 timeoutMs: 20000,118 maxRetries: 0,119 });120 assert.equal(fetched.ok, false);121 assert.match(fetched.error ?? '', /Too many redirects|Redirect/);122});123
124test('mixed batch simulation: invalid + private do not throw', () => {125 const urls = ['not-a-url', 'http://127.0.0.1/', 'https://example.com/'];126 const rows = urls.map((url) => {127 const v = validateHttpUrl(url);128 if (!v.valid) return { requestedUrl: url, ok: false, error: v.error };129 return { requestedUrl: url, ok: true };130 });131 assert.equal(rows.filter((r) => !r.ok).length, 2);132 assert.equal(rows.filter((r) => r.ok).length, 1);133});1import { test } from 'node:test';2import assert from 'node:assert/strict';3import { extractOnPageSeo } from '../src/lib/extract.mjs';4import { fetchPageWithRedirects } from '../src/lib/fetchPage.mjs';5import { runPool } from '../src/lib/runPool.mjs';6import { scoreOnPageSeo } from '../src/lib/score.mjs';7import { validateHttpUrl } from '../src/lib/validateUrl.mjs';8
9test('extractOnPageSeo reads core tags', () => {10 const html = `<!doctype html><html><head>11 <title> Hello Page </title>12 <meta name="description" content="Desc here" />13 <link rel="canonical" href="https://ex.com/canonical" />14 <meta name="robots" content="index,follow" />15 <meta property="og:title" content="OG Title" />16 <meta name="viewport" content="width=device-width, initial-scale=1" />17 <link rel="icon" href="/favicon.ico" />18 <link rel="apple-touch-icon" href="/apple.png" />19 </head><body>20 <h1> Main Heading </h1>21 <p>one two three</p>22 <a href="https://ex.com/other">internal</a>23 <a href="https://external.com/">external</a>24 </body></html>`;25 const r = extractOnPageSeo(html, 'https://ex.com/', 'https://ex.com/in');26 assert.equal(r.requestedUrl, 'https://ex.com/in');27 assert.equal(r.finalUrl, 'https://ex.com/');28 assert.equal(r.title, 'Hello Page');29 assert.equal(r.metaDescription, 'Desc here');30 assert.equal(r.canonical, 'https://ex.com/canonical');31 assert.equal(r.robotsMeta, 'index,follow');32 assert.equal(r.h1, 'Main Heading');33 assert.equal(r.h1Count, 1);34 assert.equal(r.ogTitle, 'OG Title');35 assert.equal(r.viewport, 'width=device-width, initial-scale=1');36 assert.equal(r.favicon, '/favicon.ico');37 assert.equal(r.appleTouchIcon, '/apple.png');38 assert.equal(r.links.internal, 1);39 assert.equal(r.links.external, 1);40 assert.equal(r.links.total, 2);41 assert.ok(r.wordCount >= 4);42});43
44test('relative links resolve against finalUrl as internal', () => {45 const html = `<!doctype html><html><body>46 <a href="/about">about</a>47 <a href="https://other.com/x">ext</a>48 </body></html>`;49 const r = extractOnPageSeo(html, 'https://ex.com/page', 'https://ex.com/page');50 assert.equal(r.links.internal, 1);51 assert.equal(r.links.external, 1);52});53
54test('extractOnPageSeo extracts Twitter, headings, images, JSON-LD', () => {55 const html = `<!doctype html><html lang="en-GB"><head>56 <title>JSON-LD Test Page With Enough Characters</title>57 <meta name="description" content="A meta description that is long enough to avoid short warnings in scoring." />58 <link rel="alternate" hreflang="en-US" href="https://ex.com/us/" />59 <link rel="alternate" hreflang="fr" href="https://ex.com/fr/" />60 <meta name="twitter:card" content="summary_large_image" />61 <meta name="twitter:title" content="Tw Title" />62 <script type="application/ld+json">{"@type":"Organization","name":"Acme"}</script>63 <script type="application/ld+json">{"@type":"WebSite","url":"https://ex.com/"}</script>64 </head><body>65 <h1>One</h1><h1>Two</h1><h2>Sub</h2>66 <img src="/a.png" /><img src="/b.png" alt="ok" />67 <p>hello world</p>68 </body></html>`;69 const r = extractOnPageSeo(html, 'https://ex.com/', 'https://ex.com/');70 assert.equal(r.htmlLang, 'en-GB');71 assert.equal(r.jsonLdCount, 2);72 assert.deepEqual(r.jsonLdSchemas, ['Organization', 'WebSite']);73 assert.deepEqual(r.hreflang, {74 'en-US': 'https://ex.com/us/',75 fr: 'https://ex.com/fr/',76 });77 assert.equal(r.twitterCard, 'summary_large_image');78 assert.equal(r.twitterTitle, 'Tw Title');79 assert.equal(r.h1Count, 2);80 assert.equal(r.headings.h2, 1);81 assert.equal(r.images.total, 2);82 assert.equal(r.images.missingAlt, 1);83});84
85test('scoreOnPageSeo flags missing title and multiple H1', () => {86 const scored = scoreOnPageSeo({87 title: null,88 metaDescription: null,89 h1: 'A',90 h1Count: 2,91 canonical: null,92 viewport: null,93 robotsMeta: null,94 ogTitle: null,95 ogImage: null,96 twitterCard: null,97 images: { total: 2, missingAlt: 2 },98 wordCount: 10,99 htmlLang: null,100 jsonLdCount: 0,101 });102 assert.ok(scored.score < 50);103 const codes = scored.issues.map((i) => i.code);104 assert.ok(codes.includes('missing_title'));105 assert.ok(codes.includes('multiple_h1'));106 assert.ok(codes.includes('images_missing_alt'));107});108
109test('scoreOnPageSeo deducts for missing title and meta', () => {110 const { score, issues } = scoreOnPageSeo({111 title: null,112 metaDescription: null,113 h1: 'Ok',114 h1Count: 1,115 canonical: 'https://ex.com/',116 viewport: 'width=device-width',117 wordCount: 200,118 htmlLang: 'en',119 jsonLdCount: 1,120 });121 assert.ok(score < 100);122 assert.ok(issues.some((i) => i.code === 'missing_title'));123 assert.ok(issues.some((i) => i.code === 'missing_meta_description'));124});125
126test('scoreOnPageSeo rewards a healthy page near 100', () => {127 const { score, issues } = scoreOnPageSeo({128 title: 'A solid page title that is long enough here',129 metaDescription:130 'A meta description that is comfortably between seventy and one hundred sixty characters for SEO.',131 h1: 'Main',132 h1Count: 1,133 canonical: 'https://ex.com/',134 viewport: 'width=device-width',135 ogTitle: 'OG',136 ogImage: 'https://ex.com/og.png',137 twitterCard: 'summary',138 wordCount: 250,139 htmlLang: 'en',140 jsonLdCount: 1,141 images: { total: 1, missingAlt: 0 },142 });143 assert.equal(score, 100);144 assert.equal(issues.length, 0);145});146
147test('runPool respects concurrency and completes all', async () => {148 const order = [];149 const out = await runPool([1, 2, 3, 4], 2, async (n) => {150 order.push(`start-${n}`);151 await new Promise((r) => setTimeout(r, 5));152 order.push(`end-${n}`);153 return n * 2;154 });155 assert.deepEqual(out, [2, 4, 6, 8]);156 assert.ok(order.length === 8);157});158
159test('live fetch example.com (network)', async () => {160 const res = await fetch('https://example.com/', {161 headers: { 'user-agent': 'ApifyActor-test' },162 });163 assert.ok(res.ok);164 const html = await res.text();165 const r = extractOnPageSeo(html, res.url, 'https://example.com/');166 assert.equal(r.title, 'Example Domain');167 assert.ok(r.h1 === null || typeof r.h1 === 'string');168 assert.equal(r.links.total, 1); // example.com has one link169 const scored = scoreOnPageSeo(r);170 assert.ok(typeof scored.score === 'number');171 assert.ok(Array.isArray(scored.issues));172});173
174test('validateHttpUrl rejects non-URLs and non-http schemes', () => {175 assert.equal(validateHttpUrl('not-a-url').valid, false);176 assert.match(validateHttpUrl('not-a-url').error, /Invalid URL/);177 assert.equal(validateHttpUrl('ftp://example.com/').valid, false);178 assert.equal(validateHttpUrl(' https://example.com/ ').valid, true);179});180
181test('validateHttpUrl blocks private / loopback / metadata hosts (SSRF)', () => {182 assert.equal(validateHttpUrl('http://127.0.0.1/').valid, false);183 assert.equal(validateHttpUrl('http://localhost/x').valid, false);184 assert.equal(validateHttpUrl('http://192.168.1.1/').valid, false);185 assert.equal(validateHttpUrl('http://10.0.0.5/').valid, false);186 assert.equal(validateHttpUrl('http://169.254.169.254/latest/meta').valid, false);187 assert.equal(validateHttpUrl('https://example.com/ok').valid, true);188});189
190test('fetchPageWithRedirects records http→https redirect chain (network)', async () => {191 const fetched = await fetchPageWithRedirects('http://apify.com/', {192 maxRedirects: 10,193 timeoutMs: 30000,194 maxRetries: 1,195 });196 assert.ok(fetched.ok, fetched.error ?? 'fetch failed');197 assert.ok(fetched.finalUrl.startsWith('https://'));198 assert.ok(fetched.chain.length >= 2, 'expected at least one redirect hop');199 assert.equal(fetched.chain[0].url, 'http://apify.com/');200 assert.ok(fetched.chain[0].status >= 300 && fetched.chain[0].status < 400);201 assert.equal(fetched.chain.at(-1).status, 200);202});1import { Actor } from 'apify';2
3/** PPE event name — must match monetisation wizard in Apify Console. */4export const PAGE_AUDITED_EVENT = 'page-audited';5
6/**7 * Charge one successfully audited page (PPE).8 * No-op on local runs and non-PPE Actors — Apify SDK handles that safely.9 * @returns {Promise<number>} chargedCount from the platform (0 when not billed)10 */11export async function chargePageAudited() {12 try {13 const result = await Actor.charge({ eventName: PAGE_AUDITED_EVENT });14 return result?.chargedCount ?? 0;15 } catch {16 // Local `node src/main.mjs` without a full Apify run context — do not fail the batch.17 return 0;18 }19}1import * as cheerio from 'cheerio';2
3/**4 * Meta tag content by name= or property= (first match).5 * @param {import('cheerio').CheerioAPI} $6 * @param {'name'|'property'} attr7 * @param {string} key8 */9function metaContent($, attr, key) {10 const el = $(`meta[${attr}="${key}"]`).first();11 return el.attr('content')?.trim() ?? null;12}13
14/**15 * Extract all JSON-LD script blocks as parsed objects.16 * @param {import('cheerio').CheerioAPI} $17 */18function extractJsonLd($) {19 const out = [];20 $('script[type="application/ld+json"]').each((_, el) => {21 const raw = $(el).html()?.trim();22 if (!raw) return;23 try {24 out.push(JSON.parse(raw));25 } catch {26 // ignore malformed JSON-LD27 }28 });29 return out;30}31
32/**33 * Flatten @type from JSON-LD (string or array).34 * @param {unknown} node35 * @returns {string[]}36 */37function jsonLdTypes(node) {38 if (!node || typeof node !== 'object') return [];39 const t = /** @type {{ '@type'?: string|string[] }} */ (node)['@type'];40 if (Array.isArray(t)) return t.map(String);41 if (typeof t === 'string') return [t];42 return [];43}44
45/**46 * Extract hreflang alternate links.47 * @param {import('cheerio').CheerioAPI} $48 */49function extractHreflang($) {50 const map = {};51 $('link[rel="alternate"][hreflang]').each((_, el) => {52 const lang = $(el).attr('hreflang')?.trim();53 const href = $(el).attr('href')?.trim();54 if (lang && href) map[lang] = href;55 });56 return Object.keys(map).length ? map : null;57}58
59/**60 * Count internal vs external links (deduped by absolute href).61 * Why resolve against finalUrl: relative /about must count as internal, not "external".62 * @param {import('cheerio').CheerioAPI} $63 * @param {string} finalUrl64 */65function countLinks($, finalUrl) {66 let baseHost = '';67 try {68 baseHost = new URL(finalUrl).hostname;69 } catch {70 baseHost = '';71 }72
73 const seen = new Set();74 let internal = 0;75 let external = 0;76
77 $('a[href]').each((_, el) => {78 const href = $(el).attr('href')?.trim();79 if (!href || href.startsWith('#') || href.startsWith('javascript:') || href.startsWith('mailto:') || href.startsWith('tel:')) {80 return;81 }82 let abs;83 try {84 abs = new URL(href, finalUrl).toString();85 } catch {86 return;87 }88 if (seen.has(abs)) return;89 seen.add(abs);90 try {91 const host = new URL(abs).hostname;92 if (host === baseHost) internal++;93 else external++;94 } catch {95 internal++;96 }97 });98
99 return { internal, external, total: seen.size };100}101
102/**103 * Heading hierarchy counts (h1–h6 text samples capped).104 * @param {import('cheerio').CheerioAPI} $105 */106function extractHeadings($) {107 /** @type {Record<string, number>} */108 const counts = { h1: 0, h2: 0, h3: 0, h4: 0, h5: 0, h6: 0 };109 /** @type {string[]} */110 const h1Texts = [];111
112 for (const level of Object.keys(counts)) {113 $(level).each((_, el) => {114 counts[level]++;115 if (level === 'h1') {116 const t = $(el).text().replace(/\s+/g, ' ').trim();117 if (t) h1Texts.push(t);118 }119 });120 }121
122 return { counts, h1Texts };123}124
125/**126 * Image alt coverage.127 * @param {import('cheerio').CheerioAPI} $128 */129function countImages($) {130 let total = 0;131 let missingAlt = 0;132 $('img').each((_, el) => {133 total++;134 const alt = $(el).attr('alt');135 // Missing or empty alt (decorative "" is OK)136 if (alt === undefined || alt === null) missingAlt++;137 });138 return { total, missingAlt };139}140
141/**142 * Parse HTML and return stable SEO fields for one final URL.143 * @param {string} html144 * @param {string} finalUrl — URL after redirects (for logging)145 * @param {string} requestedUrl — original input URL146 */147export function extractOnPageSeo(html, finalUrl, requestedUrl) {148 const $ = cheerio.load(html);149
150 const title = $('title').first().text().trim() || null;151 const metaDescription =152 metaContent($, 'name', 'description') ??153 metaContent($, 'property', 'og:description');154 const canonical =155 $('link[rel="canonical"]').attr('href')?.trim() ?? null;156 const robotsMeta =157 metaContent($, 'name', 'robots') ??158 metaContent($, 'name', 'googlebot');159
160 const { counts: headingCounts, h1Texts } = extractHeadings($);161 const h1 = h1Texts[0] ?? null;162 const h1Count = headingCounts.h1;163
164 const ogTitle = metaContent($, 'property', 'og:title');165 const ogDescription = metaContent($, 'property', 'og:description');166 const ogImage = metaContent($, 'property', 'og:image');167
168 // Twitter cards (name= or property= both appear in the wild)169 const twitterCard =170 metaContent($, 'name', 'twitter:card') ??171 metaContent($, 'property', 'twitter:card');172 const twitterTitle =173 metaContent($, 'name', 'twitter:title') ??174 metaContent($, 'property', 'twitter:title');175 const twitterDescription =176 metaContent($, 'name', 'twitter:description') ??177 metaContent($, 'property', 'twitter:description');178 const twitterImage =179 metaContent($, 'name', 'twitter:image') ??180 metaContent($, 'property', 'twitter:image');181
182 const jsonLd = extractJsonLd($);183 const hreflang = extractHreflang($);184 const links = countLinks($, finalUrl);185 const images = countImages($);186 const viewport = metaContent($, 'name', 'viewport');187 const htmlLang = $('html').attr('lang')?.trim() ?? null;188 const favicon = $('link[rel="icon"], link[rel="shortcut icon"]').attr('href')?.trim() ?? null;189 const appleTouchIcon = $('link[rel="apple-touch-icon"]').attr('href')?.trim() ?? null;190
191 // Word count: visible text approximation (strip scripts/styles)192 $('script, style, noscript').remove();193 const bodyText = $('body').text().replace(/\s+/g, ' ').trim();194 const wordCount = bodyText ? bodyText.split(/\s+/).length : 0;195
196 // Flatten top-level arrays + @graph — why: Google often ships `[{@type},{@type}]`.197 const schemas = [];198 /** @param {unknown} node */199 const walkJsonLd = (node) => {200 if (Array.isArray(node)) {201 for (const child of node) walkJsonLd(child);202 return;203 }204 schemas.push(...jsonLdTypes(node));205 if (node && typeof node === 'object' && Array.isArray(/** @type {{ '@graph'?: unknown[] }} */ (node)['@graph'])) {206 for (const child of /** @type {{ '@graph': unknown[] }} */ (node)['@graph']) {207 walkJsonLd(child);208 }209 }210 };211 for (const node of jsonLd) walkJsonLd(node);212
213 return {214 requestedUrl,215 finalUrl,216 title,217 metaDescription,218 canonical,219 robotsMeta,220 h1,221 h1Count,222 headings: headingCounts,223 ogTitle,224 ogDescription,225 ogImage,226 twitterCard,227 twitterTitle,228 twitterDescription,229 twitterImage,230 wordCount,231 viewport,232 htmlLang,233 favicon,234 appleTouchIcon,235 jsonLdCount: jsonLd.length,236 jsonLdSchemas: [...new Set(schemas)],237 hreflang,238 links,239 images,240 };241}1import { validateHttpUrl } from './validateUrl.mjs';2
3// Polite, browser-like UA — still identifies this Apify Store actor.4const UA =5 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 (compatible; ApifyActor/onpage-seo-bulk; +https://apify.com/store)';6
7/**8 * One hop-aware GET with optional undici ProxyAgent (Apify proxy URL).9 * @param {string} url10 * @param {{ timeoutMs: number, dispatcher?: import('undici').Dispatcher }} opts11 */12async function fetchOnce(url, opts) {13 const { timeoutMs, dispatcher } = opts;14 const ctrl = new AbortController();15 const id = setTimeout(() => ctrl.abort(), timeoutMs);16 try {17 /** @type {RequestInit & { dispatcher?: import('undici').Dispatcher }} */18 const init = {19 method: 'GET',20 redirect: 'manual',21 headers: {22 'user-agent': UA,23 accept: 'text/html,application/xhtml+xml,*/*;q=0.8',24 },25 signal: ctrl.signal,26 };27 if (dispatcher) init.dispatcher = dispatcher;28 return await fetch(url, init);29 } finally {30 clearTimeout(id);31 }32}33
34/**35 * Fetch URL following redirects manually so we can record final URL and hop count.36 * Retries only transient failures (network / 429 / 5xx on the terminal hop).37 * @param {string} startUrl38 * @param {{39 * maxRedirects: number,40 * timeoutMs: number,41 * maxRetries?: number,42 * dispatcher?: import('undici').Dispatcher,43 * }} opts44 */45export async function fetchPageWithRedirects(startUrl, opts) {46 const { maxRedirects, timeoutMs, dispatcher } = opts;47 const maxRetries = Math.min(Math.max(Number(opts.maxRetries ?? 1), 0), 5);48
49 let lastError = 'Unknown fetch error';50 /** @type {{ url: string, status: number }[]} */51 let lastChain = [];52
53 for (let attempt = 0; attempt <= maxRetries; attempt++) {54 if (attempt > 0) {55 // Why backoff: 429/5xx often clear after a short pause; hammering burns proxy $.56 await new Promise((r) => setTimeout(r, 400 * attempt));57 }58
59 const chain = [];60 let current = startUrl;61
62 try {63 for (let hop = 0; hop <= maxRedirects; hop++) {64 const res = await fetchOnce(current, { timeoutMs, dispatcher });65 chain.push({ url: current, status: res.status });66 lastChain = chain;67
68 if (res.status >= 300 && res.status < 400) {69 const loc = res.headers.get('location');70 if (!loc) {71 return {72 ok: false,73 error: `Redirect ${res.status} without Location`,74 chain,75 };76 }77 // Re-check every hop — why: open redirect to 169.254.169.254 etc.78 const next = new URL(loc, current).toString();79 const hopOk = validateHttpUrl(next);80 if (!hopOk.valid) {81 return { ok: false, error: `Blocked redirect: ${hopOk.error}`, chain };82 }83 current = hopOk.url;84 continue;85 }86
87 // Retryable terminal statuses88 if (res.status === 429 || res.status >= 500) {89 lastError = `HTTP ${res.status}`;90 break;91 }92
93 if (!res.ok) {94 return {95 ok: false,96 error: `HTTP ${res.status}`,97 chain,98 };99 }100
101 const contentType = res.headers.get('content-type') ?? null;102 const html = await res.text();103 return {104 ok: true,105 html,106 finalUrl: current,107 chain,108 contentType,109 };110 }111
112 if (chain.length > maxRedirects) {113 return { ok: false, error: 'Too many redirects', chain };114 }115 } catch (e) {116 lastError = e instanceof Error ? e.message : String(e);117 lastChain = chain;118 }119 }120
121 return { ok: false, error: lastError, chain: lastChain };122}1/**2 * Simple concurrency pool: run async tasks with max parallel workers.3 * @template T,R4 * @param {T[]} items5 * @param {number} concurrency6 * @param {(item: T, index: number) => Promise<R>} worker7 * @returns {Promise<R[]>}8 */9export async function runPool(items, concurrency, worker) {10 const results = /** @type {R[]} */ ([]);11 let idx = 0;12
13 async function workerLoop() {14 while (idx < items.length) {15 const my = idx++;16 const item = items[my];17 results[my] = await worker(item, my);18 }19 }20
21 const n = Math.min(concurrency, items.length || 1);22 await Promise.all(Array.from({ length: n }, () => workerLoop()));23 return results;24}1/**2 * Deterministic on-page SEO score + issue list from extract fields.3 * Why: agencies buy "what's wrong" more than raw tags — same extract, better sell.4 */5
6/** @typedef {{ code: string, severity: 'error'|'warn'|'info', message: string }} SeoIssue */7
8/**9 * @param {Record<string, unknown>} seo — output of extractOnPageSeo (+ optional redirectChain)10 * @returns {{ score: number, issues: SeoIssue[] }}11 */12export function scoreOnPageSeo(seo) {13 /** @type {SeoIssue[]} */14 const issues = [];15 let score = 100;16
17 /** @param {number} pts @param {SeoIssue} issue */18 const deduct = (pts, issue) => {19 issues.push(issue);20 score -= pts;21 };22
23 const title = typeof seo.title === 'string' ? seo.title : null;24 if (!title) {25 deduct(20, { code: 'missing_title', severity: 'error', message: 'Missing <title> tag' });26 } else if (title.length < 30) {27 deduct(5, { code: 'title_short', severity: 'warn', message: `Title is short (${title.length} chars; aim 30–60)` });28 } else if (title.length > 60) {29 deduct(5, { code: 'title_long', severity: 'warn', message: `Title is long (${title.length} chars; aim 30–60)` });30 }31
32 const desc = typeof seo.metaDescription === 'string' ? seo.metaDescription : null;33 if (!desc) {34 deduct(15, { code: 'missing_meta_description', severity: 'error', message: 'Missing meta description' });35 } else if (desc.length < 70) {36 deduct(4, { code: 'meta_short', severity: 'warn', message: `Meta description short (${desc.length} chars; aim 70–160)` });37 } else if (desc.length > 160) {38 deduct(4, { code: 'meta_long', severity: 'warn', message: `Meta description long (${desc.length} chars; aim 70–160)` });39 }40
41 if (!seo.h1) {42 deduct(12, { code: 'missing_h1', severity: 'error', message: 'Missing H1' });43 }44 const h1Count = typeof seo.h1Count === 'number' ? seo.h1Count : seo.h1 ? 1 : 0;45 if (h1Count > 1) {46 deduct(6, { code: 'multiple_h1', severity: 'warn', message: `Multiple H1 tags (${h1Count})` });47 }48
49 if (!seo.canonical) {50 deduct(8, { code: 'missing_canonical', severity: 'warn', message: 'Missing canonical link' });51 }52
53 if (!seo.viewport) {54 deduct(8, { code: 'missing_viewport', severity: 'warn', message: 'Missing viewport meta (mobile)' });55 }56
57 const robots = typeof seo.robotsMeta === 'string' ? seo.robotsMeta.toLowerCase() : '';58 if (robots.includes('noindex')) {59 deduct(15, { code: 'noindex', severity: 'error', message: 'Page has noindex in robots meta' });60 }61
62 if (!seo.ogTitle && !seo.ogImage) {63 deduct(4, { code: 'missing_og', severity: 'info', message: 'Missing Open Graph title and image' });64 } else if (!seo.ogImage) {65 deduct(2, { code: 'missing_og_image', severity: 'info', message: 'Missing og:image' });66 }67
68 if (!seo.twitterCard) {69 deduct(2, { code: 'missing_twitter_card', severity: 'info', message: 'Missing twitter:card' });70 }71
72 const images = /** @type {{ total?: number, missingAlt?: number } | undefined} */ (seo.images);73 if (images && images.total && images.missingAlt) {74 const pts = Math.min(10, images.missingAlt * 2);75 deduct(pts, {76 code: 'images_missing_alt',77 severity: 'warn',78 message: `${images.missingAlt}/${images.total} images missing alt text`,79 });80 }81
82 const wordCount = typeof seo.wordCount === 'number' ? seo.wordCount : 0;83 if (wordCount === 0) {84 // Why: SPA shells / non-HTML often parse as empty body — tell buyers why score tanked.85 deduct(10, {86 code: 'empty_body',87 severity: 'warn',88 message: 'Empty or near-empty body text (SPA / non-HTML / blocked content?)',89 });90 } else if (wordCount < 100) {91 deduct(5, { code: 'thin_content', severity: 'warn', message: `Thin content (~${wordCount} words)` });92 }93
94 const ct = typeof seo.contentType === 'string' ? seo.contentType.toLowerCase() : '';95 if (ct && !ct.includes('html') && !ct.includes('xml')) {96 deduct(8, {97 code: 'content_type_not_html',98 severity: 'warn',99 message: `Content-Type is not HTML (${seo.contentType})`,100 });101 }102
103 if (!seo.htmlLang) {104 deduct(3, { code: 'missing_html_lang', severity: 'info', message: 'Missing html lang attribute' });105 }106
107 const jsonLdCount = typeof seo.jsonLdCount === 'number' ? seo.jsonLdCount : 0;108 if (jsonLdCount === 0) {109 deduct(3, { code: 'missing_json_ld', severity: 'info', message: 'No JSON-LD structured data' });110 }111
112 return {113 score: Math.max(0, Math.min(100, score)),114 issues,115 };116}1/**2 * Soft-validate a user-supplied page URL before fetch.3 * Invalid / private targets become error rows — they must not abort the batch.4 * Why block private hosts: stop SSRF into localhost / RFC1918 / cloud metadata.5 * @param {string} raw6 * @returns {{ valid: true, url: string } | { valid: false, error: string }}7 */8export function validateHttpUrl(raw) {9 const trimmed = String(raw).trim();10 if (!trimmed) {11 return { valid: false, error: 'Empty URL' };12 }13 try {14 const u = new URL(trimmed);15 if (u.protocol !== 'http:' && u.protocol !== 'https:') {16 return { valid: false, error: 'Invalid URL (must be http or https)' };17 }18 if (isBlockedHostname(u.hostname)) {19 return { valid: false, error: 'Blocked host (private / loopback / metadata)' };20 }21 return { valid: true, url: u.toString() };22 } catch {23 return { valid: false, error: 'Invalid URL (must be http or https)' };24 }25}26
27/**28 * Hostname / literal-IP checks for common SSRF targets.29 * Does not resolve DNS — still blocks obvious private targets in the URL string.30 * @param {string} hostname31 */32export function isBlockedHostname(hostname) {33 const host = String(hostname).trim().toLowerCase().replace(/^\[|\]$/g, '');34 if (!host) return true;35 if (host === 'localhost' || host === 'localhost.') return true;36 if (host.endsWith('.localhost') || host.endsWith('.local') || host.endsWith('.internal')) {37 return true;38 }39 if (host === '::1' || host === '0.0.0.0') return true;40
41 // IPv4 literal42 if (/^\d{1,3}(\.\d{1,3}){3}$/.test(host)) {43 return isPrivateIpv4(host);44 }45 // IPv6 loopback / link-local / ULA (coarse)46 if (host.includes(':')) {47 if (host === '::1' || host.startsWith('fe80:') || host.startsWith('fc') || host.startsWith('fd')) {48 return true;49 }50 }51 return false;52}53
54/** @param {string} ip dotted IPv4 */55function isPrivateIpv4(ip) {56 const parts = ip.split('.').map(Number);57 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n) || n < 0 || n > 255)) {58 return true;59 }60 const [a, b] = parts;61 if (a === 10) return true; // 10.0.0.0/862 if (a === 127) return true; // loopback63 if (a === 0) return true;64 if (a === 169 && b === 254) return true; // link-local / cloud metadata65 if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/1266 if (a === 192 && b === 168) return true; // 192.168.0.0/1667 if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT 100.64.0.0/1068 return false;69}