Sitemap Freshness Auditor
Pricing
from $0.005 / actor start
Audit a public XML sitemap for lastmod coverage, stale URLs, child sitemap indexes, score, grade, and SEO maintenance recommendations.
Pricing
from $0.005 / actor start
Rating
0.0
(0)
Developer
Sanskar Jaiswal
Maintained by CommunityActor stats
0
Bookmarked
2
Total users
1
Monthly active users
13 days ago
Last modified
Categories
Share
FROM apify/actor-node:22
COPY package*.json ./RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional
COPY . ./
CMD npm start --silent{ "name": "sitemap-freshness-auditor", "version": "0.1.0", "description": "Apify Actor that audits a public XML sitemap for lastmod coverage, stale URLs, sitemap indexes, and SEO maintenance issues.", "type": "module", "private": true, "main": "src/main.js", "scripts": { "start": "node src/main.js", "test": "node --test test/*.test.js", "lint": "node --check src/main.js" }, "dependencies": { "apify": "^3.4.5" }, "devDependencies": {}}{ "actorSpecification": 1, "name": "sitemap-freshness-auditor", "title": "Sitemap Freshness Auditor", "description": "Audit a public XML sitemap for URL count, lastmod coverage, stale entries, sitemap indexes, and SEO maintenance issues.", "version": "0.1", "buildTag": "latest", "meta": { "templateId": "js-start" }, "input": "./input_schema.json", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }, "dockerfile": "./Dockerfile", "defaultRunOptions": { "build": "latest", "timeoutSecs": 60, "memoryMbytes": 512 }}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "inputUrl": { "type": "string", "description": "The URL provided as input" }, "sitemapUrl": { "type": "string", "description": "Sitemap URL selected for audit" }, "finalUrl": { "type": "string", "description": "Final fetched sitemap URL after redirects" }, "ok": { "type": "boolean", "description": "True when the sitemap is reachable and scores at least 75" }, "checkedAt": { "type": "string", "format": "date-time", "description": "ISO timestamp" }, "status": { "type": "integer", "description": "HTTP status code" }, "contentType": { "type": "string", "description": "Response content type" }, "bytes": { "type": "integer", "description": "Fetched response bytes" }, "staleAfterDays": { "type": "integer", "description": "Configured stale threshold in days" }, "score": { "type": "integer", "description": "Freshness score from 0 to 100" }, "grade": { "type": "string", "description": "Letter grade from A to F" }, "type": { "type": "string", "description": "Parsed sitemap type: urlset or sitemapindex" }, "urlCount": { "type": "integer", "description": "Number of URL entries parsed" }, "sitemapIndexCount": { "type": "integer", "description": "Number of child sitemap entries parsed" }, "lastmodCount": { "type": "integer", "description": "Number of URL entries with lastmod" }, "lastmodCoveragePercent": { "type": "integer", "description": "Percent of URL entries with lastmod" }, "staleUrlCount": { "type": "integer", "description": "Number of URLs older than staleAfterDays" }, "sampleUrls": { "type": "array", "description": "Sample parsed URL entries" }, "sitemapIndexes": { "type": "array", "description": "Parsed child sitemap entries" }, "issues": { "type": "array", "description": "Detected issues" }, "recommendations": { "type": "array", "description": "Recommended fixes" }, "error": { "type": ["string", "null"], "description": "Fetch or validation error, if any" } } }, "views": { "overview": { "title": "Overview", "transformation": { "fields": ["sitemapUrl", "ok", "checkedAt", "score", "grade", "type", "urlCount", "sitemapIndexCount", "lastmodCoveragePercent", "staleUrlCount", "issues", "recommendations"] }, "display": { "component": "table", "properties": { "sitemapUrl": { "label": "Sitemap", "format": "link" }, "ok": { "label": "OK", "format": "boolean" }, "checkedAt": { "label": "Checked", "format": "date" }, "score": { "label": "Score", "format": "number" }, "grade": { "label": "Grade", "format": "text" }, "type": { "label": "Type", "format": "text" }, "urlCount": { "label": "URLs", "format": "number" }, "sitemapIndexCount": { "label": "Child sitemaps", "format": "number" }, "lastmodCoveragePercent": { "label": "Lastmod coverage", "format": "number" }, "staleUrlCount": { "label": "Stale URLs", "format": "number" }, "issues": { "label": "Issues", "format": "array" }, "recommendations": { "label": "Recommendations", "format": "array" } } } } }}{ "title": "Sitemap Freshness Auditor", "type": "object", "schemaVersion": 1, "description": "Audit one public XML sitemap or website origin for sitemap freshness, lastmod coverage, stale URLs, and sitemap index entries.", "properties": { "startUrl": { "title": "Sitemap or website URL", "type": "string", "description": "Public sitemap XML URL or website URL. If a website URL is provided, the actor checks /sitemap.xml. HTTP and HTTPS only. Private IP ranges are blocked.", "editor": "textfield", "prefill": "https://example.com/sitemap.xml", "pattern": "^https?://.+" }, "staleAfterDays": { "title": "Stale after days", "type": "integer", "description": "A URL is marked stale when its lastmod date is older than this many days.", "minimum": 1, "maximum": 3650, "default": 365, "unit": "days", "editor": "number" }, "timeoutSeconds": { "title": "Request timeout (seconds)", "type": "integer", "description": "Timeout for fetching the sitemap.", "minimum": 3, "maximum": 30, "default": 10, "unit": "seconds", "editor": "number" }, "maxBytes": { "title": "Maximum sitemap bytes", "type": "integer", "description": "Maximum response body size to read. Larger sitemaps are stopped to keep runs bounded.", "minimum": 100000, "maximum": 5000000, "default": 2000000, "editor": "number" } }, "required": ["startUrl"]}{ "actorOutputSchemaVersion": 1, "title": "Output schema of Sitemap Freshness Auditor", "description": "Audit results are stored in the default dataset.", "properties": { "results": { "type": "string", "title": "Audit results", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1import { Actor } from 'apify';2import dns from 'node:dns/promises';3import net from 'node:net';4import { fileURLToPath } from 'node:url';5
6const USER_AGENT = 'SitemapFreshnessAuditor/0.1 (+https://apify.com)';7const DEFAULT_TIMEOUT_SECONDS = 10;8const DEFAULT_MAX_BYTES = 2 * 1024 * 1024;9const MAX_BYTES = 5 * 1024 * 1024;10const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);11
12function isPrivateIPv4(ip) {13 const parts = ip.split('.').map(Number);14 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;15 const [a, b] = parts;16 return a === 10 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || a === 127 || a === 0 || (a === 169 && b === 254);17}18
19function isPrivateIPv6(ip) {20 const normalized = ip.toLowerCase();21 return normalized === '::1' || normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe80:');22}23
24export async function normalizeAndValidateUrl(rawUrl) {25 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');26 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) throw new Error('Only HTTP and HTTPS URLs are supported');27 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;28 const url = new URL(withScheme);29 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');30 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');31
32 const literalType = net.isIP(url.hostname);33 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');34 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');35
36 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });37 for (const record of records) {38 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');39 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');40 }41 return url;42}43
44function clampInteger(value, fallback, min, max) {45 const parsed = Number(value);46 if (!Number.isFinite(parsed)) return fallback;47 return Math.min(Math.max(Math.trunc(parsed), min), max);48}49
50function decodeXml(value) {51 return (value || '').replace(/&/gi, '&').replace(/"/gi, '"').replace(/'/gi, "'").replace(/</gi, '<').replace(/>/gi, '>').trim();52}53
54function firstTag(block, tag) {55 const match = block.match(new RegExp(`<${tag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${tag}>`, 'i'));56 return match ? decodeXml(match[1]) : null;57}58
59export function resolveSitemapUrl(targetUrl) {60 const url = new URL(targetUrl);61 if (/\.xml(\.gz)?$/i.test(url.pathname) || url.pathname.toLowerCase().includes('sitemap')) return url;62 return new URL('/sitemap.xml', url.origin);63}64
65export function parseSitemapXml(xml, now = new Date(), staleAfterDays = 365) {66 const urls = [];67 const sitemapIndexes = [];68 const staleCutoff = now.getTime() - staleAfterDays * 24 * 60 * 60 * 1000;69
70 for (const match of xml.matchAll(/<url\b[^>]*>([\s\S]*?)<\/url>/gi)) {71 const loc = firstTag(match[1], 'loc');72 if (!loc) continue;73 const lastmod = firstTag(match[1], 'lastmod');74 const lastmodTime = lastmod ? Date.parse(lastmod) : NaN;75 urls.push({ loc, lastmod, hasLastmod: Boolean(lastmod), stale: Number.isFinite(lastmodTime) ? lastmodTime < staleCutoff : null });76 }77
78 for (const match of xml.matchAll(/<sitemap\b[^>]*>([\s\S]*?)<\/sitemap>/gi)) {79 const loc = firstTag(match[1], 'loc');80 if (!loc) continue;81 sitemapIndexes.push({ loc, lastmod: firstTag(match[1], 'lastmod') });82 }83
84 const withLastmod = urls.filter((item) => item.hasLastmod).length;85 const staleUrls = urls.filter((item) => item.stale).length;86 return {87 type: sitemapIndexes.length && !urls.length ? 'sitemapindex' : 'urlset',88 urlCount: urls.length,89 sitemapIndexCount: sitemapIndexes.length,90 lastmodCount: withLastmod,91 lastmodCoveragePercent: urls.length ? Math.round((withLastmod / urls.length) * 100) : 0,92 staleUrlCount: staleUrls,93 sampleUrls: urls.slice(0, 20),94 sitemapIndexes: sitemapIndexes.slice(0, 50),95 };96}97
98async function fetchText(initialUrl, timeoutSeconds, maxBytes, redirectsRemaining = 3) {99 await normalizeAndValidateUrl(initialUrl.href);100 const controller = new AbortController();101 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);102 try {103 const response = await fetch(initialUrl, { redirect: 'manual', signal: controller.signal, headers: { 'user-agent': USER_AGENT, accept: 'application/xml,text/xml,*/*;q=0.1' } });104 if (REDIRECT_STATUSES.has(response.status)) {105 if (redirectsRemaining <= 0) throw new Error('Too many redirects');106 const location = response.headers.get('location');107 if (!location) throw new Error('Redirect without Location header');108 return fetchText(new URL(location, initialUrl.href), timeoutSeconds, maxBytes, redirectsRemaining - 1);109 }110 const reader = response.body?.getReader();111 if (!reader) throw new Error('Response body is not readable');112 const chunks = [];113 let received = 0;114 while (true) {115 const { done, value } = await reader.read();116 if (done) break;117 received += value.byteLength;118 if (received > maxBytes) throw new Error(`Sitemap response exceeds ${maxBytes} byte limit`);119 chunks.push(value);120 }121 return { ok: response.ok, status: response.status, finalUrl: response.url || initialUrl.href, contentType: response.headers.get('content-type') || '', bytes: received, text: Buffer.concat(chunks).toString('utf8'), error: null };122 } catch (error) {123 return { ok: false, status: null, finalUrl: initialUrl.href, contentType: '', bytes: 0, text: '', error: error.message };124 } finally {125 clearTimeout(timeout);126 }127}128
129function score(parsed, fetchResult) {130 if (fetchResult.error || !fetchResult.ok) return { score: 0, grade: 'F', issues: [fetchResult.error || `Sitemap returned HTTP ${fetchResult.status}`], recommendations: ['Verify the sitemap URL is public, reachable, and returns XML.'] };131 let value = 100;132 const issues = [];133 const recommendations = [];134 if (parsed.type === 'sitemapindex') {135 issues.push('Input is a sitemap index; child sitemap freshness is not expanded in this bounded audit');136 recommendations.push('Audit important child sitemap URLs directly when freshness details are needed.');137 value -= 10;138 }139 if (parsed.urlCount === 0 && parsed.sitemapIndexCount === 0) {140 issues.push('No sitemap URLs or child sitemaps were parsed');141 recommendations.push('Check that the URL returns standard XML sitemap markup.');142 value -= 80;143 }144 if (parsed.urlCount > 0 && parsed.lastmodCoveragePercent < 80) {145 issues.push(`Only ${parsed.lastmodCoveragePercent}% of URLs include lastmod`);146 recommendations.push('Add lastmod values for important URLs so crawlers can prioritize recrawls.');147 value -= Math.min(35, 80 - parsed.lastmodCoveragePercent);148 }149 if (parsed.staleUrlCount > 0) {150 issues.push(`${parsed.staleUrlCount} URLs have stale lastmod dates`);151 recommendations.push('Refresh stale content timestamps only when content materially changes.');152 value -= Math.min(30, parsed.staleUrlCount * 3);153 }154 const bounded = Math.max(0, value);155 return { score: bounded, grade: bounded >= 90 ? 'A' : bounded >= 75 ? 'B' : bounded >= 60 ? 'C' : bounded >= 45 ? 'D' : 'F', issues, recommendations };156}157
158export async function auditSitemapFreshness(input) {159 const target = await normalizeAndValidateUrl(input.startUrl);160 const sitemapUrl = resolveSitemapUrl(target.href);161 const timeoutSeconds = clampInteger(input.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, 3, 30);162 const maxBytes = clampInteger(input.maxBytes, DEFAULT_MAX_BYTES, 100000, MAX_BYTES);163 const staleAfterDays = clampInteger(input.staleAfterDays, 365, 1, 3650);164 const fetched = await fetchText(sitemapUrl, timeoutSeconds, maxBytes);165 const parsed = fetched.error ? parseSitemapXml('', new Date(), staleAfterDays) : parseSitemapXml(fetched.text, new Date(), staleAfterDays);166 const scored = score(parsed, fetched);167 return { inputUrl: input.startUrl, sitemapUrl: sitemapUrl.href, finalUrl: fetched.finalUrl, ok: !fetched.error && fetched.ok && scored.score >= 75, checkedAt: new Date().toISOString(), status: fetched.status, contentType: fetched.contentType, bytes: fetched.bytes, staleAfterDays, score: scored.score, grade: scored.grade, ...parsed, issues: scored.issues, recommendations: scored.recommendations, error: fetched.error };168}169
170const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];171
172if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {173 await Actor.init();174 try {175 const result = await auditSitemapFreshness(await Actor.getInput() || {});176 await Actor.pushData(result);177 await Actor.setValue('OUTPUT', result);178 Actor.log.info('Sitemap freshness audit complete', { sitemapUrl: result.sitemapUrl, urls: result.urlCount, score: result.score, grade: result.grade });179 } finally {180 await Actor.exit();181 }182}1import test from 'node:test';2import assert from 'node:assert/strict';3import { normalizeAndValidateUrl, parseSitemapXml, resolveSitemapUrl } from '../src/main.js';4
5test('parseSitemapXml measures lastmod coverage and stale URLs', () => {6 const parsed = parseSitemapXml(`<?xml version="1.0"?>7 <urlset>8 <url><loc>https://example.com/new</loc><lastmod>2026-07-01</lastmod></url>9 <url><loc>https://example.com/old</loc><lastmod>2020-01-01</lastmod></url>10 <url><loc>https://example.com/no-date</loc></url>11 </urlset>`, new Date('2026-07-12T00:00:00Z'), 365);12 assert.equal(parsed.type, 'urlset');13 assert.equal(parsed.urlCount, 3);14 assert.equal(parsed.lastmodCount, 2);15 assert.equal(parsed.lastmodCoveragePercent, 67);16 assert.equal(parsed.staleUrlCount, 1);17});18
19test('parseSitemapXml detects sitemap indexes', () => {20 const parsed = parseSitemapXml(`<sitemapindex>21 <sitemap><loc>https://example.com/posts.xml</loc><lastmod>2026-07-01</lastmod></sitemap>22 </sitemapindex>`);23 assert.equal(parsed.type, 'sitemapindex');24 assert.equal(parsed.sitemapIndexCount, 1);25});26
27test('resolveSitemapUrl uses /sitemap.xml for site URLs', () => {28 assert.equal(resolveSitemapUrl('https://example.com/blog/page').href, 'https://example.com/sitemap.xml');29 assert.equal(resolveSitemapUrl('https://example.com/custom-sitemap.xml').href, 'https://example.com/custom-sitemap.xml');30});31
32test('normalizeAndValidateUrl blocks private IPv4 literal', async () => {33 await assert.rejects(() => normalizeAndValidateUrl('http://127.0.0.1/sitemap.xml'), /Private IPv4/);34});35
36test('normalizeAndValidateUrl rejects URL credentials', async () => {37 await assert.rejects(() => normalizeAndValidateUrl('https://user:pass@example.com/sitemap.xml'), /credentials/);38});