Compression Headers Auditor
Pricing
Pay per usage
Audit a public URL's HTTP compression headers for gzip, brotli, zstd, and deflate. Detects double-compression, re-compression of binary content, and missing Vary negotiation.
Pricing
Pay per usage
Rating
0.0
(0)
Developer
Sanskar Jaiswal
Maintained by CommunityActor stats
0
Bookmarked
2
Total users
1
Monthly active users
4 days ago
Last modified
Categories
Share
FROM apify/actor-node:20
COPY package*.json ./RUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional
COPY . ./
CMD ["npm", "start"]{ "name": "compression-headers-auditor", "version": "0.1.0", "description": "Apify Actor that audits public URLs for HTTP compression headers (Content-Encoding, Transfer-Encoding, Vary: Accept-Encoding) and returns a content-aware compressibility score, grade, and performance recommendations.", "type": "module", "private": true, "main": "src/main.js", "scripts": { "start": "node src/main.js", "test": "node --test test/*.test.js", "lint": "node --check src/main.js" }, "dependencies": { "apify": "^3.4.5" }, "devDependencies": {}}{ "actorSpecification": 1, "name": "compression-headers-auditor", "title": "Compression Headers Auditor", "description": "Audit a public URL's HTTP compression headers (Content-Encoding, Transfer-Encoding, Vary: Accept-Encoding). Detects gzip, brotli, zstd, deflate, double-compression, and re-compression of binary content. Returns a compressibility score, grade, and devops recommendations.", "version": "0.1", "buildTag": "latest", "meta": { "templateId": "js-start" }, "input": "./input_schema.json", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }, "dockerfile": "./Dockerfile", "defaultRunOptions": { "build": "latest", "timeoutSecs": 60, "memoryMbytes": 512 }}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "inputUrl": { "type": "string", "description": "The URL provided as input" }, "finalUrl": { "type": "string", "description": "Final URL after redirects" }, "https": { "type": "boolean", "description": "Whether the final response was served over HTTPS" }, "status": { "type": "integer", "description": "Final HTTP status code" }, "contentType": { "type": "string", "description": "Content-Type of the final response" }, "resourceClass": { "type": "string", "description": "static-asset | html | already-compressed | other" }, "contentEncoding": { "type": "string", "description": "Parsed Content-Encoding value (gzip, br, zstd, deflate, identity), or null if absent" }, "transferEncoding": { "type": "string", "description": "Parsed Transfer-Encoding value (e.g., chunked), or null if absent" }, "compressions": { "type": "array", "description": "Ordered list of detected compression algorithms in Content-Encoding" }, "varyAcceptEncoding": { "type": "boolean", "description": "Whether Vary includes Accept-Encoding" }, "vary": { "type": "array", "description": "Tokenized Vary header values" }, "contentLength": { "type": "integer", "description": "Content-Length response header value, or null" }, "headers": { "type": "array", "description": "Per-header analysis objects (name, status, note, recommendation)" }, "issues": { "type": "array", "description": "Aggregated issue descriptions" }, "score": { "type": "integer", "description": "Compression readiness score (0-100)" }, "grade": { "type": "string", "description": "Letter grade (A+, A, B, C, D, E, F)" }, "checkedAt": { "type": "string", "format": "date-time", "description": "ISO timestamp" }, "recommendations": { "type": "array", "description": "Actionable recommendations for improving compression" } } }, "views": { "overview": { "title": "Overview", "transformation": { "fields": ["finalUrl", "https", "status", "contentType", "resourceClass", "contentEncoding", "varyAcceptEncoding", "grade", "score", "issues", "recommendations", "checkedAt"] }, "display": { "component": "table", "properties": { "finalUrl": { "label": "URL", "format": "link" }, "https": { "label": "HTTPS", "format": "boolean" }, "status": { "label": "Status", "format": "number" }, "contentType": { "label": "Content-Type", "format": "text" }, "resourceClass": { "label": "Resource class", "format": "text" }, "contentEncoding": { "label": "Content-Encoding", "format": "text" }, "varyAcceptEncoding": { "label": "Vary: Accept-Encoding", "format": "boolean" }, "grade": { "label": "Grade", "format": "text" }, "score": { "label": "Score", "format": "number" }, "issues": { "label": "Issues", "format": "array" }, "recommendations": { "label": "Recommendations", "format": "array" }, "checkedAt": { "label": "Checked", "format": "date" } } } } }}{ "title": "Compression Headers Auditor", "type": "object", "schemaVersion": 1, "description": "Audit a public URL's HTTP compression headers in one API call. Detects gzip, brotli, zstd, and deflate Content-Encoding, double-compression on already-compressed content types, missing Vary: Accept-Encoding, and returns a content-aware compressibility score, grade, and performance recommendations.", "properties": { "startUrl": { "title": "URL to audit", "type": "string", "description": "Public URL to audit. The actor fetches headers once and inspects compression-related response headers. HTTP and HTTPS only. Private IP ranges are blocked.", "editor": "textfield", "prefill": "https://example.com", "pattern": "^https?://.+" }, "useHeadRequest": { "title": "Use HEAD instead of GET", "type": "boolean", "description": "When true, the actor issues a HEAD request; some servers return different headers on HEAD. Defaults to false (GET, then drain body).", "default": false, "editor": "checkbox" }, "timeoutSeconds": { "title": "Request timeout (seconds)", "type": "integer", "description": "Timeout for the HTTP request.", "minimum": 3, "maximum": 30, "default": 10, "unit": "seconds", "editor": "number" } }, "required": ["startUrl"]}{ "actorOutputSchemaVersion": 1, "title": "Output schema of Compression Headers Auditor", "description": "Audit results are stored in the default dataset.", "properties": { "results": { "type": "string", "title": "Audit results", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1import { Actor } from 'apify';2import dns from 'node:dns/promises';3import net from 'node:net';4import { fileURLToPath } from 'node:url';5
6const USER_AGENT = 'CompressionHeadersAuditor/0.1 (+https://apify.com)';7const DEFAULT_TIMEOUT_SECONDS = 10;8
9// ponytail: list of content-type prefixes that are already-compressed binary formats.10// Re-compressing these wastes CPU at the edge and rarely saves bytes.11const ALREADY_COMPRESSED = [12 /^image\/(jpeg|png|gif|webp|avif|heic|heif)/i,13 /^video\//i,14 /^audio\//i,15 /^application\/zip/i,16 /^application\/(x-)?gzip/i,17 /^application\/x-bzip2/i,18 /^application\/x-7z-compressed/i,19 /^application\/x-tar/i,20 /^application\/x-rar/i,21 /^application\/pdf/i,22 /^application\/wasm/i,23 /^font\/(woff|woff2)/i,24 /^application\/octet-stream/i,25];26
27/* ---------------------------------------------------------------------------28 * SSRF protection — shared pattern used across URL-fetching factory actors.29 * Allows only HTTP/HTTPS, rejects URL credentials, rejects localhost/private30 * IP literals, DNS-resolves hostnames, and rejects DNS that resolves to31 * private IP ranges.32 * ------------------------------------------------------------------------- */33
34function isPrivateIPv4(ip) {35 const parts = ip.split('.').map(Number);36 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;37 const [a, b] = parts;38 return a === 1039 || (a === 172 && b >= 16 && b <= 31)40 || (a === 192 && b === 168)41 || a === 12742 || a === 043 || (a === 169 && b === 254);44}45
46function isPrivateIPv6(ip) {47 const normalized = ip.toLowerCase();48 return normalized === '::1'49 || normalized.startsWith('fc')50 || normalized.startsWith('fd')51 || normalized.startsWith('fe80:');52}53
54export async function normalizeAndValidateUrl(rawUrl) {55 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');56 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {57 throw new Error('Only HTTP and HTTPS URLs are supported');58 }59
60 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;61 const url = new URL(withScheme);62 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');63 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');64
65 const literalType = net.isIP(url.hostname);66 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');67 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');68
69 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });70 for (const record of records) {71 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');72 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');73 }74 return url;75}76
77/* ---------------------------------------------------------------------------78 * HTTP fetch with manual redirect revalidation (SSRF-safe).79 * Body is drained and discarded — we only inspect response headers.80 * ------------------------------------------------------------------------- */81
82async function fetchHeaders(initialUrl, timeoutSeconds, useHead, redirectsRemaining = 3) {83 await normalizeAndValidateUrl(initialUrl.href);84 const controller = new AbortController();85 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);86 try {87 const response = await fetch(initialUrl, {88 method: useHead ? 'HEAD' : 'GET',89 redirect: 'manual',90 signal: controller.signal,91 headers: {92 'user-agent': USER_AGENT,93 accept: 'text/html,application/xhtml+xml,application/json,*/*;q=0.1',94 // Send Accept-Encoding so the server is willing to negotiate compression.95 'accept-encoding': 'gzip, deflate, br, zstd',96 },97 });98
99 if ([301, 302, 303, 307, 308].includes(response.status)) {100 if (redirectsRemaining <= 0) throw new Error('Too many redirects');101 const location = response.headers.get('location');102 if (!location) throw new Error('Redirect without Location header');103 const nextUrl = new URL(location, initialUrl.href);104 await normalizeAndValidateUrl(nextUrl.href);105 return fetchHeaders(nextUrl, timeoutSeconds, useHead, redirectsRemaining - 1);106 }107
108 try { await response.arrayBuffer(); } catch { /* ignore body errors */ }109
110 const headerMap = {};111 response.headers.forEach((value, key) => { headerMap[key.toLowerCase()] = value; });112
113 return {114 ok: response.ok,115 status: response.status,116 finalUrl: new URL(response.url || initialUrl.href).href,117 https: (response.url || initialUrl.href).startsWith('https://'),118 headers: headerMap,119 error: null,120 };121 } catch (error) {122 return {123 ok: false,124 status: null,125 finalUrl: initialUrl.href,126 https: initialUrl.protocol === 'https:',127 headers: {},128 error: error.message,129 };130 } finally {131 clearTimeout(timeout);132 }133}134
135/* ---------------------------------------------------------------------------136 * Compression-specific parsing and analysis.137 * ------------------------------------------------------------------------- */138
139const KNOWN_COMPRESSIONS = new Set(['gzip', 'br', 'deflate', 'zstd']);140
141// Parse a Content-Encoding header into an ordered list of compression tokens.142// e.g. "br, gzip" => ['br','gzip']; "identity" => []; "x-gzip" => ['x-gzip']143export function parseContentEncoding(value) {144 if (!value || !value.trim()) return [];145 // identity = no compression146 const raw = value.trim().toLowerCase();147 if (raw === 'identity') return [];148 return raw.split(',').map((t) => t.trim()).filter(Boolean);149}150
151export function resourceClassFromContentType(contentType) {152 if (!contentType) return 'other';153 const base = contentType.split(';')[0].trim().toLowerCase();154 if (/^text\/html$|^application\/xhtml\+xml$/.test(base)) return 'html';155 if (ALREADY_COMPRESSED.some((re) => re.test(base))) return 'already-compressed';156 if (/^text\/css$|^text\/javascript$|^application\/javascript$|^application\/json$|^application\/xml$|^text\/xml$|^text\/plain$/.test(base)) {157 return 'static-asset';158 }159 // ponytail: any image/* not in ALREADY_COMPRESSED (e.g. image/svg+xml) is already-compressed for practical purposes160 if (/^image\//.test(base)) return 'already-compressed';161 return 'other';162}163
164// Build a summary object from raw headers.165export function analyzeCompressionHeaders(headerMap) {166 const contentTypeRaw = headerMap['content-type'] || '';167 const contentType = contentTypeRaw;168 const resourceClass = resourceClassFromContentType(contentType);169
170 const contentEncodingRaw = headerMap['content-encoding'] || '';171 const compressions = parseContentEncoding(contentEncodingRaw);172 const contentEncoding = compressions.length ? compressions.join(', ') : null;173
174 const transferEncodingRaw = headerMap['transfer-encoding'] || '';175 const transferEncoding = transferEncodingRaw ? transferEncodingRaw.trim() : null;176
177 const varyRaw = headerMap['vary'] || '';178 const vary = varyRaw ? varyRaw.split(',').map((v) => v.trim().toLowerCase()).filter(Boolean) : [];179 const varyAcceptEncoding = vary.some((v) => v === 'accept-encoding' || v === '*');180
181 const contentLengthRaw = headerMap['content-length'];182 let contentLength = null;183 if (contentLengthRaw !== undefined) {184 const n = Number(contentLengthRaw);185 if (!Number.isNaN(n) && n >= 0) contentLength = Math.floor(n);186 }187
188 return {189 contentType,190 resourceClass,191 compressions,192 contentEncoding,193 transferEncoding,194 vary,195 varyAcceptEncoding,196 contentLength,197 };198}199
200/* ---------------------------------------------------------------------------201 * Per-header status evaluations.202 * ------------------------------------------------------------------------- */203
204function evalContentEncoding(summary) {205 if (!summary.compressions.length) {206 if (summary.resourceClass === 'already-compressed') {207 return { status: 'good', note: 'No Content-Encoding on already-compressed content (image/video/font/archive). Correct — do not re-compress.' };208 }209 if (summary.resourceClass === 'html' || summary.resourceClass === 'static-asset') {210 return {211 status: 'warn',212 note: 'No Content-Encoding. Text-based resources should be compressed (gzip or brotli) to reduce transfer size.',213 recommendation: 'Enable Content-Encoding: gzip or br at the edge or origin for text-based responses (HTML, CSS, JS, JSON, XML, plain text).',214 };215 }216 return { status: 'info', note: 'No Content-Encoding.' };217 }218
219 const unknown = summary.compressions.filter((c) => !KNOWN_COMPRESSIONS.has(c));220 if (unknown.length) {221 return {222 status: 'warn',223 note: `Content-Encoding: ${summary.contentEncoding}. Unknown encoding(s): ${unknown.join(', ')}.`,224 recommendation: 'Use a standard Content-Encoding (gzip, br, deflate, zstd) to avoid client incompatibility.',225 };226 }227
228 // Double-compression: layered encodings (e.g., 'gzip, br' is sometimes valid transport chain, but layered 'gzip, gzip' is almost always wrong)229 const unique = new Set(summary.compressions);230 const doubled = summary.compressions.length - unique.size;231 if (doubled > 0) {232 return {233 status: 'warn',234 note: `Content-Encoding: ${summary.contentEncoding}. Duplicate compression algorithm detected; double-compression wastes CPU and can break clients.`,235 recommendation: 'Apply each compression algorithm at most once. If the origin compressed the resource, the edge should not re-compress it again.',236 };237 }238
239 if (summary.resourceClass === 'already-compressed') {240 return {241 status: 'warn',242 note: `Content-Encoding: ${summary.contentEncoding} on already-compressed content (${summary.contentType.split(';')[0]}). Re-compressing binary formats wastes CPU and rarely saves bytes.`,243 recommendation: 'Exclude already-compressed content types (images, videos, fonts, archives, PDF, WASM) from compression at the edge and origin.',244 };245 }246
247 const notes = [];248 if (summary.compressions.includes('br')) notes.push('brotli (good text compression)');249 if (summary.compressions.includes('zstd')) notes.push('zstd (fast modern compression)');250 if (summary.compressions.includes('gzip')) notes.push('gzip (widely compatible)');251 if (summary.compressions.includes('deflate')) notes.push('deflate (raw)');252 return {253 status: 'good',254 note: `Content-Encoding: ${summary.contentEncoding}. ${notes.join('; ')}.`,255 };256}257
258function evalVary(summary) {259 if (!summary.compressions.length) {260 // ponytail: no Content-Encoding = no negotiation needed; absence is fine261 if (!summary.vary.length) return { status: 'info', note: 'No Vary header (no Content-Encoding present, so no negotiation needed).' };262 if (summary.varyAcceptEncoding) {263 return {264 status: 'info',265 note: 'Vary includes Accept-Encoding, but no Content-Encoding is present. Harmless but unnecessary.',266 };267 }268 return { status: 'info', note: `Vary: ${summary.vary.join(', ')}` };269 }270
271 // Content-Encoding is present → Vary MUST include Accept-Encoding272 if (!summary.varyAcceptEncoding) {273 return {274 status: 'warn',275 note: 'Content-Encoding is set, but Vary does not include Accept-Encoding. Intermediary caches may serve a compressed response to a client that does not support it.',276 recommendation: 'Add Accept-Encoding to the Vary header whenever Content-Encoding is negotiated, so caches do not serve brotli/gzip to clients that only accept identity.',277 };278 }279 return { status: 'good', note: 'Vary includes Accept-Encoding. Content negotiation for compression is correctly signalled.' };280}281
282function evalTransferEncoding(summary) {283 if (!summary.transferEncoding) {284 return { status: 'info', note: 'No Transfer-Encoding header.' };285 }286 const te = summary.transferEncoding.toLowerCase();287 // Check for compression first — "gzip, chunked" should warn on the gzip portion.288 const teTokens = te.split(',').map((t) => t.trim());289 const hasCompressionToken = teTokens.some((t) => /gzip|deflate|br|zstd|compress/.test(t));290 if (hasCompressionToken) {291 return {292 status: 'warn',293 note: `Transfer-Encoding includes compression: ${te}. Transfer-Encoding compression is largely unsupported in HTTP/2+ and some intermediaries.`,294 recommendation: 'Prefer Content-Encoding for compression. Avoid Transfer-Encoding: gzip except in legacy HTTP/1.1 pipelines.',295 };296 }297 if (/chunked/.test(te)) {298 return { status: 'info', note: `Transfer-Encoding: ${te}. Chunked streaming is standard for dynamic responses.` };299 }300 return { status: 'info', note: `Transfer-Encoding: ${summary.transferEncoding}` };301}302
303function evalCompressionChoice(summary) {304 if (!summary.compressions.length) return { status: 'info', note: 'No compression applied — cannot evaluate algorithm choice.' };305
306 // ponytail: brotli for text > gzip for compatibility; zstd emerging.307 const hasBr = summary.compressions.includes('br');308 const hasGzip = summary.compressions.includes('gzip');309 const hasZstd = summary.compressions.includes('zstd');310
311 if (summary.resourceClass === 'html' || summary.resourceClass === 'static-asset') {312 if (hasBr) return { status: 'good', note: 'Brotli is applied to text-based content — best compression for HTML, CSS, JS.' };313 if (hasZstd) return { status: 'good', note: 'zstd is applied — fast modern compression with good ratio.' };314 if (hasGzip) return { status: 'info', note: 'gzip is applied — universally compatible but brotli offers ~15-25% better compression for text. Zstd offers faster decompression.' };315 return { status: 'info', note: `Compression: ${summary.contentEncoding}` };316 }317
318 return { status: 'info', note: `Compression: ${summary.contentEncoding} on ${summary.resourceClass}.` };319}320
321const HEADER_CHECKS = [322 { name: 'content-encoding', title: 'Content-Encoding', weight: 40, fn: evalContentEncoding },323 { name: 'vary', title: 'Vary: Accept-Encoding', weight: 30, fn: evalVary },324 { name: 'compression-choice', title: 'Compression Algorithm', weight: 15, fn: evalCompressionChoice },325 { name: 'transfer-encoding', title: 'Transfer-Encoding', weight: 15, fn: evalTransferEncoding },326];327
328export function buildHeaderReports(summary) {329 const reports = [];330 let earned = 0;331 let possible = 0;332 for (const check of HEADER_CHECKS) {333 const evaluation = check.fn(summary);334 // ponytail: 'info' = no-op for scoring; 'missing'/'warn' count against possible335 if (evaluation.status !== 'info') possible += check.weight;336 if (evaluation.status === 'good') earned += check.weight;337 else if (evaluation.status === 'warn') earned += Math.round(check.weight * 0.5);338 reports.push({339 name: check.title,340 header: check.name,341 status: evaluation.status,342 note: evaluation.note,343 weight: check.weight,344 recommendation: evaluation.recommendation || null,345 });346 }347 return { reports, earned, possible };348}349
350export function scoreAudit(earned, possible) {351 if (possible === 0) return 0;352 return Math.min(100, Math.max(0, Math.round((earned / possible) * 100)));353}354
355export function gradeFromScore(score) {356 if (score >= 95) return 'A+';357 if (score >= 85) return 'A';358 if (score >= 75) return 'B';359 if (score >= 65) return 'C';360 if (score >= 50) return 'D';361 if (score >= 30) return 'E';362 return 'F';363}364
365export function buildRecommendations(reports, summary) {366 const recs = new Set();367 for (const r of reports) {368 if (r.recommendation) recs.add(r.recommendation);369 }370
371 // Targeted cross-cutting recommendations.372 if (!summary.compressions.length && (summary.resourceClass === 'html' || summary.resourceClass === 'static-asset')) {373 recs.add('Enable compression (gzip or brotli) for this text-based resource. Brotli at level 4-6 typically beats gzip by 15-25%.');374 }375 if (summary.compressions.length && summary.resourceClass === 'already-compressed') {376 recs.add('Remove the Content-Encoding for already-compressed content types. Configure the edge/server to skip compression for images, fonts, videos, archives, PDF, and WASM.');377 }378 if (summary.compressions.length && !summary.varyAcceptEncoding) {379 recs.add('Add Accept-Encoding to the Vary header to prevent caches from serving a compressed variant to clients that did not request it.');380 }381 if (summary.compressions.length > 1) {382 recs.add('Multiple Content-Encoding values detected. Ensure layered compression is intentional (some CDNs chain br→identity); duplicate compression should be removed.');383 }384 if (recs.size === 0) {385 recs.add('Compression headers look well-configured for this content type. Run this audit after edge/CDN changes to catch regressions.');386 }387 return [...recs];388}389
390/* ---------------------------------------------------------------------------391 * Main audit pipeline.392 * ------------------------------------------------------------------------- */393
394export async function auditCompressionHeaders(input) {395 const startUrl = await normalizeAndValidateUrl(input.startUrl);396 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);397 const useHead = Boolean(input.useHeadRequest);398
399 const fetchResult = await fetchHeaders(startUrl, timeoutSeconds, useHead);400
401 if (fetchResult.error) {402 return {403 inputUrl: input.startUrl,404 finalUrl: fetchResult.finalUrl,405 https: fetchResult.https,406 status: null,407 contentType: '',408 resourceClass: 'other',409 contentEncoding: null,410 transferEncoding: null,411 compressions: [],412 varyAcceptEncoding: false,413 vary: [],414 contentLength: null,415 headers: [],416 issues: [],417 score: 0,418 grade: 'F',419 checkedAt: new Date().toISOString(),420 recommendations: ['The request failed before headers could be inspected. Verify the URL is reachable and try again.'],421 error: fetchResult.error,422 };423 }424
425 const summary = analyzeCompressionHeaders(fetchResult.headers);426 const { reports, earned, possible } = buildHeaderReports(summary);427 const score = scoreAudit(earned, possible);428 const grade = gradeFromScore(score);429 const recommendations = buildRecommendations(reports, summary);430
431 const issues = reports432 .filter((r) => r.status === 'warn' || r.status === 'missing')433 .map((r) => `${r.name}: ${r.note}`);434
435 return {436 inputUrl: input.startUrl,437 finalUrl: fetchResult.finalUrl,438 https: fetchResult.https,439 status: fetchResult.status,440 contentType: summary.contentType,441 resourceClass: summary.resourceClass,442 contentEncoding: summary.contentEncoding,443 transferEncoding: summary.transferEncoding,444 compressions: summary.compressions,445 varyAcceptEncoding: summary.varyAcceptEncoding,446 vary: summary.vary,447 contentLength: summary.contentLength,448 headers: reports,449 issues,450 score,451 grade,452 checkedAt: new Date().toISOString(),453 recommendations,454 };455}456
457/* ---------------------------------------------------------------------------458 * Apify runtime entry point.459 * ------------------------------------------------------------------------- */460
461const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];462
463if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {464 await Actor.init();465 try {466 const input = await Actor.getInput();467 const result = await auditCompressionHeaders(input || {});468 await Actor.pushData(result);469 await Actor.setValue('OUTPUT', result);470 Actor.log.info('Compression headers audit complete', { finalUrl: result.finalUrl, score: result.score, grade: result.grade });471 } finally {472 await Actor.exit();473 }474}1import test from 'node:test';2import assert from 'node:assert/strict';3import {4 analyzeCompressionHeaders,5 buildHeaderReports,6 buildRecommendations,7 gradeFromScore,8 normalizeAndValidateUrl,9 parseContentEncoding,10 resourceClassFromContentType,11 scoreAudit,12} from '../src/main.js';13
14/* ---------------------------------------------------------------------------15 * SSRF / URL validation security tests16 * ------------------------------------------------------------------------- */17
18test('normalizeAndValidateUrl blocks private IPv4 literal', async () => {19 await assert.rejects(() => normalizeAndValidateUrl('http://127.0.0.1/'), /Private IPv4/);20});21
22test('normalizeAndValidateUrl blocks 169.254 link-local', async () => {23 await assert.rejects(() => normalizeAndValidateUrl('http://169.254.169.254/'), /Private IPv4/);24});25
26test('normalizeAndValidateUrl rejects URL credentials', async () => {27 await assert.rejects(() => normalizeAndValidateUrl('https://user:pass@example.com'), /credentials/);28});29
30test('normalizeAndValidateUrl rejects non-HTTP schemes', async () => {31 await assert.rejects(() => normalizeAndValidateUrl('file:///etc/hosts'), /HTTP and HTTPS/);32});33
34test('normalizeAndValidateUrl rejects ftp scheme', async () => {35 await assert.rejects(() => normalizeAndValidateUrl('ftp://example.com/file'), /HTTP and HTTPS/);36});37
38test('normalizeAndValidateUrl accepts public hostname and defaults to https', async () => {39 const url = await normalizeAndValidateUrl('example.com');40 assert.equal(url.hostname, 'example.com');41 assert.equal(url.protocol, 'https:');42});43
44/* ---------------------------------------------------------------------------45 * Content-Encoding parsing tests46 * ------------------------------------------------------------------------- */47
48test('parseContentEncoding extracts single gzip', () => {49 assert.deepEqual(parseContentEncoding('gzip'), ['gzip']);50});51
52test('parseContentEncoding extracts layered br, gzip', () => {53 assert.deepEqual(parseContentEncoding('br, gzip'), ['br', 'gzip']);54});55
56test('parseContentEncoding treats identity as no compression', () => {57 assert.deepEqual(parseContentEncoding('identity'), []);58});59
60test('parseContentEncoding handles empty value', () => {61 assert.deepEqual(parseContentEncoding(''), []);62 assert.deepEqual(parseContentEncoding(undefined), []);63 assert.deepEqual(parseContentEncoding(' '), []);64});65
66test('parseContentEncoding preserves x-prefixed aliases', () => {67 assert.deepEqual(parseContentEncoding('x-gzip'), ['x-gzip']);68});69
70/* ---------------------------------------------------------------------------71 * Resource classification tests72 * ------------------------------------------------------------------------- */73
74test('resourceClassFromContentType classifies html vs static vs already-compressed', () => {75 assert.equal(resourceClassFromContentType('text/html; charset=utf-8'), 'html');76 assert.equal(resourceClassFromContentType('application/xhtml+xml'), 'html');77 assert.equal(resourceClassFromContentType('text/css'), 'static-asset');78 assert.equal(resourceClassFromContentType('application/json'), 'static-asset');79 assert.equal(resourceClassFromContentType('image/jpeg'), 'already-compressed');80 assert.equal(resourceClassFromContentType('image/png'), 'already-compressed');81 assert.equal(resourceClassFromContentType('font/woff2'), 'already-compressed');82 assert.equal(resourceClassFromContentType('video/mp4'), 'already-compressed');83 assert.equal(resourceClassFromContentType('application/pdf'), 'already-compressed');84 assert.equal(resourceClassFromContentType('application/wasm'), 'already-compressed');85 assert.equal(resourceClassFromContentType('image/svg+xml'), 'already-compressed');86 assert.equal(resourceClassFromContentType('application/zip'), 'already-compressed');87 assert.equal(resourceClassFromContentType(''), 'other');88});89
90/* ---------------------------------------------------------------------------91 * Header analysis behavioral tests92 * ------------------------------------------------------------------------- */93
94const BASE_HEADERS = (extra = {}) => {95 const h = { 'content-type': 'text/html; charset=utf-8' };96 Object.assign(h, extra);97 const out = {};98 for (const [k, v] of Object.entries(h)) out[k.toLowerCase()] = v;99 return out;100};101
102test('html with brotli + Vary: Accept-Encoding is good', () => {103 const summary = analyzeCompressionHeaders(BASE_HEADERS({104 'content-encoding': 'br',105 'vary': 'Accept-Encoding',106 }));107 assert.equal(summary.resourceClass, 'html');108 assert.deepEqual(summary.compressions, ['br']);109 assert.equal(summary.varyAcceptEncoding, true);110 const { reports, earned, possible } = buildHeaderReports(summary);111 const ce = reports.find((r) => r.header === 'content-encoding');112 assert.equal(ce.status, 'good');113 const vary = reports.find((r) => r.header === 'vary');114 assert.equal(vary.status, 'good');115 assert.ok(earned > 0);116 assert.ok(possible > 0 && possible <= 100);117});118
119test('html without Content-Encoding is warned', () => {120 const summary = analyzeCompressionHeaders(BASE_HEADERS({}));121 const { reports } = buildHeaderReports(summary);122 const ce = reports.find((r) => r.header === 'content-encoding');123 assert.equal(ce.status, 'warn');124 assert.ok(ce.recommendation.includes('gzip'));125});126
127test('Content-Encoding without Vary: Accept-Encoding is warned', () => {128 const summary = analyzeCompressionHeaders(BASE_HEADERS({129 'content-encoding': 'gzip',130 }));131 const { reports } = buildHeaderReports(summary);132 const vary = reports.find((r) => r.header === 'vary');133 assert.equal(vary.status, 'warn');134 assert.ok(vary.recommendation.includes('Accept-Encoding'));135});136
137test('already-compressed content with Content-Encoding is warned', () => {138 const summary = analyzeCompressionHeaders(BASE_HEADERS({139 'content-type': 'image/jpeg',140 'content-encoding': 'gzip',141 'vary': 'Accept-Encoding',142 }));143 assert.equal(summary.resourceClass, 'already-compressed');144 const { reports } = buildHeaderReports(summary);145 const ce = reports.find((r) => r.header === 'content-encoding');146 assert.equal(ce.status, 'warn');147 assert.ok(ce.note.includes('already-compressed'));148});149
150test('already-compressed content without Content-Encoding is good', () => {151 const summary = analyzeCompressionHeaders(BASE_HEADERS({152 'content-type': 'font/woff2',153 }));154 const { reports } = buildHeaderReports(summary);155 const ce = reports.find((r) => r.header === 'content-encoding');156 assert.equal(ce.status, 'good');157});158
159test('double compression (gzip, gzip) is warned', () => {160 const summary = analyzeCompressionHeaders(BASE_HEADERS({161 'content-encoding': 'gzip, gzip',162 'vary': 'Accept-Encoding',163 }));164 const { reports } = buildHeaderReports(summary);165 const ce = reports.find((r) => r.header === 'content-encoding');166 assert.equal(ce.status, 'warn');167 assert.ok(ce.note.includes('Duplicate'));168});169
170test('unknown Content-Encoding is warned', () => {171 const summary = analyzeCompressionHeaders(BASE_HEADERS({172 'content-encoding': 'sdch',173 'vary': 'Accept-Encoding',174 }));175 const { reports } = buildHeaderReports(summary);176 const ce = reports.find((r) => r.header === 'content-encoding');177 assert.equal(ce.status, 'warn');178 assert.ok(ce.note.includes('Unknown'));179});180
181test('Transfer-Encoding: gzip is warned (legacy)', () => {182 const summary = analyzeCompressionHeaders(BASE_HEADERS({183 'content-encoding': 'gzip',184 'vary': 'Accept-Encoding',185 'transfer-encoding': 'gzip, chunked',186 }));187 const { reports } = buildHeaderReports(summary);188 const te = reports.find((r) => r.header === 'transfer-encoding');189 assert.equal(te.status, 'warn');190 assert.ok(te.recommendation.includes('Content-Encoding'));191});192
193test('Transfer-Encoding: chunked is info', () => {194 const summary = analyzeCompressionHeaders(BASE_HEADERS({195 'content-encoding': 'gzip',196 'vary': 'Accept-Encoding',197 'transfer-encoding': 'chunked',198 }));199 const { reports } = buildHeaderReports(summary);200 const te = reports.find((r) => r.header === 'transfer-encoding');201 assert.equal(te.status, 'info');202});203
204test('static asset with brotli scores high', () => {205 const summary = analyzeCompressionHeaders(BASE_HEADERS({206 'content-type': 'text/css',207 'content-encoding': 'br',208 'vary': 'Accept-Encoding',209 }));210 const { earned, possible } = buildHeaderReports(summary);211 const score = scoreAudit(earned, possible);212 assert.ok(score >= 90, `expected score >= 90, got ${score}`);213});214
215test('static asset with gzip only is info on compression-choice (gzip is compatible)', () => {216 const summary = analyzeCompressionHeaders(BASE_HEADERS({217 'content-type': 'application/javascript',218 'content-encoding': 'gzip',219 'vary': 'Accept-Encoding',220 }));221 const { reports } = buildHeaderReports(summary);222 const choice = reports.find((r) => r.header === 'compression-choice');223 assert.equal(choice.status, 'info');224 assert.ok(choice.note.includes('brotli'));225});226
227test('zstd compression is recognized as good for text', () => {228 const summary = analyzeCompressionHeaders(BASE_HEADERS({229 'content-type': 'text/html; charset=utf-8',230 'content-encoding': 'zstd',231 'vary': 'Accept-Encoding',232 }));233 const { reports } = buildHeaderReports(summary);234 const ce = reports.find((r) => r.header === 'content-encoding');235 assert.equal(ce.status, 'good');236 const choice = reports.find((r) => r.header === 'compression-choice');237 assert.equal(choice.status, 'good');238});239
240test('identity Content-Encoding is treated as no compression', () => {241 const summary = analyzeCompressionHeaders(BASE_HEADERS({242 'content-encoding': 'identity',243 }));244 assert.deepEqual(summary.compressions, []);245 assert.equal(summary.contentEncoding, null);246});247
248test('Content-Length is parsed as integer', () => {249 const summary = analyzeCompressionHeaders(BASE_HEADERS({250 'content-encoding': 'gzip',251 'vary': 'Accept-Encoding',252 'content-length': '12345',253 }));254 assert.equal(summary.contentLength, 12345);255});256
257test('Vary: * counts as Accept-Encoding match', () => {258 const summary = analyzeCompressionHeaders(BASE_HEADERS({259 'content-encoding': 'gzip',260 'vary': '*',261 }));262 assert.equal(summary.varyAcceptEncoding, true);263 const { reports } = buildHeaderReports(summary);264 const vary = reports.find((r) => r.header === 'vary');265 assert.equal(vary.status, 'good');266});267
268/* ---------------------------------------------------------------------------269 * Scoring tests270 * ------------------------------------------------------------------------- */271
272test('scoreAudit returns 0 when no points possible', () => {273 assert.equal(scoreAudit(0, 0), 0);274});275
276test('scoreAudit clamps to 0-100', () => {277 assert.equal(scoreAudit(-5, 90), 0);278 assert.equal(scoreAudit(200, 90), 100);279});280
281test('gradeFromScore returns expected letter grades', () => {282 assert.equal(gradeFromScore(100), 'A+');283 assert.equal(gradeFromScore(90), 'A');284 assert.equal(gradeFromScore(80), 'B');285 assert.equal(gradeFromScore(70), 'C');286 assert.equal(gradeFromScore(55), 'D');287 assert.equal(gradeFromScore(40), 'E');288 assert.equal(gradeFromScore(10), 'F');289});290
291/* ---------------------------------------------------------------------------292 * Recommendation tests293 * ------------------------------------------------------------------------- */294
295test('buildRecommendations suggests enabling compression for text without encoding', () => {296 const summary = analyzeCompressionHeaders(BASE_HEADERS({}));297 const { reports } = buildHeaderReports(summary);298 const recs = buildRecommendations(reports, summary);299 assert.ok(recs.some((r) => r.includes('brotli') || r.includes('gzip')));300});301
302test('buildRecommendations says exclude already-compressed when image is re-compressed', () => {303 const summary = analyzeCompressionHeaders(BASE_HEADERS({304 'content-type': 'image/png',305 'content-encoding': 'gzip',306 'vary': 'Accept-Encoding',307 }));308 const { reports } = buildHeaderReports(summary);309 const recs = buildRecommendations(reports, summary);310 assert.ok(recs.some((r) => r.includes('already-compressed')));311});312
313test('buildRecommendations adds positive note when well-configured', () => {314 const summary = analyzeCompressionHeaders(BASE_HEADERS({315 'content-type': 'text/css',316 'content-encoding': 'br',317 'vary': 'Accept-Encoding',318 }));319 const { reports } = buildHeaderReports(summary);320 const recs = buildRecommendations(reports, summary);321 assert.ok(recs.some((r) => r.includes('well-configured')));322});