Content-Type & X-Content-Type-Options Auditor
Pricing
Pay per usage
Audit one public URL for Content-Type accuracy, X-Content-Type-Options nosniff, and MIME confusion risk via magic-byte sniffing.
Pricing
Pay per usage
Rating
0.0
(0)
Developer
Sanskar Jaiswal
Maintained by CommunityActor stats
0
Bookmarked
2
Total users
1
Monthly active users
5 days ago
Last modified
Categories
Share
FROM apify/actor-node:22
WORKDIR /usr/src/appCOPY package*.json ./RUN npm install --omit=devCOPY . ./CMD ["npm", "start"]{ "name": "content-type-auditor", "version": "0.1.0", "description": "Apify Actor that audits one public URL for Content-Type accuracy, X-Content-Type-Options nosniff, and MIME confusion risk via magic-byte sniffing.", "type": "module", "private": true, "main": "src/main.js", "scripts": { "start": "node src/main.js", "test": "node --test test/*.test.js", "lint": "node --check src/main.js" }, "dependencies": { "apify": "^3.4.5" }, "devDependencies": {}}{ "actorSpecification": 1, "name": "content-type-auditor", "title": "Content-Type & X-Content-Type-Options Auditor", "description": "Audit one public URL for accurate Content-Type declaration and X-Content-Type-Options nosniff header, with magic-byte sniffing to detect MIME confusion risk.", "version": "0.1", "buildTag": "latest", "meta": { "templateId": "js-start" }, "input": "./input_schema.json", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }, "dockerfile": "./Dockerfile", "defaultRunOptions": { "build": "latest", "timeoutSecs": 120, "memoryMbytes": 512 }}{ "actorSpecification": 1, "views": { "overview": { "title": "Content-Type audits", "transformation": { "fields": [ "inputUrl", "finalUrl", "ok", "score", "grade", "declaredContentType", "sniffedType", "typeMismatch", "hasNosniff", "checkedAt", "error" ] }, "display": { "component": "table", "properties": { "inputUrl": { "label": "Input URL", "format": "link" }, "finalUrl": { "label": "Final URL", "format": "link" }, "ok": { "label": "OK", "format": "boolean" }, "score": { "label": "Score" }, "grade": { "label": "Grade" }, "declaredContentType": { "label": "Declared Content-Type" }, "sniffedType": { "label": "Sniffed type" }, "typeMismatch": { "label": "Type mismatch", "format": "boolean" }, "hasNosniff": { "label": "nosniff", "format": "boolean" }, "checkedAt": { "label": "Checked at", "format": "date" }, "error": { "label": "Page error" } } } } }, "fields": { "inputUrl": { "type": "string", "title": "Input URL" }, "normalizedInputUrl": { "type": "string", "title": "Normalized input URL" }, "finalUrl": { "type": "string", "title": "Final URL" }, "ok": { "type": "boolean", "title": "OK" }, "checkedAt": { "type": "string", "title": "Checked at", "format": "date-time" }, "score": { "type": "integer", "title": "Score" }, "grade": { "type": "string", "title": "Grade" }, "httpStatus": { "type": ["integer", "null"], "title": "HTTP status" }, "declaredContentType": { "type": ["string", "null"], "title": "Declared Content-Type" }, "declaredCharset": { "type": ["string", "null"], "title": "Declared charset" }, "xContentTypeOptions": { "type": ["string", "null"], "title": "X-Content-Type-Options header value" }, "hasNosniff": { "type": "boolean", "title": "nosniff present" }, "sniffedType": { "type": ["string", "null"], "title": "Magic-byte sniffed type" }, "typeMismatch": { "type": "boolean", "title": "Type mismatch" }, "confusionRisk": { "type": "string", "title": "MIME confusion risk level" }, "issues": { "type": "array", "title": "Issues" }, "recommendations": { "type": "array", "title": "Recommendations" }, "error": { "type": ["string", "null"], "title": "Page error" } }}{ "title": "Content-Type & X-Content-Type-Options Auditor input", "description": "Fetch one public URL and inspect its Content-Type header, X-Content-Type-Options header, and magic-byte sniff for MIME confusion risk. URLs with credentials and private network targets are rejected.", "type": "object", "schemaVersion": 1, "properties": { "startUrl": { "title": "URL", "type": "string", "description": "Public HTTP or HTTPS URL to audit.", "editor": "textfield", "prefill": "https://example.com/" }, "timeoutSeconds": { "title": "Timeout seconds", "type": "integer", "description": "Request timeout from 3 to 30 seconds.", "editor": "number", "default": 10, "minimum": 3, "maximum": 30 }, "maxBodyBytes": { "title": "Maximum body bytes to sniff", "type": "integer", "description": "Maximum response body bytes to read for magic-byte sniffing. Defaults to 65 KB and is capped at 512 KB.", "editor": "number", "default": 65536, "minimum": 1024, "maximum": 524288 } }, "required": ["startUrl"]}{ "actorOutputSchemaVersion": 1, "title": "Content-Type Audit Results", "properties": { "results": { "type": "string", "title": "Results", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1import { Actor } from 'apify';2import dns from 'node:dns/promises';3import net from 'node:net';4import { fileURLToPath } from 'node:url';5
6const USER_AGENT = 'ContentTypeAuditor/0.1 (+https://apify.com)';7const DEFAULT_TIMEOUT_SECONDS = 10;8const DEFAULT_MAX_BODY_BYTES = 65536;9const MAX_BODY_BYTES = 524288;10const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);11
12function isPrivateIPv4(ip) {13 const parts = ip.split('.').map(Number);14 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;15 const [a, b] = parts;16 return a === 10 || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || a === 127 || a === 0 || (a === 169 && b === 254);17}18
19function isPrivateIPv6(ip) {20 const normalized = ip.toLowerCase();21 return normalized === '::1' || normalized.startsWith('fc') || normalized.startsWith('fd') || normalized.startsWith('fe80:');22}23
24export async function normalizeAndValidateUrl(rawUrl) {25 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');26 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) throw new Error('Only HTTP and HTTPS URLs are supported');27 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;28 const url = new URL(withScheme);29 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');30 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');31
32 const literalType = net.isIP(url.hostname);33 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');34 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');35
36 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });37 for (const record of records) {38 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');39 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');40 }41 return url;42}43
44function clampInteger(value, fallback, min, max) {45 const parsed = Number(value);46 if (!Number.isFinite(parsed)) return fallback;47 return Math.min(Math.max(Math.trunc(parsed), min), max);48}49
50// ponytail: magic-byte signatures for common web types. Keep the list short —51// we only need enough to flag a declared Content-Type that disagrees with bytes.52const MAGIC_SIGNATURES = [53 { type: 'image/png', bytes: [0x89, 0x50, 0x4e, 0x47] },54 { type: 'image/jpeg', bytes: [0xff, 0xd8, 0xff] },55 { type: 'image/gif', bytes: [0x47, 0x49, 0x46, 0x38] },56 { type: 'image/webp', bytes: [0x52, 0x49, 0x46, 0x46] }, // RIFF; WebP checked via offset 8-1157 { type: 'image/svg+xml', isText: true, textMatch: '<svg' },58 { type: 'image/x-icon', bytes: [0x00, 0x00, 0x01, 0x00] },59 { type: 'application/pdf', bytes: [0x25, 0x50, 0x44, 0x46] },60 { type: 'application/zip', bytes: [0x50, 0x4b, 0x03, 0x04] },61 { type: 'application/gzip', bytes: [0x1f, 0x8b] },62 { type: 'application/x-bzip2', bytes: [0x42, 0x5a, 0x68] },63 { type: 'font/woff', bytes: [0x77, 0x4f, 0x46, 0x46] },64 { type: 'font/woff2', bytes: [0x77, 0x4f, 0x46, 0x32] },65 { type: 'font/ttf', bytes: [0x00, 0x01, 0x00, 0x00] },66 { type: 'font/otf', bytes: [0x4f, 0x54, 0x54, 0x4f] },67 { type: 'audio/mpeg', bytes: [0x49, 0x44, 0x33] }, // ID3 tag68 { type: 'audio/mpeg-alt', bytes: [0xff, 0xfb] }, // MP3 frame sync69 { type: 'video/mp4', bytes: [0x00, 0x00, 0x00] }, // box header; ftyp at offset 470 { type: 'video/webm', bytes: [0x1a, 0x45, 0xdf, 0xa3] }, // EBML71];72
73const TEXT_BYTES = [74 [0xef, 0xbb, 0xbf], // UTF-8 BOM75];76const JSON_PREFIX = '{';77const XML_PREFIX = '<?xml';78
79export function sniffMagicBytes(buf) {80 if (!buf || buf.length === 0) return null;81
82 // Text-based checks first (SVG, JSON, XML, HTML)83 const head = buf.subarray(0, Math.min(buf.length, 256)).toString('utf8').trimStart().toLowerCase();84 if (head.startsWith('<svg')) return 'image/svg+xml';85 if (head.startsWith(XML_PREFIX)) return 'application/xml';86 if (head.startsWith(JSON_PREFIX)) return 'application/json';87 if (head.startsWith('<!doctype html') || head.startsWith('<html') || head.startsWith('<head')) return 'text/html';88
89 // Binary signature checks90 for (const sig of MAGIC_SIGNATURES) {91 if (sig.isText) continue; // handled above92 if (sig.bytes.every((b, i) => buf[i] === b)) {93 // Refine RIFF: check for WEBP at offset 894 if (sig.type === 'image/webp') {95 const riffType = buf.subarray(8, 12).toString('ascii');96 if (riffType === 'WEBP') return 'image/webp';97 return null; // RIFF but not WebP98 }99 // Refine MP3 frame sync100 if (sig.type === 'audio/mpeg-alt') return 'audio/mpeg';101 // Refine MP4 box: check ftyp102 if (sig.type === 'video/mp4') {103 const ftyp = buf.subarray(4, 8).toString('ascii');104 if (ftyp === 'ftyp') return 'video/mp4';105 return null;106 }107 return sig.type;108 }109 }110
111 // BOM check112 for (const bom of TEXT_BYTES) {113 if (bom.every((b, i) => buf[i] === b)) return 'text/plain';114 }115
116 return null;117}118
119function parseContentType(value) {120 if (!value || typeof value !== 'string') return { type: null, charset: null };121 const parts = value.split(';').map((p) => p.trim());122 const type = parts[0].toLowerCase() || null;123 let charset = null;124 for (let i = 1; i < parts.length; i++) {125 const kv = parts[i].split('=');126 if (kv[0].trim().toLowerCase() === 'charset') charset = (kv[1] || '').trim().replace(/^["']|["']$/g, '').toLowerCase();127 }128 return { type, charset };129}130
131// ponytail: normalize types for comparison — strip parameters and collapse132// common aliases so that text/html and text/html; charset=utf-8 agree.133function normalizeType(t) {134 if (!t) return null;135 return t.split(';')[0].trim().toLowerCase();136}137
138function typesCompatible(declared, sniffed) {139 const d = normalizeType(declared);140 const s = normalizeType(sniffed);141 if (!d || !s) return true; // no sniff result → no mismatch142 if (d === s) return true;143 // HTML family144 const htmlTypes = new Set(['text/html', 'application/xhtml+xml']);145 if (htmlTypes.has(d) && htmlTypes.has(s)) return true;146 // XML family147 const xmlTypes = new Set(['application/xml', 'text/xml', 'application/svg+xml', 'image/svg+xml']);148 if (xmlTypes.has(d) && xmlTypes.has(s)) return true;149 // JSON family150 const jsonTypes = new Set(['application/json', 'text/json', 'application/json+oembed', 'application/ld+json']);151 if (jsonTypes.has(d) && jsonTypes.has(s)) return true;152 return false;153}154
155function classifyResource(type) {156 if (!type) return 'unknown';157 if (type.startsWith('text/html') || type.startsWith('application/xhtml')) return 'html';158 if (type.startsWith('image/')) return 'image';159 if (type.startsWith('font/') || type.includes('font')) return 'font';160 if (type.startsWith('text/css')) return 'stylesheet';161 if (type.includes('javascript') || type === 'application/javascript' || type === 'text/javascript') return 'script';162 if (type.startsWith('audio/') || type.startsWith('video/')) return 'media';163 if (type.startsWith('application/pdf')) return 'document';164 if (type.startsWith('application/zip') || type.startsWith('application/gzip') || type.startsWith('application/x-bzip2')) return 'archive';165 if (type.startsWith('text/')) return 'text';166 return 'other';167}168
169async function fetchUrl(initialUrl, timeoutSeconds, maxBodyBytes, redirectsRemaining = 3) {170 await normalizeAndValidateUrl(initialUrl.href);171 const controller = new AbortController();172 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);173 try {174 const response = await fetch(initialUrl, {175 redirect: 'manual',176 signal: controller.signal,177 headers: { 'user-agent': USER_AGENT, accept: '*/*' },178 });179 if (REDIRECT_STATUSES.has(response.status)) {180 if (redirectsRemaining <= 0) throw new Error('Too many redirects');181 const location = response.headers.get('location');182 if (!location) throw new Error('Redirect without Location header');183 return fetchUrl(new URL(location, initialUrl.href), timeoutSeconds, maxBodyBytes, redirectsRemaining - 1);184 }185 const reader = response.body?.getReader();186 let bodyBuf = null;187 if (reader) {188 const chunks = [];189 let received = 0;190 while (true) {191 const { done, value } = await reader.read();192 if (done) break;193 received += value.byteLength;194 if (received > maxBodyBytes) {195 // Take what we have for sniffing196 chunks.push(value.subarray(0, maxBodyBytes - (received - value.byteLength)));197 break;198 }199 chunks.push(value);200 }201 bodyBuf = Buffer.concat(chunks);202 }203 return {204 ok: response.ok,205 status: response.status,206 finalUrl: response.url || initialUrl.href,207 contentType: response.headers.get('content-type'),208 xContentTypeOptions: response.headers.get('x-content-type-options'),209 body: bodyBuf,210 error: null,211 };212 } catch (error) {213 return { ok: false, status: null, finalUrl: initialUrl.href, contentType: null, xContentTypeOptions: null, body: null, error: error.message };214 } finally {215 clearTimeout(timeout);216 }217}218
219function scoreResults(result) {220 if (result.error) return { score: 0, grade: 'F', issues: [result.error], recommendations: ['Verify the URL is public, reachable, and returns a response.'] };221
222 let score = 100;223 const issues = [];224 const recommendations = [];225
226 // X-Content-Type-Options: nosniff — critical for script/stylesheet types227 const declared = parseContentType(result.declaredContentType);228 const resourceClass = classifyResource(declared.type);229 const isExecutable = resourceClass === 'script' || resourceClass === 'stylesheet';230
231 if (!result.hasNosniff) {232 if (isExecutable) {233 score -= 30;234 issues.push('X-Content-Type-Options: nosniff is missing on a script or stylesheet resource');235 recommendations.push('Add X-Content-Type-Options: nosniff to prevent MIME-sniffing execution attacks on scripts and stylesheets.');236 } else {237 score -= 10;238 issues.push('X-Content-Type-Options: nosniff header is not set');239 recommendations.push('Add X-Content-Type-Options: nosniff to all responses to prevent browser MIME sniffing.');240 }241 }242
243 // Content-Type presence244 if (!declared.type) {245 score -= 25;246 issues.push('Content-Type header is missing');247 recommendations.push('Set an explicit Content-Type header so browsers do not guess the resource type from bytes.');248 } else if (declared.type === 'application/octet-stream') {249 // Generic fallback type250 if (result.sniffedType && result.sniffedType !== 'application/octet-stream') {251 score -= 10;252 issues.push(`Content-Type is generic application/octet-stream but bytes look like ${result.sniffedType}`);253 recommendations.push(`Set Content-Type to ${result.sniffedType} instead of the generic application/octet-stream.`);254 }255 }256
257 // Type mismatch between declared and sniffed258 if (result.typeMismatch) {259 if (isExecutable) {260 score -= 35;261 issues.push(`Content-Type mismatch: declared ${declared.type} but bytes appear to be ${result.sniffedType} on a script/stylesheet resource`);262 recommendations.push(`Fix the Content-Type to match the actual bytes (${result.sniffedType}). Without nosniff, browsers may execute mismatched content.`);263 } else {264 score -= 15;265 issues.push(`Content-Type mismatch: declared ${declared.type} but bytes appear to be ${result.sniffedType}`);266 recommendations.push(`Fix the Content-Type to match the actual bytes (${result.sniffedType}).`);267 }268 }269
270 // charset missing on text/html271 if (declared.type === 'text/html' && !declared.charset) {272 score -= 5;273 issues.push('HTML response is missing a charset declaration');274 recommendations.push('Add a charset parameter to the Content-Type header (e.g. text/html; charset=utf-8) to prevent encoding sniffing attacks.');275 }276
277 if (!issues.length) {278 recommendations.push('Content-Type declaration, charset, and X-Content-Type-Options look correct for this resource.');279 }280
281 const bounded = Math.max(0, score);282 const grade = bounded >= 90 ? 'A' : bounded >= 75 ? 'B' : bounded >= 60 ? 'C' : bounded >= 45 ? 'D' : 'F';283 return { score: bounded, grade, issues, recommendations };284}285
286export async function auditContentType(input) {287 const startUrl = await normalizeAndValidateUrl(input.startUrl);288 const timeoutSeconds = clampInteger(input.timeoutSeconds, DEFAULT_TIMEOUT_SECONDS, 3, 30);289 const maxBodyBytes = clampInteger(input.maxBodyBytes, DEFAULT_MAX_BODY_BYTES, 1024, MAX_BODY_BYTES);290
291 const fetched = await fetchUrl(startUrl, timeoutSeconds, maxBodyBytes);292
293 const ctHeader = fetched.contentType;294 const xctoHeader = fetched.xContentTypeOptions;295 const { type: declaredType, charset: declaredCharset } = parseContentType(ctHeader);296 const sniffedType = fetched.body ? sniffMagicBytes(fetched.body) : null;297 const typeMismatch = sniffedType !== null && !typesCompatible(declaredType, sniffedType);298
299 let confusionRisk = 'low';300 if (typeMismatch && (classifyResource(declaredType) === 'script' || classifyResource(declaredType) === 'stylesheet')) {301 confusionRisk = 'critical';302 } else if (typeMismatch) {303 confusionRisk = 'medium';304 } else if (!declaredType || declaredType === 'application/octet-stream') {305 confusionRisk = 'medium';306 } else if (!xctoHeader || xctoHeader.toLowerCase() !== 'nosniff') {307 confusionRisk = 'low';308 }309
310 const hasNosniff = xctoHeader !== null && xctoHeader.toLowerCase().includes('nosniff');311
312 const result = {313 inputUrl: input.startUrl,314 normalizedInputUrl: startUrl.href,315 finalUrl: fetched.finalUrl,316 ok: !fetched.error && fetched.ok,317 checkedAt: new Date().toISOString(),318 httpStatus: fetched.status,319 declaredContentType: ctHeader,320 declaredType,321 declaredCharset,322 xContentTypeOptions: xctoHeader,323 hasNosniff,324 sniffedType,325 typeMismatch,326 confusionRisk,327 issues: [],328 recommendations: [],329 error: fetched.error,330 };331
332 const scored = scoreResults(result);333 result.score = scored.score;334 result.grade = scored.grade;335 result.issues = scored.issues;336 result.recommendations = scored.recommendations;337
338 return result;339}340
341const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];342
343if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {344 await Actor.init();345 try {346 const result = await auditContentType(await Actor.getInput() || {});347 await Actor.pushData(result);348 await Actor.setValue('OUTPUT', result);349 Actor.log.info('Content-Type audit complete', { finalUrl: result.finalUrl, score: result.score, grade: result.grade, typeMismatch: result.typeMismatch, hasNosniff: result.hasNosniff });350 } finally {351 await Actor.exit();352 }353}1import test from 'node:test';2import assert from 'node:assert/strict';3import { sniffMagicBytes, normalizeAndValidateUrl } from '../src/main.js';4
5test('sniffMagicBytes detects PNG', () => {6 const buf = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);7 assert.equal(sniffMagicBytes(buf), 'image/png');8});9
10test('sniffMagicBytes detects JPEG', () => {11 const buf = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10]);12 assert.equal(sniffMagicBytes(buf), 'image/jpeg');13});14
15test('sniffMagicBytes detects PDF', () => {16 const buf = Buffer.from('%PDF-1.4\n');17 assert.equal(sniffMagicBytes(buf), 'application/pdf');18});19
20test('sniffMagicBytes detects SVG via text content', () => {21 const buf = Buffer.from('<svg xmlns="http://www.w3.org/2000/svg"');22 assert.equal(sniffMagicBytes(buf), 'image/svg+xml');23});24
25test('sniffMagicBytes detects JSON', () => {26 const buf = Buffer.from('{"key":"value"}');27 assert.equal(sniffMagicBytes(buf), 'application/json');28});29
30test('sniffMagicBytes detects HTML', () => {31 const buf = Buffer.from('<!DOCTYPE html>\n<html><head></head></html>');32 assert.equal(sniffMagicBytes(buf), 'text/html');33});34
35test('sniffMagicBytes detects WebP via RIFF+WEBP', () => {36 const buf = Buffer.from([0x52, 0x49, 0x46, 0x46, 0x00, 0x00, 0x00, 0x00, 0x57, 0x45, 0x42, 0x50]);37 assert.equal(sniffMagicBytes(buf), 'image/webp');38});39
40test('sniffMagicBytes returns null for empty buffer', () => {41 assert.equal(sniffMagicBytes(Buffer.alloc(0)), null);42});43
44test('sniffMagicBytes returns null for unknown binary', () => {45 const buf = Buffer.from([0x00, 0x00, 0x00, 0x00, 0x00, 0x00]);46 assert.equal(sniffMagicBytes(buf), null);47});48
49test('sniffMagicBytes detects gzip', () => {50 const buf = Buffer.from([0x1f, 0x8b, 0x08, 0x00]);51 assert.equal(sniffMagicBytes(buf), 'application/gzip');52});53
54test('sniffMagicBytes detects WOFF2 font', () => {55 const buf = Buffer.from([0x77, 0x4f, 0x46, 0x32, 0x00, 0x01]);56 assert.equal(sniffMagicBytes(buf), 'font/woff2');57});58
59test('normalizeAndValidateUrl blocks private and credentialed targets', async () => {60 await assert.rejects(() => normalizeAndValidateUrl('http://127.0.0.1/'), /Private IPv4/);61 await assert.rejects(() => normalizeAndValidateUrl('https://user:pass@example.com/'), /credentials/);62 await assert.rejects(() => normalizeAndValidateUrl('file:///etc/passwd'), /Only HTTP/);63});64
65test('normalizeAndValidateUrl blocks private DNS resolution', async () => {66 // localhost resolves to 127.0.0.1 on most systems67 await assert.rejects(() => normalizeAndValidateUrl('http://localhost/'), /private|Private|DNS/);68});