ads.txt Auditor
Pricing
Pay per usage
Fetch and deeply audit a public website's /ads.txt file for IAB Authorized Digital Sellers spec compliance.
Pricing
Pay per usage
Rating
0.0
(0)
Developer
Sanskar Jaiswal
Maintained by CommunityActor stats
0
Bookmarked
2
Total users
1
Monthly active users
4 days ago
Last modified
Categories
Share
FROM apify/actor-node:22
WORKDIR /usr/src/appCOPY package*.json ./RUN npm install --omit=devCOPY . ./CMD ["npm", "start"]{ "name": "ads-txt-auditor", "version": "0.1.0", "description": "Apify Actor that fetches and deeply audits a public website's ads.txt file for IAB Authorized Digital Sellers spec compliance. Validates entry formatting, DIRECT/RESELLER classification, authority ID presence, duplicate detection, and checks app-ads.txt availability. Returns a readiness score, letter grade, issues, and recommendations.", "type": "module", "private": true, "main": "src/main.js", "scripts": { "start": "node src/main.js", "test": "node --test test/*.test.js", "lint": "node --check src/main.js" }, "dependencies": { "apify": "^3.4.5" }, "devDependencies": {}}{ "actorSpecification": 1, "name": "ads-txt-auditor", "title": "ads.txt Auditor", "description": "Fetch and audit a public website's /ads.txt file for IAB Authorized Digital Sellers spec compliance. Validates entry formatting, DIRECT/RESELLER classification, authority IDs, duplicates, and app-ads.txt availability. Returns score, grade, issues, and recommendations.", "version": "0.1", "buildTag": "latest", "meta": { "templateId": "js-start" }, "input": "./input_schema.json", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }, "dockerfile": "./Dockerfile", "defaultRunOptions": { "build": "latest", "timeoutSecs": 120, "memoryMbytes": 512 }}{ "actorSpecification": 1, "views": { "overview": { "title": "ads.txt audits", "transformation": { "fields": [ "inputUrl", "finalUrl", "ok", "adsTxtFound", "appAdsTxtFound", "score", "grade", "entryCount", "directCount", "resellerCount", "duplicateCount", "invalidEntryCount", "checkedAt", "error" ] }, "display": { "component": "table", "properties": { "inputUrl": { "label": "Input URL", "format": "link" }, "finalUrl": { "label": "Final URL", "format": "link" }, "ok": { "label": "OK", "format": "boolean" }, "adsTxtFound": { "label": "ads.txt", "format": "boolean" }, "appAdsTxtFound": { "label": "app-ads.txt", "format": "boolean" }, "score": { "label": "Score" }, "grade": { "label": "Grade" }, "entryCount": { "label": "Entries" }, "directCount": { "label": "DIRECT" }, "resellerCount": { "label": "RESELLER" }, "duplicateCount": { "label": "Duplicates" }, "invalidEntryCount": { "label": "Invalid" }, "checkedAt": { "label": "Checked at", "format": "date" }, "error": { "label": "Error" } } } } }, "fields": { "inputUrl": { "type": "string", "title": "Input URL" }, "normalizedInputUrl": { "type": "string", "title": "Normalized input URL" }, "finalUrl": { "type": "string", "title": "Final URL" }, "https": { "type": "boolean", "title": "HTTPS" }, "ok": { "type": "boolean", "title": "OK" }, "checkedAt": { "type": "string", "title": "Checked at", "format": "date-time" }, "adsTxtFound": { "type": "boolean", "title": "ads.txt found" }, "adsTxtStatus": { "type": ["integer", "null"], "title": "ads.txt HTTP status" }, "adsTxtUrl": { "type": ["string", "null"], "title": "ads.txt URL" }, "adsTxtContentType": { "type": ["string", "null"], "title": "ads.txt Content-Type" }, "adsTxtLineCount": { "type": "integer", "title": "ads.txt line count" }, "rawAdsTxt": { "type": ["string", "null"], "title": "Raw ads.txt content" }, "appAdsTxtFound": { "type": "boolean", "title": "app-ads.txt found" }, "appAdsTxtStatus": { "type": ["integer", "null"], "title": "app-ads.txt HTTP status" }, "appAdsTxtUrl": { "type": ["string", "null"], "title": "app-ads.txt URL" }, "appAdsTxtEntryCount": { "type": "integer", "title": "app-ads.txt entry count" }, "entries": { "type": "array", "title": "Parsed ads.txt entries" }, "entryCount": { "type": "integer", "title": "Entry count" }, "directCount": { "type": "integer", "title": "DIRECT count" }, "resellerCount": { "type": "integer", "title": "RESELLER count" }, "commentCount": { "type": "integer", "title": "Comment count" }, "blankLineCount": { "type": "integer", "title": "Blank line count" }, "duplicateCount": { "type": "integer", "title": "Duplicate count" }, "invalidEntryCount": { "type": "integer", "title": "Invalid entry count" }, "invalidEntries": { "type": "array", "title": "Invalid entries" }, "duplicateEntries": { "type": "array", "title": "Duplicate entries" }, "uniqueDomains": { "type": "array", "title": "Unique ad system domains" }, "uniqueDomainCount": { "type": "integer", "title": "Unique ad system domain count" }, "score": { "type": "integer", "title": "Score" }, "grade": { "type": "string", "title": "Grade" }, "issues": { "type": "array", "title": "Issues" }, "recommendations": { "type": "array", "title": "Recommendations" }, "error": { "type": ["string", "null"], "title": "Error" } }}{ "title": "ads.txt Auditor input", "description": "Fetch and deeply audit a public website's ads.txt file for IAB Authorized Digital Sellers spec compliance. The actor parses every entry, validates field formatting, checks DIRECT/RESELLER account types, verifies authority ID presence, detects duplicate entries, and checks for app-ads.txt availability. Returns a readiness score, letter grade, issues, and recommendations. URLs with credentials and private network targets are rejected.", "type": "object", "schemaVersion": 1, "properties": { "startUrl": { "title": "URL", "type": "string", "description": "Public HTTP or HTTPS website URL to audit. The actor fetches /ads.txt from this domain.", "editor": "textfield", "prefill": "https://example.com/" }, "checkAppAdsTxt": { "title": "Check app-ads.txt", "type": "boolean", "description": "Also check for /app-ads.txt availability and validity. Defaults to true.", "editor": "checkbox", "default": true }, "timeoutSeconds": { "title": "Timeout seconds", "type": "integer", "description": "Request timeout from 3 to 30 seconds.", "editor": "number", "default": 10, "minimum": 3, "maximum": 30 } }, "required": ["startUrl"]}{ "actorOutputSchemaVersion": 1, "title": "ads.txt Audit Results", "properties": { "results": { "type": "string", "title": "Results", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1import { Actor } from 'apify';2import dns from 'node:dns/promises';3import net from 'node:net';4import { fileURLToPath } from 'node:url';5
6const USER_AGENT = 'AdsTxtAuditor/0.1 (+https://apify.com)';7const DEFAULT_TIMEOUT_SECONDS = 10;8const MAX_BODY_BYTES = 2 * 1024 * 1024; // 2 MB cap9
10/* ---------------------------------------------------------------------------11 * SSRF protection — shared pattern used across URL-fetching factory actors.12 * Allows only HTTP/HTTPS, rejects URL credentials, rejects localhost/private13 * IP literals, DNS-resolves hostnames, and rejects DNS that resolves to14 * private IP ranges.15 * ------------------------------------------------------------------------- */16
17function isPrivateIPv4(ip) {18 const parts = ip.split('.').map(Number);19 if (parts.length !== 4 || parts.some((n) => Number.isNaN(n))) return false;20 const [a, b] = parts;21 return a === 1022 || (a === 172 && b >= 16 && b <= 31)23 || (a === 192 && b === 168)24 || a === 12725 || a === 026 || (a === 169 && b === 254);27}28
29function isPrivateIPv6(ip) {30 const normalized = ip.toLowerCase();31 return normalized === '::1'32 || normalized.startsWith('fc')33 || normalized.startsWith('fd')34 || normalized.startsWith('fe80:');35}36
37export async function normalizeAndValidateUrl(rawUrl) {38 if (!rawUrl || typeof rawUrl !== 'string') throw new Error('startUrl is required');39 if (/^[a-z][a-z0-9+.-]*:/i.test(rawUrl) && !/^https?:\/\//i.test(rawUrl)) {40 throw new Error('Only HTTP and HTTPS URLs are supported');41 }42 const withScheme = /^https?:\/\//i.test(rawUrl) ? rawUrl : `https://${rawUrl}`;43 const url = new URL(withScheme);44 if (!['http:', 'https:'].includes(url.protocol)) throw new Error('Only HTTP and HTTPS URLs are supported');45 if (!url.hostname || url.username || url.password) throw new Error('URL must be public and must not include credentials');46
47 const literalType = net.isIP(url.hostname);48 if (literalType === 4 && isPrivateIPv4(url.hostname)) throw new Error('Private IPv4 targets are blocked');49 if (literalType === 6 && isPrivateIPv6(url.hostname)) throw new Error('Private IPv6 targets are blocked');50
51 const records = literalType ? [{ address: url.hostname, family: literalType }] : await dns.lookup(url.hostname, { all: true });52 for (const record of records) {53 if (record.family === 4 && isPrivateIPv4(record.address)) throw new Error('DNS resolves to a private IPv4 address; blocked for SSRF safety');54 if (record.family === 6 && isPrivateIPv6(record.address)) throw new Error('DNS resolves to a private IPv6 address; blocked for SSRF safety');55 }56 return url;57}58
59/* ---------------------------------------------------------------------------60 * HTTP text fetch with manual redirect revalidation (SSRF-safe).61 * Returns body text and metadata for a well-known file path.62 * ------------------------------------------------------------------------- */63
64async function fetchTextFile(initialUrl, timeoutSeconds, redirectsRemaining = 3) {65 await normalizeAndValidateUrl(initialUrl.href);66 const controller = new AbortController();67 const timeout = setTimeout(() => controller.abort(), timeoutSeconds * 1000);68 try {69 const response = await fetch(initialUrl, {70 redirect: 'manual',71 signal: controller.signal,72 headers: {73 'user-agent': USER_AGENT,74 accept: 'text/plain,*/*;q=0.1',75 },76 });77
78 if ([301, 302, 303, 307, 308].includes(response.status)) {79 if (redirectsRemaining <= 0) throw new Error('Too many redirects');80 const location = response.headers.get('location');81 if (!location) throw new Error('Redirect without Location header');82 const nextUrl = new URL(location, initialUrl.href);83 try { await response.arrayBuffer(); } catch { /* drain */ }84 return fetchTextFile(nextUrl, timeoutSeconds, redirectsRemaining - 1);85 }86
87 const contentType = response.headers.get('content-type') || null;88
89 if (!response.ok) {90 try { await response.arrayBuffer(); } catch { /* drain */ }91 return {92 ok: false,93 status: response.status,94 finalUrl: response.url || initialUrl.href,95 https: (response.url || initialUrl.href).startsWith('https://'),96 contentType,97 body: null,98 error: null,99 };100 }101
102 // ponytail: read up to MAX_BODY_BYTES then stop; prevents huge-file DoS.103 const buffer = await response.arrayBuffer();104 const bytes = new Uint8Array(buffer.slice(0, MAX_BODY_BYTES));105 const body = new TextDecoder('utf-8', { fatal: false }).decode(bytes);106
107 return {108 ok: true,109 status: response.status,110 finalUrl: response.url || initialUrl.href,111 https: (response.url || initialUrl.href).startsWith('https://'),112 contentType,113 body,114 error: null,115 };116 } catch (error) {117 return {118 ok: false,119 status: null,120 finalUrl: initialUrl.href,121 https: initialUrl.protocol === 'https:',122 contentType: null,123 body: null,124 error: error.message,125 };126 } finally {127 clearTimeout(timeout);128 }129}130
131/* ---------------------------------------------------------------------------132 * ads.txt line parser (IAB Authorized Digital Sellers spec v1.1).133 *134 * Each non-comment, non-blank line:135 * <adSystemDomain>, <publisherAccountId>, <accountType>, [<authorityId>]136 *137 * accountType: "DIRECT" or "RESELLER" (case-insensitive).138 * authorityId: optional TAG-ID or similar certification ID.139 *140 * v1.1 adds optional OWNERDOMAIN and MANAGERDOMAIN variables:141 * OWNERDOMAIN=<domain>142 * MANAGERDOMAIN=<domain>143 *144 * Lines starting with # are comments. Blank lines are ignored.145 * ------------------------------------------------------------------------- */146
147export function parseAdsTxtLine(line) {148 const trimmed = line.trim();149
150 if (trimmed === '') return { type: 'blank', raw: line };151 if (trimmed.startsWith('#')) return { type: 'comment', raw: line };152
153 // Variable assignments (OWNERDOMAIN, MANAGERDOMAIN, etc.)154 const varMatch = /^([A-Za-z][A-Za-z0-9_]*)\s*=\s*(.*)$/.exec(trimmed);155 if (varMatch) {156 const varName = varMatch[1].toUpperCase();157 const varValue = varMatch[2].trim();158 if (['OWNERDOMAIN', 'MANAGERDOMAIN'].includes(varName)) {159 return { type: 'variable', name: varName, value: varValue, raw: line };160 }161 // Unknown variable — treat as a non-standard extension, don't fail.162 return { type: 'variable', name: varName, value: varValue, raw: line };163 }164
165 // Data record: comma-separated fields.166 const fields = trimmed.split(',').map((f) => f.trim());167
168 // Must have at least 3 fields: domain, publisherId, accountType.169 if (fields.length < 3) {170 return {171 type: 'invalid',172 raw: line,173 reason: 'Entry has fewer than 3 fields (expected: adSystemDomain, publisherAccountId, accountType[, authorityId]).',174 };175 }176
177 const [adSystemDomain, publisherAccountId, accountTypeRaw, authorityId] = fields;178 const accountType = accountTypeRaw ? accountTypeRaw.toUpperCase() : '';179
180 if (!adSystemDomain) {181 return { type: 'invalid', raw: line, reason: 'Ad system domain is empty.' };182 }183 if (!publisherAccountId) {184 return { type: 'invalid', raw: line, reason: 'Publisher account ID is empty.' };185 }186 if (accountType !== 'DIRECT' && accountType !== 'RESELLER') {187 return {188 type: 'invalid',189 raw: line,190 reason: `Account type "${accountTypeRaw}" is not DIRECT or RESELLER.`,191 };192 }193
194 return {195 type: 'entry',196 adSystemDomain: adSystemDomain.toLowerCase(),197 publisherAccountId,198 accountType,199 authorityId: authorityId || null,200 raw: line,201 };202}203
204export function parseAdsTxt(body) {205 if (!body || typeof body !== 'string') {206 return {207 entries: [],208 comments: [],209 variables: [],210 invalidEntries: [],211 blankLines: 0,212 lineCount: 0,213 };214 }215
216 const lines = body.split(/\r?\n/);217 const entries = [];218 const comments = [];219 const variables = [];220 const invalidEntries = [];221 let blankLines = 0;222
223 for (const line of lines) {224 const parsed = parseAdsTxtLine(line);225 switch (parsed.type) {226 case 'blank': blankLines++; break;227 case 'comment': comments.push(parsed); break;228 case 'variable': variables.push(parsed); break;229 case 'entry': entries.push(parsed); break;230 case 'invalid': invalidEntries.push(parsed); break;231 }232 }233
234 return {235 entries,236 comments,237 variables,238 invalidEntries,239 blankLines,240 lineCount: lines.length,241 };242}243
244/* ---------------------------------------------------------------------------245 * Duplicate detection.246 * Two entries are duplicates if adSystemDomain + publisherAccountId + accountType247 * are all identical (authorityId differences are noted but not the primary key).248 * ------------------------------------------------------------------------- */249
250export function detectDuplicates(entries) {251 const seen = new Map();252 const duplicates = [];253
254 for (const entry of entries) {255 const key = `${entry.adSystemDomain},${entry.publisherAccountId},${entry.accountType}`;256 if (seen.has(key)) {257 duplicates.push({258 ...entry,259 duplicateOf: seen.get(key),260 });261 } else {262 seen.set(key, entry.raw);263 }264 }265
266 return duplicates;267}268
269/* ---------------------------------------------------------------------------270 * Per-entry validation.271 * ------------------------------------------------------------------------- */272
273export function validateEntry(entry) {274 const issues = [];275
276 // Ad system domain should look like a valid domain.277 if (!/\./.test(entry.adSystemDomain)) {278 issues.push({279 severity: 'warn',280 message: `Ad system domain "${entry.adSystemDomain}" does not contain a dot; it may not be a valid domain name.`,281 });282 }283
284 // RESELLER entries should have an authority ID (recommended by IAB).285 if (entry.accountType === 'RESELLER' && !entry.authorityId) {286 issues.push({287 severity: 'warn',288 message: `RESELLER entry for "${entry.adSystemDomain}" has no authority ID; IAB recommends including the TAG-ID for resellers.`,289 });290 }291
292 // DIRECT entries without authority ID — less critical but worth noting.293 if (entry.accountType === 'DIRECT' && !entry.authorityId) {294 issues.push({295 severity: 'info',296 message: `DIRECT entry for "${entry.adSystemDomain}" has no authority ID; including it improves supply chain transparency.`,297 });298 }299
300 return { ...entry, issues };301}302
303/* ---------------------------------------------------------------------------304 * Scoring and grading.305 * ------------------------------------------------------------------------- */306
307export function scoreAdsTxt(analysis) {308 if (!analysis.adsTxtFound) {309 return {310 score: 0,311 grade: 'F',312 issues: ['No ads.txt file found at /ads.txt. Buyers cannot verify authorized sellers for this domain.'],313 recommendations: [314 'Publish an ads.txt file at the domain root (https://<domain>/ads.txt) listing all authorized digital sellers.',315 'Follow the IAB Authorized Digital Sellers spec: each line is "adSystemDomain, publisherAccountId, DIRECT|RESELLER, authorityId".',316 'Include the TAG-ID as the authority ID for RESELLER entries to improve supply chain transparency.',317 ],318 };319 }320
321 let score = 100;322 const issues = [];323
324 // Penalize invalid entries.325 if (analysis.invalidEntryCount > 0) {326 const penalty = Math.min(40, analysis.invalidEntryCount * 10);327 score -= penalty;328 issues.push(`${analysis.invalidEntryCount} invalid entry/entries detected that do not conform to the IAB ads.txt format.`);329 }330
331 // Penalize duplicate entries.332 if (analysis.duplicateCount > 0) {333 const penalty = Math.min(20, analysis.duplicateCount * 5);334 score -= penalty;335 issues.push(`${analysis.duplicateCount} duplicate entry/entries found; remove duplicates to avoid buyer confusion.`);336 }337
338 // Penalize RESELLER entries without authority ID.339 const resellerNoAuthority = analysis.entries.filter(340 (e) => e.accountType === 'RESELLER' && !e.authorityId,341 );342 if (resellerNoAuthority.length > 0) {343 const penalty = Math.min(15, resellerNoAuthority.length * 3);344 score -= penalty;345 issues.push(`${resellerNoAuthority.length} RESELLER entry/entries are missing an authority ID (TAG-ID); IAB recommends including it.`);346 }347
348 // Warn on missing app-ads.txt if requested.349 if (analysis.appAdsTxtChecked && !analysis.appAdsTxtFound) {350 score -= 5;351 issues.push('app-ads.txt was not found at /app-ads.txt; if the domain has a mobile app, publish app-ads.txt for in-app inventory verification.');352 }353
354 // Warn on non-HTTPS ads.txt delivery.355 if (analysis.adsTxtFound && !analysis.https) {356 score -= 10;357 issues.push('ads.txt is served over HTTP, not HTTPS; buyers and crawlers may not reliably discover it.');358 }359
360 // Warn on incorrect Content-Type.361 if (analysis.adsTxtFound && analysis.adsTxtContentType) {362 const ct = analysis.adsTxtContentType.toLowerCase();363 if (!ct.includes('text/plain') && !ct.includes('text/plain')) {364 score -= 3;365 issues.push(`ads.txt is served with Content-Type "${analysis.adsTxtContentType}" instead of "text/plain"; some crawlers may reject it.`);366 }367 }368
369 // Positive: has OWNERDOMAIN or MANAGERDOMAIN (v1.1 transparency).370 const hasOwnerDomain = analysis.variables.some((v) => v.name === 'OWNERDOMAIN');371 const hasManagerDomain = analysis.variables.some((v) => v.name === 'MANAGERDOMAIN');372 if (hasOwnerDomain || hasManagerDomain) {373 score = Math.min(100, score + 3);374 }375
376 // Penalize zero entries (empty file or only comments).377 if (analysis.entryCount === 0) {378 score -= 30;379 issues.push('ads.txt file was found but contains zero valid entries; it may be empty or contain only comments.');380 }381
382 const bounded = Math.max(0, Math.min(100, score));383 const grade = bounded >= 95 ? 'A+'384 : bounded >= 85 ? 'A'385 : bounded >= 75 ? 'B'386 : bounded >= 65 ? 'C'387 : bounded >= 50 ? 'D'388 : bounded >= 30 ? 'E'389 : 'F';390
391 const recommendations = buildRecommendations(analysis, issues, bounded);392 return { score: bounded, grade, issues, recommendations };393}394
395function buildRecommendations(analysis, issues, score) {396 const recs = [...issues];397
398 if (!analysis.adsTxtFound) return recs;399
400 if (analysis.invalidEntryCount > 0) {401 recs.push('Fix or remove invalid entries. Each data line must be: "adSystemDomain, publisherAccountId, DIRECT|RESELLER, authorityId".');402 }403 if (analysis.duplicateCount > 0) {404 recs.push('Remove duplicate entries (same adSystemDomain + publisherAccountId + accountType) to avoid buyer confusion.');405 }406 const resellerNoAuthority = analysis.entries.filter(407 (e) => e.accountType === 'RESELLER' && !e.authorityId,408 );409 if (resellerNoAuthority.length > 0) {410 recs.push('Add the TAG-ID authority ID to RESELLER entries to improve supply chain transparency.');411 }412 if (analysis.entryCount === 0) {413 recs.push('Add at least one valid entry to the ads.txt file.');414 }415 if (analysis.appAdsTxtChecked && !analysis.appAdsTxtFound) {416 recs.push('If the domain has a mobile app, publish an app-ads.txt file at the root for in-app inventory verification.');417 }418 if (!analysis.https) {419 recs.push('Serve ads.txt over HTTPS to ensure reliable crawler discovery.');420 }421 if (score >= 95 && issues.length === 0) {422 recs.push('ads.txt posture is strong and spec-compliant. Continue monitoring after deploys and CDN cutovers.');423 }424
425 return recs.length ? recs : ['No ads.txt issues detected.'];426}427
428/* ---------------------------------------------------------------------------429 * Main audit pipeline.430 * ------------------------------------------------------------------------- */431
432export async function auditAdsTxt(input) {433 const timeoutSeconds = Math.min(Math.max(Number(input.timeoutSeconds || DEFAULT_TIMEOUT_SECONDS), 3), 30);434 const checkAppAdsTxt = input.checkAppAdsTxt !== false;435
436 let startUrl;437 try {438 startUrl = await normalizeAndValidateUrl(input.startUrl);439 } catch (validationError) {440 return errorResult(input.startUrl, validationError.message);441 }442
443 const adsTxtUrl = new URL('/ads.txt', startUrl.origin);444
445 let adsTxtFetched;446 try {447 adsTxtFetched = await fetchTextFile(adsTxtUrl, timeoutSeconds);448 } catch (fetchError) {449 return errorResult(input.startUrl, fetchError.message);450 }451
452 if (adsTxtFetched.error) {453 return errorResult(input.startUrl, adsTxtFetched.error, adsTxtFetched.finalUrl, adsTxtFetched.https, adsTxtFetched.status);454 }455
456 const adsTxtFound = adsTxtFetched.ok && adsTxtFetched.body !== null;457 const parsed = parseAdsTxt(adsTxtFetched.body);458 const duplicates = detectDuplicates(parsed.entries);459 const validatedEntries = parsed.entries.map(validateEntry);460
461 const directCount = validatedEntries.filter((e) => e.accountType === 'DIRECT').length;462 const resellerCount = validatedEntries.filter((e) => e.accountType === 'RESELLER').length;463 const uniqueDomains = [...new Set(validatedEntries.map((e) => e.adSystemDomain))];464
465 // Check app-ads.txt if requested and ads.txt was found.466 let appAdsTxtFound = false;467 let appAdsTxtStatus = null;468 let appAdsTxtUrl = null;469 let appAdsTxtEntryCount = 0;470
471 if (checkAppAdsTxt) {472 const appAdsTxtUrlObj = new URL('/app-ads.txt', startUrl.origin);473 appAdsTxtUrl = appAdsTxtUrlObj.href;474 const appFetched = await fetchTextFile(appAdsTxtUrlObj, timeoutSeconds);475 if (appFetched.ok && appFetched.body !== null) {476 appAdsTxtFound = true;477 appAdsTxtStatus = appFetched.status;478 const appParsed = parseAdsTxt(appFetched.body);479 appAdsTxtEntryCount = appParsed.entries.length;480 } else {481 appAdsTxtStatus = appFetched.status;482 }483 }484
485 const analysis = {486 adsTxtFound,487 adsTxtContentType: adsTxtFetched.contentType,488 https: adsTxtFetched.https,489 entries: validatedEntries,490 entryCount: validatedEntries.length,491 invalidEntryCount: parsed.invalidEntries.length,492 duplicateCount: duplicates.length,493 variables: parsed.variables,494 appAdsTxtChecked: checkAppAdsTxt,495 appAdsTxtFound,496 };497
498 const scored = scoreAdsTxt(analysis);499
500 return {501 inputUrl: input.startUrl,502 normalizedInputUrl: startUrl.href,503 finalUrl: adsTxtFetched.finalUrl,504 https: adsTxtFetched.https,505 ok: true,506 checkedAt: new Date().toISOString(),507 adsTxtFound,508 adsTxtStatus: adsTxtFetched.status,509 adsTxtUrl: adsTxtUrl.href,510 adsTxtContentType: adsTxtFetched.contentType,511 adsTxtLineCount: parsed.lineCount,512 rawAdsTxt: adsTxtFetched.body,513 appAdsTxtFound,514 appAdsTxtStatus,515 appAdsTxtUrl,516 appAdsTxtEntryCount,517 entries: validatedEntries,518 entryCount: validatedEntries.length,519 directCount,520 resellerCount,521 commentCount: parsed.comments.length,522 blankLineCount: parsed.blankLines,523 duplicateCount: duplicates.length,524 invalidEntryCount: parsed.invalidEntries.length,525 invalidEntries: parsed.invalidEntries,526 duplicateEntries: duplicates,527 uniqueDomains,528 uniqueDomainCount: uniqueDomains.length,529 variables: parsed.variables,530 score: scored.score,531 grade: scored.grade,532 issues: scored.issues,533 recommendations: scored.recommendations,534 error: null,535 };536}537
538function errorResult(inputUrl, errorMsg, finalUrl, https, httpStatus) {539 return {540 inputUrl,541 normalizedInputUrl: null,542 finalUrl: finalUrl || (typeof inputUrl === 'string' ? inputUrl : null),543 https: https ?? false,544 ok: false,545 checkedAt: new Date().toISOString(),546 adsTxtFound: false,547 adsTxtStatus: httpStatus ?? null,548 adsTxtUrl: null,549 adsTxtContentType: null,550 adsTxtLineCount: 0,551 rawAdsTxt: null,552 appAdsTxtFound: false,553 appAdsTxtStatus: null,554 appAdsTxtUrl: null,555 appAdsTxtEntryCount: 0,556 entries: [],557 entryCount: 0,558 directCount: 0,559 resellerCount: 0,560 commentCount: 0,561 blankLineCount: 0,562 duplicateCount: 0,563 invalidEntryCount: 0,564 invalidEntries: [],565 duplicateEntries: [],566 uniqueDomains: [],567 uniqueDomainCount: 0,568 variables: [],569 score: 0,570 grade: 'F',571 issues: ['The request failed before ads.txt could be inspected. Verify the URL is reachable and try again.'],572 recommendations: ['Verify the URL is public, reachable, and returns a response.'],573 error: errorMsg,574 };575}576
577/* ---------------------------------------------------------------------------578 * Apify runtime entry point.579 * ------------------------------------------------------------------------- */580
581const isExecutedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1];582
583if (process.env.NODE_ENV !== 'test' && isExecutedDirectly) {584 await Actor.init();585 try {586 const input = await Actor.getInput();587 const result = await auditAdsTxt(input || {});588 await Actor.pushData(result);589 await Actor.setValue('OUTPUT', result);590 Actor.log.info('ads.txt audit complete', { adsTxtFound: result.adsTxtFound, score: result.score, grade: result.grade, entryCount: result.entryCount });591 } finally {592 await Actor.exit();593 }594}1import test from 'node:test';2import assert from 'node:assert/strict';3import {4 parseAdsTxtLine,5 parseAdsTxt,6 detectDuplicates,7 validateEntry,8 scoreAdsTxt,9 normalizeAndValidateUrl,10 auditAdsTxt,11} from '../src/main.js';12
13/* --- parseAdsTxtLine: parsing ------------------------------------------ */14
15test('parseAdsTxtLine parses a valid DIRECT entry with authority ID', () => {16 const r = parseAdsTxtLine('greenbold.com, 1234, DIRECT, 5345b8ed5e');17 assert.equal(r.type, 'entry');18 assert.equal(r.adSystemDomain, 'greenbold.com');19 assert.equal(r.publisherAccountId, '1234');20 assert.equal(r.accountType, 'DIRECT');21 assert.equal(r.authorityId, '5345b8ed5e');22});23
24test('parseAdsTxtLine parses a valid RESELLER entry without authority ID', () => {25 const r = parseAdsTxtLine('rubiconproject.com, 5678, RESELLER');26 assert.equal(r.type, 'entry');27 assert.equal(r.adSystemDomain, 'rubiconproject.com');28 assert.equal(r.accountType, 'RESELLER');29 assert.equal(r.authorityId, null);30});31
32test('parseAdsTxtLine handles blank lines', () => {33 const r = parseAdsTxtLine('');34 assert.equal(r.type, 'blank');35});36
37test('parseAdsTxtLine handles comment lines', () => {38 const r = parseAdsTxtLine('# This is a comment');39 assert.equal(r.type, 'comment');40});41
42test('parseAdsTxtLine handles OWNERDOMAIN variable', () => {43 const r = parseAdsTxtLine('OWNERDOMAIN=example.com');44 assert.equal(r.type, 'variable');45 assert.equal(r.name, 'OWNERDOMAIN');46 assert.equal(r.value, 'example.com');47});48
49test('parseAdsTxtLine handles MANAGERDOMAIN variable', () => {50 const r = parseAdsTxtLine('MANAGERDOMAIN=ssp.example.com');51 assert.equal(r.type, 'variable');52 assert.equal(r.name, 'MANAGERDOMAIN');53});54
55test('parseAdsTxtLine rejects entry with fewer than 3 fields', () => {56 const r = parseAdsTxtLine('greenbold.com, 1234');57 assert.equal(r.type, 'invalid');58 assert.ok(r.reason.includes('fewer than 3 fields'));59});60
61test('parseAdsTxtLine rejects invalid account type', () => {62 const r = parseAdsTxtLine('greenbold.com, 1234, INDIRECT');63 assert.equal(r.type, 'invalid');64 assert.ok(r.reason.includes('DIRECT or RESELLER'));65});66
67test('parseAdsTxtLine handles case-insensitive account type', () => {68 const r = parseAdsTxtLine('greenbold.com, 1234, direct, abc123');69 assert.equal(r.type, 'entry');70 assert.equal(r.accountType, 'DIRECT');71});72
73/* --- parseAdsTxt: full body parsing ----------------------------------- */74
75test('parseAdsTxt parses a multi-line ads.txt body', () => {76 const body = '# comment\ngreenbold.com, 1234, DIRECT, abc123\n\nrubiconproject.com, 5678, RESELLER\nOWNERDOMAIN=example.com\n';77 const result = parseAdsTxt(body);78 assert.equal(result.entries.length, 2);79 assert.equal(result.comments.length, 1);80 assert.equal(result.variables.length, 1);81 assert.equal(result.blankLines, 2);82 assert.equal(result.lineCount, 6);83});84
85test('parseAdsTxt returns empty result for null input', () => {86 const result = parseAdsTxt(null);87 assert.equal(result.entries.length, 0);88});89
90/* --- detectDuplicates -------------------------------------------------- */91
92test('detectDuplicates finds duplicate entries', () => {93 const entries = [94 { adSystemDomain: 'a.com', publisherAccountId: '1', accountType: 'DIRECT', raw: 'a.com, 1, DIRECT' },95 { adSystemDomain: 'b.com', publisherAccountId: '2', accountType: 'RESELLER', raw: 'b.com, 2, RESELLER' },96 { adSystemDomain: 'a.com', publisherAccountId: '1', accountType: 'DIRECT', raw: 'a.com, 1, DIRECT' },97 ];98 const dupes = detectDuplicates(entries);99 assert.equal(dupes.length, 1);100 assert.equal(dupes[0].adSystemDomain, 'a.com');101});102
103test('detectDuplicates returns empty for no duplicates', () => {104 const entries = [105 { adSystemDomain: 'a.com', publisherAccountId: '1', accountType: 'DIRECT', raw: 'a.com, 1, DIRECT' },106 { adSystemDomain: 'b.com', publisherAccountId: '2', accountType: 'RESELLER', raw: 'b.com, 2, RESELLER' },107 ];108 assert.equal(detectDuplicates(entries).length, 0);109});110
111/* --- validateEntry ----------------------------------------------------- */112
113test('validateEntry warns on RESELLER without authority ID', () => {114 const entry = { adSystemDomain: 'a.com', publisherAccountId: '1', accountType: 'RESELLER', authorityId: null, raw: 'a.com, 1, RESELLER' };115 const v = validateEntry(entry);116 assert.ok(v.issues.some((i) => i.severity === 'warn' && i.message.includes('authority ID')));117});118
119test('validateEntry warns on domain without a dot', () => {120 const entry = { adSystemDomain: 'localhost', publisherAccountId: '1', accountType: 'DIRECT', authorityId: 'abc', raw: 'localhost, 1, DIRECT, abc' };121 const v = validateEntry(entry);122 assert.ok(v.issues.some((i) => i.severity === 'warn' && i.message.includes('dot')));123});124
125/* --- scoreAdsTxt ------------------------------------------------------- */126
127test('scoreAdsTxt returns F when ads.txt not found', () => {128 const scored = scoreAdsTxt({ adsTxtFound: false, entries: [], entryCount: 0, invalidEntryCount: 0, duplicateCount: 0, variables: [], appAdsTxtChecked: false, appAdsTxtFound: false });129 assert.equal(scored.score, 0);130 assert.equal(scored.grade, 'F');131});132
133test('scoreAdsTxt returns high score for clean file', () => {134 const analysis = {135 adsTxtFound: true,136 https: true,137 adsTxtContentType: 'text/plain; charset=utf-8',138 entries: [139 { adSystemDomain: 'a.com', publisherAccountId: '1', accountType: 'DIRECT', authorityId: 'abc' },140 { adSystemDomain: 'b.com', publisherAccountId: '2', accountType: 'RESELLER', authorityId: 'def' },141 ],142 entryCount: 2,143 invalidEntryCount: 0,144 duplicateCount: 0,145 variables: [{ name: 'OWNERDOMAIN', value: 'example.com' }],146 appAdsTxtChecked: false,147 appAdsTxtFound: false,148 };149 const scored = scoreAdsTxt(analysis);150 assert.ok(scored.score >= 90);151 assert.equal(scored.grade, 'A+');152});153
154test('scoreAdsTxt penalizes invalid entries', () => {155 const analysis = {156 adsTxtFound: true,157 https: true,158 adsTxtContentType: 'text/plain',159 entries: [{ adSystemDomain: 'a.com', publisherAccountId: '1', accountType: 'DIRECT', authorityId: 'abc' }],160 entryCount: 1,161 invalidEntryCount: 3,162 duplicateCount: 0,163 variables: [],164 appAdsTxtChecked: false,165 appAdsTxtFound: false,166 };167 const scored = scoreAdsTxt(analysis);168 assert.ok(scored.score < 75);169 assert.ok(scored.issues.some((i) => i.includes('invalid')));170});171
172/* --- normalizeAndValidateUrl: SSRF guards ------------------------------ */173
174test('normalizeAndValidateUrl blocks private IPv4 literals', async () => {175 await assert.rejects(() => normalizeAndValidateUrl('http://10.0.0.1/'), /Private IPv4/);176 await assert.rejects(() => normalizeAndValidateUrl('http://192.168.1.1/'), /Private IPv4/);177});178
179test('normalizeAndValidateUrl blocks localhost DNS resolution', async () => {180 await assert.rejects(() => normalizeAndValidateUrl('http://localhost/'), /private|Private|DNS/);181});182
183test('normalizeAndValidateUrl rejects credentials in URL', async () => {184 await assert.rejects(() => normalizeAndValidateUrl('https://user:pass@example.com/'), /credentials/);185});186
187test('normalizeAndValidateUrl rejects non-HTTP schemes', async () => {188 await assert.rejects(() => normalizeAndValidateUrl('file:///etc/passwd'), /Only HTTP/);189});190
191test('normalizeAndValidateUrl defaults missing scheme to https', async () => {192 const url = await normalizeAndValidateUrl('apify.com');193 assert.equal(url.protocol, 'https:');194 assert.equal(url.hostname, 'apify.com');195});196
197/* --- auditAdsTxt: end-to-end error path -------------------------------- */198
199test('auditAdsTxt returns an F-grade error object when the fetch fails', async () => {200 const result = await auditAdsTxt({ startUrl: 'https://this-host-does-not-exist-cron-test.invalid/' });201 assert.equal(result.ok, false);202 assert.equal(result.score, 0);203 assert.equal(result.grade, 'F');204 assert.equal(result.adsTxtFound, false);205 assert.ok(Array.isArray(result.issues));206 assert.ok(result.issues.length > 0);207});