Website Intelligence API
Pricing
from $4.00 / 1,000 reports
Go to Apify Store
Website Intelligence API
Analyze website technologies, comments, structured data, robots.txt, and sitemaps in one run.
Website Intelligence API
Pricing
from $4.00 / 1,000 reports
Analyze website technologies, comments, structured data, robots.txt, and sitemaps in one run.
You can access the Website Intelligence API programmatically from your own applications by using the Apify API. You can also choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in API & Integrations in Apify Console.
{ "openapi": "3.0.1", "info": { "version": "1.0", "x-build-id": "XsJdThgjo7gzgbjQu" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/ib4ngz~website-intelligence-api/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-ib4ngz-website-intelligence-api", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/ib4ngz~website-intelligence-api/runs": { "post": { "operationId": "runs-sync-ib4ngz-website-intelligence-api", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/ib4ngz~website-intelligence-api/run-sync": { "post": { "operationId": "run-sync-ib4ngz-website-intelligence-api", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "minItems": 1, "type": "array", "description": "One or more HTTP(S) pages to analyze. Duplicate pages are crawled once while output cardinality is preserved.", "default": [ { "url": "https://apify.com" } ], "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxConcurrency": { "title": "Maximum browser concurrency", "minimum": 1, "maximum": 20, "type": "integer", "description": "Maximum number of pages analyzed concurrently.", "default": 3 }, "maxRequestRetries": { "title": "Request retries", "minimum": 0, "maximum": 5, "type": "integer", "description": "Retries after a failed browser navigation.", "default": 2 }, "timeoutSecs": { "title": "Request timeout", "minimum": 5, "maximum": 120, "type": "integer", "description": "Timeout in seconds for browser navigation and site-resource HTTP requests.", "default": 45 }, "commentsMaxDepth": { "title": "Comment discovery depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "Internal-link levels searched for comment systems. Use 0 to check only supplied pages.", "default": 1 }, "schemaMaxDepth": { "title": "Schema.org discovery depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "Internal-link levels searched for Schema.org extraction and validation. Use 0 for supplied pages only.", "default": 0 }, "brokenDataMaxDepth": { "title": "Broken structured-data discovery depth", "minimum": 0, "maximum": 10, "type": "integer", "description": "Internal-link levels scanned for broken structured-data findings. Use 0 for supplied pages only.", "default": 0 }, "maxDiscoveryPages": { "title": "Maximum discovered pages", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Maximum additional pages followed across the run, excluding supplied Start URLs.", "default": 100 }, "maxLinksPerPage": { "title": "Maximum links per page", "minimum": 1, "maximum": 500, "type": "integer", "description": "Maximum eligible same-hostname links considered from each analyzed page.", "default": 50 }, "analyzeComments": { "title": "Analyze comment systems", "type": "boolean", "description": "Detect embedded and native comment systems, including lazy-loaded widgets.", "default": true }, "analyzeTechnologies": { "title": "Detect technologies", "type": "boolean", "description": "Detect CMS, analytics, commerce, frontend, and infrastructure technologies.", "default": true }, "detectVersions": { "title": "Detect technology versions", "type": "boolean", "description": "Attempt to extract version numbers from available technology evidence.", "default": true }, "includeEvidence": { "title": "Include detection evidence", "type": "boolean", "description": "Include bounded evidence explaining each technology detection.", "default": true }, "analyzeStructuredData": { "title": "Analyze structured data", "type": "boolean", "description": "Extract and validate JSON-LD, Microdata, and RDFa from each page.", "default": true }, "includeRawStructuredData": { "title": "Include bounded raw structured data", "type": "boolean", "description": "Includes bounded source fragments, never the full page HTML.", "default": false }, "analyzeRobots": { "title": "Analyze robots.txt", "type": "boolean", "description": "Fetch and evaluate robots.txt once for each unique website origin.", "default": true }, "analyzeSitemap": { "title": "Analyze sitemaps", "type": "boolean", "description": "Discover and inspect sitemap files once for each unique website origin.", "default": true }, "userAgent": { "title": "Robots user agent", "type": "string", "description": "User-agent token used when evaluating robots.txt rules.", "default": "*" }, "maxSitemaps": { "title": "Maximum sitemap files", "minimum": 1, "maximum": 500, "type": "integer", "description": "Maximum number of sitemap and sitemap-index files processed per origin.", "default": 50 }, "maxSitemapUrls": { "title": "Maximum sitemap URLs", "minimum": 1, "maximum": 100000, "type": "integer", "description": "Maximum number of unique URLs collected from sitemaps per origin.", "default": 10000 }, "followSitemapIndex": { "title": "Follow sitemap indexes", "type": "boolean", "description": "Follow child sitemap references found in sitemap index files.", "default": true }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Optional Apify Proxy or custom proxy settings used by browser and site-resource requests.", "default": { "useApifyProxy": false } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } }}OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for Website Intelligence API from the options below:
If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients: