Katana Web Crawler (ProjectDiscovery)
Pricing
from $0.01 / actor start
Crawl websites with ProjectDiscovery Katana and stream results as structured JSONL into an Apify dataset.
Pricing
from $0.01 / actor start
Rating
0.0
(0)
Developer
R.L.
Maintained by CommunityActor stats
0
Bookmarked
4
Total users
0
Monthly active users
19 days ago
Last modified
Categories
Share
You can access the Katana Web Crawler (ProjectDiscovery) programmatically from your own applications by using the Apify API. You can also choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in API & Integrations in Apify Console.
{ "openapi": "3.0.1", "info": { "version": "0.1", "x-build-id": "epbbgw3LQt5pWWATQ" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/rl1987~pd-katana/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-rl1987-pd-katana", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/rl1987~pd-katana/runs": { "post": { "operationId": "runs-sync-rl1987-pd-katana", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/rl1987~pd-katana/run-sync": { "post": { "operationId": "run-sync-rl1987-pd-katana", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "urls" ], "properties": { "urls": { "title": "Target URLs", "type": "array", "description": "One or more target URLs / domains to crawl. Each entry is passed to Katana as a seed (-u).", "items": { "type": "string" } }, "exclude": { "title": "Exclude hosts", "type": "array", "description": "Exclude hosts matching a filter: 'cdn', 'private-ips', a CIDR, an IP, or a regex (Katana -exclude).", "items": { "type": "string" } }, "depth": { "title": "Crawl depth", "minimum": 1, "maximum": 100, "type": "integer", "description": "Maximum depth to crawl (Katana -depth).", "default": 3 }, "crawlDuration": { "title": "Max crawl duration (seconds)", "minimum": 0, "type": "integer", "description": "Maximum time to crawl per target in seconds (Katana -crawl-duration). 0 = unlimited.", "default": 0 }, "strategy": { "title": "Visit strategy", "enum": [ "depth-first", "breadth-first" ], "type": "string", "description": "Traversal order (Katana -strategy).", "default": "depth-first" }, "jsCrawl": { "title": "JavaScript crawling", "type": "boolean", "description": "Parse & crawl endpoints inside JavaScript files (Katana -js-crawl).", "default": false }, "jsluice": { "title": "Jsluice parsing", "type": "boolean", "description": "Enable jsluice parsing in JavaScript files. Memory intensive (Katana -jsluice).", "default": false }, "knownFiles": { "title": "Crawl known files", "enum": [ "none", "all", "robotstxt", "sitemapxml" ], "type": "string", "description": "Crawl known files such as robots.txt and sitemap.xml (requires depth >= 3).", "default": "none" }, "automaticFormFill": { "title": "Automatic form fill", "type": "boolean", "description": "Automatically fill and submit forms while crawling. Experimental (Katana -automatic-form-fill).", "default": false }, "formExtraction": { "title": "Extract forms", "type": "boolean", "description": "Extract form, input, textarea & select elements into output (Katana -form-extraction).", "default": false }, "techDetect": { "title": "Technology detection", "type": "boolean", "description": "Enable technology detection on crawled responses (Katana -tech-detect).", "default": false }, "knowledgeBase": { "title": "Knowledge base classification", "type": "boolean", "description": "Enable knowledge base classification (Katana -knowledge-base).", "default": false }, "pathClimb": { "title": "Path climb", "type": "boolean", "description": "Auto-crawl parent paths of discovered URLs (Katana -path-climb).", "default": false }, "ignoreQueryParams": { "title": "Ignore query params", "type": "boolean", "description": "Ignore crawling the same path with different query-param values (Katana -ignore-query-params).", "default": false }, "filterSimilar": { "title": "Filter similar URLs", "type": "boolean", "description": "Filter crawling of similar-looking URLs, e.g. /users/123 and /users/456 (Katana -filter-similar).", "default": false }, "filterSimilarThreshold": { "title": "Similar filter threshold", "minimum": 0, "type": "integer", "description": "Distinct values before a path position is treated as a parameter (Katana -filter-similar-threshold). 0 = default.", "default": 0 }, "disableRedirects": { "title": "Disable redirects", "type": "boolean", "description": "Do not follow HTTP redirects (Katana -disable-redirects).", "default": false }, "tlsImpersonate": { "title": "TLS impersonation (JA3)", "type": "boolean", "description": "Enable experimental client-hello (JA3) TLS randomization (Katana -tls-impersonate).", "default": false }, "retry": { "title": "Retries", "minimum": 0, "type": "integer", "description": "Number of times to retry a failed request (Katana -retry). 0 = Katana default.", "default": 0 }, "timeout": { "title": "Request timeout (seconds)", "minimum": 1, "type": "integer", "description": "Time to wait for a request in seconds (Katana -timeout).", "default": 10 }, "timeStable": { "title": "Page stable wait (seconds)", "minimum": 0, "type": "integer", "description": "Time to wait until the page is stable, headless mode (Katana -time-stable). 0 = default.", "default": 0 }, "maxResponseSize": { "title": "Max response size (bytes)", "minimum": 0, "type": "integer", "description": "Maximum response size to read (Katana -max-response-size). 0 = Katana default (4 MB).", "default": 0 }, "headers": { "title": "Custom headers", "type": "array", "description": "Custom headers/cookies for every request, in 'Name: value' format (Katana -headers).", "items": { "type": "string" } }, "resolvers": { "title": "Custom DNS resolvers", "type": "array", "description": "Custom DNS resolvers, e.g. 1.1.1.1, 8.8.8.8 (Katana -resolvers).", "items": { "type": "string" } }, "headless": { "title": "Headless mode", "type": "boolean", "description": "Enable headless Chromium crawling. Slower but handles JS-heavy sites (Katana -headless).", "default": false }, "hybrid": { "title": "Hybrid crawling", "type": "boolean", "description": "Enable headless hybrid crawling combining standard and headless (Katana -hybrid).", "default": false }, "xhrExtraction": { "title": "XHR extraction", "type": "boolean", "description": "Extract XHR request URL & method into output (Katana -xhr-extraction).", "default": false }, "noIncognito": { "title": "No incognito", "type": "boolean", "description": "Start headless Chrome without incognito mode (Katana -no-incognito).", "default": false }, "headlessOptions": { "title": "Headless Chrome options", "type": "array", "description": "Additional flags passed to headless Chrome (Katana -headless-options).", "items": { "type": "string" } }, "chromeWsUrl": { "title": "Remote Chrome WS URL", "type": "string", "description": "Connect to a Chrome instance launched elsewhere via its debugger WebSocket URL (Katana -chrome-ws-url)." }, "captchaSolverProvider": { "title": "Captcha solver provider", "type": "string", "description": "Captcha solver provider, e.g. capsolver (Katana -captcha-solver-provider)." }, "captchaSolverKey": { "title": "Captcha solver API key", "type": "string", "description": "API key for the captcha solver provider (Katana -captcha-solver-key)." }, "fieldScope": { "title": "Field scope", "type": "string", "description": "Pre-defined scope (dn, rdn, fqdn) or a custom regex (Katana -field-scope).", "default": "rdn" }, "crawlScope": { "title": "In-scope regexes", "type": "array", "description": "Regexes of URLs that are in scope and should be followed (Katana -crawl-scope).", "items": { "type": "string" } }, "crawlOutScope": { "title": "Out-of-scope regexes", "type": "array", "description": "Regexes of URLs that are out of scope and should not be followed (Katana -crawl-out-scope).", "items": { "type": "string" } }, "noScope": { "title": "Disable scope", "type": "boolean", "description": "Disable the host-based default scope, crawling everything reachable (Katana -no-scope).", "default": false }, "displayOutScope": { "title": "Display out-of-scope", "type": "boolean", "description": "Display external endpoints found during scoped crawling (Katana -display-out-scope).", "default": false }, "matchRegex": { "title": "Match output regexes", "type": "array", "description": "Only output URLs matching these regexes (Katana -match-regex).", "items": { "type": "string" } }, "filterRegex": { "title": "Filter output regexes", "type": "array", "description": "Drop output URLs matching these regexes (Katana -filter-regex).", "items": { "type": "string" } }, "extensionMatch": { "title": "Match extensions", "type": "array", "description": "Only output URLs with these extensions, e.g. php, html, js (Katana -extension-match).", "items": { "type": "string" } }, "extensionFilter": { "title": "Filter extensions", "type": "array", "description": "Drop output URLs with these extensions, e.g. png, css (Katana -extension-filter).", "items": { "type": "string" } }, "noDefaultExtFilter": { "title": "No default extension filter", "type": "boolean", "description": "Remove the built-in default extension filter list (Katana -no-default-ext-filter).", "default": false }, "matchCondition": { "title": "Match condition (DSL)", "type": "string", "description": "Match responses with a DSL-based condition (Katana -match-condition)." }, "filterCondition": { "title": "Filter condition (DSL)", "type": "string", "description": "Filter responses with a DSL-based condition (Katana -filter-condition)." }, "filterPageType": { "title": "Filter page types", "type": "array", "description": "Filter responses by page type: error, captcha, parked (Katana -filter-page-type).", "items": { "type": "string" } }, "disableUniqueFilter": { "title": "Disable unique filter", "type": "boolean", "description": "Disable duplicate-content filtering (Katana -disable-unique-filter).", "default": false }, "concurrency": { "title": "Concurrency", "minimum": 1, "type": "integer", "description": "Number of concurrent fetchers (Katana -concurrency).", "default": 10 }, "parallelism": { "title": "Parallelism", "minimum": 1, "type": "integer", "description": "Number of inputs processed in parallel (Katana -parallelism).", "default": 10 }, "rateLimit": { "title": "Rate limit (req/sec)", "minimum": 1, "type": "integer", "description": "Maximum requests to send per second (Katana -rate-limit).", "default": 150 }, "rateLimitMinute": { "title": "Rate limit (req/min)", "minimum": 0, "type": "integer", "description": "Maximum requests to send per minute (Katana -rate-limit-minute). 0 = disabled.", "default": 0 }, "delay": { "title": "Request delay (seconds)", "minimum": 0, "type": "integer", "description": "Delay between each request in seconds (Katana -delay). 0 = none.", "default": 0 }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Proxy to route Katana's requests through. Use Apify Proxy (datacenter or residential), or supply your own proxy URLs. Leave 'Use Apify Proxy' off and empty for no proxy.", "default": { "useApifyProxy": false } }, "proxyUrl": { "title": "Proxy URL (manual override)", "type": "string", "description": "HTTP/SOCKS5 proxy, e.g. http://user:pass@host:port (Katana -proxy). Takes precedence over Proxy configuration when set. Leave empty to use Proxy configuration instead." }, "flattenOutput": { "title": "Flatten output", "type": "boolean", "description": "Flatten each record's nested request/response objects into dotted top-level fields (e.g. response.status_code) so every field shows as its own dataset column. Disable to keep the raw nested Katana JSON.", "default": true }, "omitBody": { "title": "Omit response body", "type": "boolean", "description": "Omit the response body from JSONL output to reduce dataset size (Katana -omit-body).", "default": true }, "omitRaw": { "title": "Omit raw request/response", "type": "boolean", "description": "Omit raw request/response dumps from JSONL output (Katana -omit-raw).", "default": true }, "storeField": { "title": "Store field", "type": "string", "description": "Additional per-host field to store, e.g. url,path,fqdn,rdn (Katana -store-field)." }, "excludeOutputFields": { "title": "Exclude output fields", "type": "array", "description": "Fields to exclude from the JSONL output records (Katana -exclude-output-fields).", "items": { "type": "string" } }, "verbose": { "title": "Verbose logs", "type": "boolean", "description": "Enable verbose Katana logging (Katana -verbose). Logs go to the run log, not the dataset.", "default": false }, "extraArgs": { "title": "Extra Katana arguments", "type": "array", "description": "Additional raw Katana CLI flags for anything not exposed above, passed verbatim, e.g. [\"-time-stable\", \"2\"].", "items": { "type": "string" } } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } }}OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for Katana Web Crawler (ProjectDiscovery) from the options below:
If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients: