Crawl4AI
Pricing
Pay per usage
Wraps the Crawl4AI open-source library for retrieving text content from websites.
Pricing
Pay per usage
Rating
3.3
(2)
Developer
Jan Buchar
Maintained by CommunityActor stats
16
Bookmarked
797
Total users
4
Monthly active users
36 days
Issues response
23 days ago
Last modified
Categories
Share
You can access the Crawl4AI programmatically from your own applications by using the Apify API. You can also choose the language preference from below. To use the Apify API, youโll need an Apify account and your API token, found in API & Integrations in Apify Console.
{ "openapi": "3.0.1", "info": { "version": "0.1", "x-build-id": "KXEpqxU5kj4HFUbdk" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/janbuchar~crawl4ai/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-janbuchar-crawl4ai", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/janbuchar~crawl4ai/runs": { "post": { "operationId": "runs-sync-janbuchar-crawl4ai", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/janbuchar~crawl4ai/run-sync": { "post": { "operationId": "run-sync-janbuchar-crawl4ai", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "URLs to start with", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "type": "string", "title": "URL of a web page", "format": "uri" } } } }, "maxCrawlDepth": { "title": "Maximum crawl depth", "minimum": 0, "type": "integer", "description": "How many levels of links should the crawler follow. The Start URLs have a depth of 0, all links found there have a depth of 1, and so on.", "default": 2 }, "maxPages": { "title": "Maximum pages", "minimum": 1, "type": "integer", "description": "Stop the crawl after this many pages have been crawled successfully. Leave empty for no limit." }, "crawlStrategy": { "title": "Crawl strategy", "enum": [ "bfs", "dfs", "bestFirst" ], "type": "string", "description": "How the crawler traverses the link graph. `bestFirst` visits the most promising URLs first and requires **Keywords** to be useful.", "default": "bfs" }, "keywords": { "title": "Keywords", "type": "array", "description": "Used by the `bestFirst` strategy to score URLs. URLs containing these words are crawled sooner.", "items": { "type": "string" } }, "includeExternalLinks": { "title": "Follow external links", "type": "boolean", "description": "Follow links pointing outside of the domain of the page they were found on. Combine with **Allowed domains** to keep the crawl bounded.", "default": false }, "includeUrlGlobs": { "title": "Include URL globs", "type": "array", "description": "Only follow links matching at least one of these glob patterns, e.g. `*/docs/*`. Start URLs are always crawled.", "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "Exclude URL globs", "type": "array", "description": "Never follow links matching any of these glob patterns.", "items": { "type": "string" } }, "allowedDomains": { "title": "Allowed domains", "type": "array", "description": "Only follow links on these domains. Leave empty to allow all.", "items": { "type": "string" } }, "blockedDomains": { "title": "Blocked domains", "type": "array", "description": "Never follow links on these domains.", "items": { "type": "string" } }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Skip URLs disallowed by the target website's robots.txt.", "default": false }, "proxyConfiguration": { "title": "Proxy configuration", "type": "object", "description": "Select proxies to be used by your crawler." }, "maxConcurrency": { "title": "Maximum concurrency", "minimum": 1, "type": "integer", "description": "Upper limit on how many pages are crawled in parallel. The crawler backs off below this on its own when the run approaches its memory allocation, so treat it as a ceiling rather than a target.", "default": 5 }, "maxRetries": { "title": "Maximum retries", "minimum": 0, "type": "integer", "description": "How many times to retry a page that failed to load.", "default": 2 }, "pageTimeoutSecs": { "title": "Page timeout", "minimum": 1, "type": "integer", "description": "How long to wait for a page to load before giving up, in seconds.", "default": 60 }, "waitUntil": { "title": "Wait until", "enum": [ "domcontentloaded", "load", "networkidle", "commit" ], "type": "string", "description": "Which page lifecycle event to consider as \"loaded\".", "default": "domcontentloaded" }, "magicMode": { "title": "Use magic mode", "type": "boolean", "description": "Masks browser automation signals, simulates human-like behavior, overrides navigator properties, handles cookie consent popups, manages browser fingerprinting and randomizes timing patterns.", "default": true }, "scanFullPage": { "title": "Scroll through the whole page", "type": "boolean", "description": "Scroll to the bottom of the page before extracting content. Necessary for lazy-loaded and infinite-scroll pages.", "default": false }, "cssSelector": { "title": "CSS selector", "type": "string", "description": "Ensures only a part of the website is processed" }, "excludedTags": { "title": "Excluded tags", "type": "array", "description": "HTML tags to strip before extracting content, e.g. `nav`, `footer`, `aside`.", "items": { "type": "string" } }, "markdownContentFilter": { "title": "Markdown content filter", "enum": [ "none", "pruning", "bm25" ], "type": "string", "description": "Produces an additional pruned `fit_markdown` alongside the raw one. `pruning` drops low-density boilerplate, `bm25` keeps content relevant to a query.", "default": "none" }, "markdownContentFilterQuery": { "title": "Markdown content filter query", "type": "string", "description": "Query used by the BM25 markdown content filter." }, "waitFor": { "title": "Wait for", "type": "string", "description": "CSS selector or JS function to wait for before processing a page" }, "jsCode": { "title": "Javascript code to execute on each page", "type": "string", "description": "Function executed for each request" }, "saveScreenshots": { "title": "Save screenshots", "type": "boolean", "description": "Should the crawler save screenshots?", "default": false }, "savePdfs": { "title": "Save PDFs", "type": "boolean", "description": "Render each page to a PDF and store it in the key-value store.", "default": false }, "extractionStrategy": { "title": "Extraction strategy", "enum": [ "llm", "jsonCss", "cosine" ], "type": "string", "description": "Specifies how structured data is extracted from each page" }, "llmProvider": { "title": "LLM provider", "type": "string", "description": "LiteLLM-style model identifier of the LLM used to extract data, e.g. `openai/gpt-4o` or `anthropic/claude-sonnet-4-5`." }, "llmApiToken": { "title": "API token", "type": "string", "description": "Token used for authentication with the LLM provider" }, "llmBaseUrl": { "title": "Base URL", "type": "string", "description": "Custom API endpoint of the LLM provider." }, "llmInstruction": { "title": "Instruction", "type": "string", "description": "Instructions (prompt) for the LLM" }, "llmSchema": { "title": "Schema", "type": "string", "description": "JSON schema describing the shape of the data the LLM should return. Leave empty to let the LLM decide." }, "jsonCssSchema": { "title": "JsonCss schema", "type": "string", "description": "Description of the data to be extracted, see https://docs.crawl4ai.com/extraction/no-llm-strategies/" }, "cosineSemanticFilter": { "title": "Semantic filter", "type": "string", "description": "**Removed.** The cosine extraction strategy required PyTorch and is no longer supported. Use the LLM or JsonCSS strategy instead." }, "cosineWordCountThreshold": { "title": "Word count threshold", "type": "integer", "description": "**Removed.** See above." }, "cosineLinkageMethod": { "title": "Linkage method", "type": "string", "description": "**Removed.** See above." }, "cosineTopK": { "title": "Top K", "type": "integer", "description": "**Removed.** See above." }, "cosineModelName": { "title": "Model name", "type": "string", "description": "**Removed.** See above." }, "debugLog": { "title": "Debug logging", "type": "boolean", "description": "Print debug-level log messages", "default": false } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } }}OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for Crawl4AI from the options below:
If youโd like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients: