Common Crawl Corpus Builder
Pricing
$43.00 / 1,000 report generateds
Go to Apify Store
Common Crawl Corpus Builder
Build targeted text corpora from Common Crawl with provenance.
Common Crawl Corpus Builder
Pricing
$43.00 / 1,000 report generateds
Build targeted text corpora from Common Crawl with provenance.
You can access the Common Crawl Corpus Builder programmatically from your own applications by using the Apify API. You can also choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in Integrations settings in Apify Console.
{ "openapi": "3.0.1", "info": { "version": "1.0", "x-build-id": "yp2dWtUmUkJ5ZdgZ8" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/zentrafoundry~common-crawl-targeted-corpus-builder/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-zentrafoundry-common-crawl-targeted-corpus-builder", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/zentrafoundry~common-crawl-targeted-corpus-builder/runs": { "post": { "operationId": "runs-sync-zentrafoundry-common-crawl-targeted-corpus-builder", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/zentrafoundry~common-crawl-targeted-corpus-builder/run-sync": { "post": { "operationId": "run-sync-zentrafoundry-common-crawl-targeted-corpus-builder", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "sourceMode" ], "properties": { "buyerQuery": { "title": "Regulatory Status Signal query", "type": "string", "description": "Buyer-safe query or scope for the records returned by this product.", "default": "buyer-ready records with source evidence" }, "taskIntent": { "title": "Regulatory Status Signal task intent", "type": "string", "description": "Stable product-specific purpose for this saved Apify task.", "default": "buyer-ready-product-run" }, "sourceMode": { "title": "Run sample or approved live sources", "enum": [ "sample", "startUrls" ], "type": "string", "description": "Sample emits public-safe Regulatory Status Signal rows. Approved live source mode keeps the same output fields and only uses owner-approved public URLs.", "default": "sample" }, "outputMode": { "title": "Regulatory Status Signal output mode", "enum": [ "sample-records", "buyer-ready-records" ], "type": "string", "description": "Use sample records for Apify Store QA or buyer-ready records for approved Regulatory Status Signal delivery.", "default": "sample-records" }, "startUrls": { "title": "Approved Regulatory Status Signal source URLs", "type": "array", "description": "Public URLs to use for live Regulatory Status Signal extraction after source-policy approval. Leave empty for sample mode.", "items": { "type": "object", "required": [ "url" ], "properties": { "url": { "title": "URL", "type": "string", "description": "Approved public Regulatory Status Signal source URL." } } }, "default": [] }, "maxItems": { "title": "Maximum Regulatory Status Signal rows", "minimum": 1, "maximum": 1000, "type": "integer", "description": "Caps the number of Regulatory Status Signal rows written to the dataset.", "default": 1 }, "perSourceLimit": { "title": "Maximum rows per source", "minimum": 1, "maximum": 100, "type": "integer", "description": "Caps validated rows from any one source before cross-source deduplication.", "default": 25 }, "maxTotalChargeUsd": { "title": "Maximum total charge (USD)", "minimum": 0.01, "maximum": 1000, "type": "number", "description": "Buyer-selected spend ceiling; the Apify run-level maximum remains authoritative.", "default": 5 }, "overallTimeoutSecs": { "title": "Overall timeout seconds", "minimum": 30, "maximum": 3600, "type": "integer", "description": "Stops additional source work once the bounded run deadline is reached.", "default": 900 }, "requestTimeoutSecs": { "title": "Source request timeout seconds", "minimum": 5, "maximum": 120, "type": "integer", "description": "Timeout applied independently to each approved source request.", "default": 30 }, "maxRequestRetries": { "title": "Maximum source retries", "minimum": 0, "maximum": 5, "type": "integer", "description": "Bounded retry count for transient source failures.", "default": 2 }, "sinceLastRun": { "title": "Only emit new logical records", "type": "boolean", "description": "Uses stable Actor state to skip logical records delivered by earlier runs.", "default": false }, "deltaMode": { "title": "Delta mode", "type": "boolean", "description": "Preserves stable deduplication keys for recurring tasks and schedules.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } }}OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for Common Crawl Corpus Builder from the options below:
If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients: