Website to RAG Dataset
Pricing
from $1.50 / 1,000 processed pages
Convert public websites, docs, blogs, and XML sitemaps into clean Markdown, structured metadata, and stable chunks for RAG pipelines and vector databases.
You can access the Website to RAG Dataset programmatically from your own applications by using the Apify API. You can also choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in Integrations settings in Apify Console.
{ "openapi": "3.0.1", "info": { "version": "0.1", "x-build-id": "d10vubhUFofNsbdg6" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/sebastian-actors~website-to-rag-dataset/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-sebastian-actors-website-to-rag-dataset", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/sebastian-actors~website-to-rag-dataset/runs": { "post": { "operationId": "runs-sync-sebastian-actors-website-to-rag-dataset", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/sebastian-actors~website-to-rag-dataset/run-sync": { "post": { "operationId": "run-sync-sebastian-actors-website-to-rag-dataset", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "properties": { "urls": { "title": "URLs", "type": "array", "description": "List of public web pages to process. You can provide individual URLs, docs pages, blog posts, or landing pages.", "items": { "type": "string" }, "default": [] }, "sitemapUrl": { "title": "Sitemap URL", "type": "string", "description": "Optional XML sitemap or sitemap index URL. If provided, URLs from the sitemap are processed up to Max pages." }, "maxPages": { "title": "Max pages", "minimum": 1, "maximum": 10000, "type": "integer", "description": "Maximum number of pages to process.", "default": 100 }, "sameDomainOnly": { "title": "Same domain only", "type": "boolean", "description": "Only process URLs on the same hostname as the first URL or sitemap URL.", "default": true }, "respectRobotsTxt": { "title": "Respect robots.txt", "type": "boolean", "description": "Check robots.txt and skip URLs disallowed for the configured user agent.", "default": true }, "outputMode": { "title": "Output mode", "enum": [ "pages", "chunks", "pages_and_chunks" ], "type": "string", "description": "Choose whether to output page rows, chunk rows, or both.", "default": "pages_and_chunks" }, "includeMarkdown": { "title": "Include Markdown", "type": "boolean", "description": "Include cleaned Markdown in page output rows.", "default": true }, "includeSchema": { "title": "Include schema metadata", "type": "boolean", "description": "Extract JSON-LD schema types and key schema fields where available.", "default": true }, "includeFreshnessSignals": { "title": "Include freshness signals", "type": "boolean", "description": "Extract Last-Modified, sitemap lastmod, datePublished, and dateModified signals when available.", "default": true }, "includeLinks": { "title": "Include links", "type": "boolean", "description": "Include internal and external links found in the main content.", "default": false }, "includeHtml": { "title": "Include cleaned HTML", "type": "boolean", "description": "Include cleaned HTML in page rows. Disabled by default to keep output smaller.", "default": false }, "chunkSize": { "title": "Chunk size", "minimum": 200, "maximum": 4000, "type": "integer", "description": "Approximate chunk size in estimated tokens. Token estimate uses a simple characters/4 heuristic.", "default": 1200 }, "chunkOverlap": { "title": "Chunk overlap", "minimum": 0, "maximum": 1000, "type": "integer", "description": "Approximate overlap between chunks in estimated tokens.", "default": 150 }, "minContentChars": { "title": "Minimum content characters", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Pages with less cleaned content than this value are marked as low-content.", "default": 250 }, "maxConcurrency": { "title": "Max concurrency", "minimum": 1, "maximum": 25, "type": "integer", "description": "Number of pages fetched in parallel.", "default": 5 }, "requestTimeoutSecs": { "title": "Request timeout seconds", "minimum": 5, "maximum": 120, "type": "integer", "description": "Timeout for each page request.", "default": 20 }, "maxSitemaps": { "title": "Max sitemaps", "minimum": 1, "maximum": 1000, "type": "integer", "description": "Maximum number of sitemap and sitemap-index files to fetch.", "default": 100 }, "userAgent": { "title": "User agent", "type": "string", "description": "User agent used for requests and robots.txt checks.", "default": "WebsiteToRagDatasetBot/1.0 (+https://apify.com)" }, "urlAllowlistPatterns": { "title": "URL allowlist patterns", "type": "array", "description": "Optional regex patterns. If provided, only matching URLs are processed.", "items": { "type": "string" }, "default": [] }, "urlBlocklistPatterns": { "title": "URL blocklist patterns", "type": "array", "description": "Optional regex patterns. Matching URLs are skipped.", "items": { "type": "string" }, "default": [] }, "removeSelectors": { "title": "Remove CSS selectors", "type": "array", "description": "CSS selectors to remove before Markdown conversion.", "items": { "type": "string" }, "default": [ "nav", "footer", "header", "aside", "script", "style", "noscript", "iframe", "form" ] }, "onlySelectors": { "title": "Only CSS selectors", "type": "array", "description": "Optional CSS selectors for the content area. If provided, only matched elements are used.", "items": { "type": "string" }, "default": [] }, "dedupeByCanonical": { "title": "Dedupe by canonical URL", "type": "boolean", "description": "Avoid processing duplicate pages with the same canonical URL.", "default": true }, "saveRunSummaryToKeyValueStore": { "title": "Save run summary", "type": "boolean", "description": "Save run-level summary statistics to the default key-value store under OUTPUT.", "default": true } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } }}OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for Website to RAG Dataset from the options below:
If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients: