Intelligent Website Crawler
Under maintenancePricing
Pay per usage
Go to Apify Store
Intelligent Website Crawler
Under maintenancePricing
Pay per usage
You can access the Intelligent Website Crawler programmatically from your own applications by using the Apify API. You can also choose the language preference from below. To use the Apify API, you’ll need an Apify account and your API token, found in API & Integrations in Apify Console.
{ "openapi": "3.0.1", "info": { "version": "1.0", "x-build-id": "k0ivH0bXsxwUwu6Tq" }, "servers": [ { "url": "https://api.apify.com/v2" } ], "paths": { "/acts/happitap~intelligent-website-crawler/run-sync-get-dataset-items": { "post": { "operationId": "run-sync-get-dataset-items-happitap-intelligent-website-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for its completion, and returns Actor's dataset items in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } }, "/acts/happitap~intelligent-website-crawler/runs": { "post": { "operationId": "runs-sync-happitap-intelligent-website-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor and returns information about the initiated run in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK", "content": { "application/json": { "schema": { "$ref": "#/components/schemas/runsResponseSchema" } } } } } } }, "/acts/happitap~intelligent-website-crawler/run-sync": { "post": { "operationId": "run-sync-happitap-intelligent-website-crawler", "x-openai-isConsequential": false, "summary": "Executes an Actor, waits for completion, and returns the OUTPUT from Key-value store in response.", "tags": [ "Run Actor" ], "requestBody": { "required": true, "content": { "application/json": { "schema": { "$ref": "#/components/schemas/inputSchema" } } } }, "parameters": [ { "name": "token", "in": "query", "required": true, "schema": { "type": "string" }, "description": "Enter your Apify token here" } ], "responses": { "200": { "description": "OK" } } } } }, "components": { "schemas": { "inputSchema": { "type": "object", "required": [ "startUrls" ], "properties": { "startUrls": { "title": "Start URLs", "type": "array", "description": "Array of URLs to scrape. Each URL should be an object with a 'url' property and optional 'method'.", "items": { "type": "string" } }, "crawlerType": { "title": "Crawler Type", "enum": [ "playwright:adaptive", "playwright:firefox", "playwright:chrome", "puppeteer" ], "type": "string", "description": "Type of crawler to use", "default": "playwright:adaptive" }, "maxCrawlPages": { "title": "Max Pages to Crawl", "minimum": 1, "maximum": 999999, "type": "integer", "description": "Maximum number of pages to crawl", "default": 100 }, "maxCrawlDepth": { "title": "Max Crawl Depth", "minimum": 0, "maximum": 99, "type": "integer", "description": "Maximum depth to crawl from start URLs", "default": 3 }, "maxConcurrency": { "title": "Max Concurrency", "minimum": 1, "maximum": 200, "type": "integer", "description": "Maximum concurrent browser instances. Higher values = faster crawling but more memory usage. Pattern learning is thread-safe for high concurrency.", "default": 50 }, "maxRequestRetries": { "title": "Max Request Retries", "minimum": 0, "maximum": 10, "type": "integer", "description": "Maximum number of retries for failed requests", "default": 3 }, "requestTimeoutSecs": { "title": "Request Timeout (seconds)", "minimum": 1, "maximum": 300, "type": "integer", "description": "Timeout for each request in seconds", "default": 60 }, "dynamicContentWaitSecs": { "title": "Dynamic Content Wait (seconds)", "minimum": 0, "maximum": 60, "type": "integer", "description": "Time to wait for dynamic content to load", "default": 10 }, "includeUrlGlobs": { "title": "Include URL Globs", "type": "array", "description": "Glob patterns for URLs to include", "default": [], "items": { "type": "string" } }, "excludeUrlGlobs": { "title": "Exclude URL Globs", "type": "array", "description": "Glob patterns for URLs to exclude", "default": [], "items": { "type": "string" } }, "useSitemaps": { "title": "Use Sitemaps", "type": "boolean", "description": "Whether to automatically discover and use sitemaps", "default": true }, "aggressivePrune": { "title": "Aggressive Prune", "type": "boolean", "description": "More aggressively prune content to reduce token usage", "default": false }, "blockMedia": { "title": "Block Media", "type": "boolean", "description": "Block images, videos, and other media to speed up crawling", "default": true }, "saveMarkdown": { "title": "Save Markdown", "type": "boolean", "description": "Save page content as markdown", "default": true }, "saveHtml": { "title": "Save HTML", "type": "boolean", "description": "Save raw HTML content", "default": false }, "saveScreenshots": { "title": "Save Screenshots", "type": "boolean", "description": "Take and save screenshots of pages", "default": false }, "removeElementsCssSelector": { "title": "Remove Elements CSS Selector", "type": "string", "description": "CSS selector for elements to remove before processing", "default": "nav, footer, script, style, noscript, svg, img[src^='data:'], [role=\"alert\"], [role=\"banner\"], [role=\"dialog\"], [role=\"alertdialog\"], [role=\"region\"][aria-label*=\"skip\" i], [aria-modal=\"true\"]" }, "clickElementsCssSelector": { "title": "Click Elements CSS Selector", "type": "string", "description": "CSS selector for elements to click (e.g., expand buttons)", "default": "[aria-expanded=\"false\"]" }, "waitForSelector": { "title": "Wait for Selector", "type": "string", "description": "CSS selector to wait for before processing page", "default": "" }, "maxScrollHeightPixels": { "title": "Max Scroll Height (pixels)", "minimum": 0, "maximum": 50000, "type": "integer", "description": "Maximum height to scroll for infinite scroll pages", "default": 5000 }, "readableTextCharThreshold": { "title": "Readable Text Character Threshold", "minimum": 0, "maximum": 10000, "type": "integer", "description": "Minimum characters required for content to be considered readable", "default": 100 }, "removeCookieWarnings": { "title": "Remove Cookie Warnings", "type": "boolean", "description": "Automatically remove cookie consent banners", "default": true }, "respectRobotsTxtFile": { "title": "Respect robots.txt", "type": "boolean", "description": "Whether to respect robots.txt file", "default": true }, "proxyConfiguration": { "title": "Proxy Configuration", "type": "object", "description": "Proxy settings for requests" }, "enableAI": { "title": "Enable AI Processing", "type": "boolean", "description": "Enable AI-powered page classification, smart pattern learning, and intelligent content extraction. Highly recommended for structured data extraction.", "default": true }, "aiModel": { "title": "AI Model", "enum": [ "gpt-3.5-turbo", "gpt-4", "gpt-4-turbo" ], "type": "string", "description": "OpenAI model for AI processing. gpt-3.5-turbo is cost-effective for most use cases. Use gpt-4 for complex content extraction.", "default": "gpt-3.5-turbo" }, "enablePatternLearning": { "title": "Enable Smart Pattern Learning", "type": "boolean", "description": "Enable smart DOM pattern learning to dramatically reduce AI API costs (90-99% savings). Learns page structures once and reuses for similar pages. Essential for large sites.", "default": true }, "patternSimilarityThreshold": { "title": "Pattern Similarity Threshold", "minimum": 0, "maximum": 1, "type": "number", "description": "Threshold for considering pages similar (0-1)", "default": 0.8 }, "maxAICallsPerRun": { "title": "Max AI Calls Per Run", "minimum": 1, "maximum": 1000, "type": "integer", "description": "Maximum AI API calls allowed per run. With pattern learning, you can crawl thousands of pages with just 10-50 AI calls. Start conservative and increase as needed.", "default": 50 }, "taskType": { "title": "Primary Task Type", "enum": [ "auto", "Summarize content", "Extract products", "Extract services", "Extract FAQ content", "Extract contact information" ], "type": "string", "description": "Primary task for content processing (auto-detection recommended)", "default": "auto" }, "useSitemap": { "title": "Use Website Sitemap", "type": "boolean", "description": "Automatically discover pages from website sitemap.xml. Combines with Start URLs for comprehensive crawling.", "default": false }, "sitemapUrls": { "title": "Sitemap URLs", "type": "array", "description": "URLs of sitemaps to crawl", "items": { "type": "string" } }, "sitemapMaxPages": { "title": "Max Pages from Sitemap", "minimum": 1, "maximum": 50000, "type": "integer", "description": "Maximum number of pages to extract from sitemaps. Use to limit crawl scope for large sites.", "default": 1000 }, "sitemapFilterPatterns": { "title": "Sitemap URL Filter Patterns", "type": "array", "description": "Filter sitemap URLs by patterns (e.g., '/product/', '/category/'). Include patterns you want, exclude with '!' prefix.", "items": { "type": "string" }, "default": [] } } }, "runsResponseSchema": { "type": "object", "properties": { "data": { "type": "object", "properties": { "id": { "type": "string" }, "actId": { "type": "string" }, "userId": { "type": "string" }, "startedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "finishedAt": { "type": "string", "format": "date-time", "example": "2025-01-08T00:00:00.000Z" }, "status": { "type": "string", "example": "READY" }, "meta": { "type": "object", "properties": { "origin": { "type": "string", "example": "API" }, "userAgent": { "type": "string" } } }, "stats": { "type": "object", "properties": { "inputBodyLen": { "type": "integer", "example": 2000 }, "rebootCount": { "type": "integer", "example": 0 }, "restartCount": { "type": "integer", "example": 0 }, "resurrectCount": { "type": "integer", "example": 0 }, "computeUnits": { "type": "integer", "example": 0 } } }, "options": { "type": "object", "properties": { "build": { "type": "string", "example": "latest" }, "timeoutSecs": { "type": "integer", "example": 300 }, "memoryMbytes": { "type": "integer", "example": 1024 }, "diskMbytes": { "type": "integer", "example": 2048 } } }, "buildId": { "type": "string" }, "defaultKeyValueStoreId": { "type": "string" }, "defaultDatasetId": { "type": "string" }, "defaultRequestQueueId": { "type": "string" }, "buildNumber": { "type": "string", "example": "1.0.0" }, "containerUrl": { "type": "string" }, "usage": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "integer", "example": 1 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } }, "usageTotalUsd": { "type": "number", "example": 0.00005 }, "usageUsd": { "type": "object", "properties": { "ACTOR_COMPUTE_UNITS": { "type": "integer", "example": 0 }, "DATASET_READS": { "type": "integer", "example": 0 }, "DATASET_WRITES": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_READS": { "type": "integer", "example": 0 }, "KEY_VALUE_STORE_WRITES": { "type": "number", "example": 0.00005 }, "KEY_VALUE_STORE_LISTS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_READS": { "type": "integer", "example": 0 }, "REQUEST_QUEUE_WRITES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_INTERNAL_GBYTES": { "type": "integer", "example": 0 }, "DATA_TRANSFER_EXTERNAL_GBYTES": { "type": "integer", "example": 0 }, "PROXY_RESIDENTIAL_TRANSFER_GBYTES": { "type": "integer", "example": 0 }, "PROXY_SERPS": { "type": "integer", "example": 0 } } } } } } } } }}OpenAPI is a standard for designing and describing RESTful APIs, allowing developers to define API structure, endpoints, and data formats in a machine-readable way. It simplifies API development, integration, and documentation.
OpenAPI is effective when used with AI agents and GPTs by standardizing how these systems interact with various APIs, for reliable integrations and efficient communication.
By defining machine-readable API specifications, OpenAPI allows AI models like GPTs to understand and use varied data sources, improving accuracy. This accelerates development, reduces errors, and provides context-aware responses, making OpenAPI a core component for AI applications.
You can download the OpenAPI definitions for Intelligent Website Crawler from the options below:
If you’d like to learn more about how OpenAPI powers GPTs, read our blog post.
You can also check out our other API clients: