AI Web Scraper - Powered by Crawl4AI

1# Set API token
2API_TOKEN=<YOUR_API_TOKEN>
3
4# Prepare Actor input
5cat > input.json << 'EOF'
6{
7  "startUrls": [
8    {
9      "url": "https://www.cnbc.com/2025/03/12/googles-deepmind-says-it-will-use-ai-models-to-power-physical-robots.html"
10    }
11  ],
12  "browserConfig": {
13    "browser_type": "chromium",
14    "headless": true,
15    "verbose_logging": false,
16    "ignore_https_errors": true,
17    "user_agent": "random",
18    "proxy": "",
19    "viewport_width": 1280,
20    "viewport_height": 720,
21    "accept_downloads": false,
22    "extra_headers": {}
23  },
24  "crawlerConfig": {
25    "cache_mode": "BYPASS",
26    "page_timeout": 20000,
27    "simulate_user": true,
28    "override_navigator": true,
29    "magic": true,
30    "remove_overlay_elements": true,
31    "delay_before_return_html": 0.75,
32    "wait_for": "",
33    "screenshot": false,
34    "pdf": false,
35    "enable_rate_limiting": false,
36    "memory_threshold_percent": 90,
37    "word_count_threshold": 200,
38    "css_selector": "",
39    "excluded_tags": [],
40    "excluded_selector": "",
41    "only_text": false,
42    "prettify": false,
43    "keep_data_attributes": false,
44    "remove_forms": false,
45    "bypass_cache": false,
46    "disable_cache": false,
47    "no_cache_read": false,
48    "no_cache_write": false,
49    "wait_until": "domcontentloaded",
50    "wait_for_images": false,
51    "check_robots_txt": false,
52    "mean_delay": 0.1,
53    "max_range": 0.3,
54    "js_code": "",
55    "js_only": false,
56    "ignore_body_visibility": true,
57    "scan_full_page": false,
58    "scroll_delay": 0.2,
59    "process_iframes": false,
60    "adjust_viewport_to_content": false,
61    "screenshot_wait_for": 0,
62    "screenshot_height_threshold": 20000,
63    "image_description_min_word_threshold": 50,
64    "image_score_threshold": 3,
65    "exclude_external_images": false,
66    "exclude_social_media_domains": [],
67    "exclude_external_links": false,
68    "exclude_social_media_links": false,
69    "exclude_domains": [],
70    "verbose": true,
71    "log_console": false,
72    "stream": false
73  },
74  "deepCrawlConfig": {
75    "max_pages": 100,
76    "max_depth": 3,
77    "include_external": false,
78    "score_threshold": 0.5,
79    "filter_chain": [],
80    "keywords": [
81      "crawl",
82      "example",
83      "async",
84      "configuration"
85    ],
86    "weight": 0.7
87  },
88  "markdownConfig": {
89    "ignore_links": false,
90    "ignore_images": false,
91    "escape_html": true,
92    "skip_internal_links": false,
93    "include_sup_sub": false,
94    "citations": false,
95    "body_width": 80,
96    "fit_markdown": false
97  },
98  "contentFilterConfig": {
99    "type": "pruning",
100    "user_query": "",
101    "threshold": 0.45,
102    "min_word_threshold": 5,
103    "bm25_threshold": 1.2,
104    "apply_llm_filter": false,
105    "semantic_filter": "",
106    "word_count_threshold": 10,
107    "sim_threshold": 0.3,
108    "max_dist": 0.2,
109    "top_k": 3,
110    "linkage_method": "ward"
111  },
112  "userAgentConfig": {
113    "user_agent_mode": "random",
114    "device_type": "desktop",
115    "browser_type": "chrome",
116    "num_browsers": 1
117  },
118  "llmConfig": {
119    "provider": "groq/deepseek-r1-distill-llama-70b",
120    "api_token": "",
121    "instruction": "Summarize content in clean markdown.",
122    "base_url": "",
123    "chunk_token_threshold": 2048,
124    "apply_chunking": true,
125    "input_format": "markdown",
126    "temperature": 0.7,
127    "max_tokens": 4096
128  },
129  "extractionSchema": {
130    "name": "Custom Extraction",
131    "baseSelector": "div.article",
132    "fields": [
133      {
134        "name": "title",
135        "selector": "h1",
136        "type": "text"
137      },
138      {
139        "name": "link",
140        "selector": "a",
141        "type": "attribute",
142        "attribute": "href"
143      }
144    ]
145  }
146}
147EOF
148
149# Run the Actor using an HTTP API
150# See the full API reference at https://docs.apify.com/api/v2
151curl "https://api.apify.com/v2/acts/raizen~ai-web-scraper/runs?token=$API_TOKEN" \
152  -X POST \
153  -d @input.json \
154  -H 'Content-Type: application/json'