AI Web Scraper - Powered by Crawl4AI

1# Set API token
2API_TOKEN=<YOUR_API_TOKEN>
3
4# Prepare Actor input
5cat > input.json << 'EOF'
6{
7  "browserConfig": {
8    "browser_type": "chromium",
9    "headless": true,
10    "verbose_logging": false,
11    "ignore_https_errors": true,
12    "user_agent": "random",
13    "proxy": "",
14    "viewport_width": 1280,
15    "viewport_height": 720,
16    "accept_downloads": false,
17    "extra_headers": {}
18  },
19  "crawlerConfig": {
20    "cache_mode": "BYPASS",
21    "page_timeout": 20000,
22    "simulate_user": true,
23    "remove_overlay_elements": true,
24    "delay_before_return_html": 1,
25    "wait_for": "1",
26    "screenshot": false,
27    "pdf": false,
28    "enable_rate_limiting": false,
29    "semaphore_count": 10,
30    "memory_threshold_percent": 70,
31    "word_count_threshold": 200,
32    "css_selector": "",
33    "excluded_tags": [],
34    "excluded_selector": "",
35    "only_text": false,
36    "prettify": false,
37    "keep_data_attributes": false,
38    "remove_forms": false,
39    "bypass_cache": false,
40    "disable_cache": false,
41    "no_cache_read": false,
42    "no_cache_write": false,
43    "wait_until": "domcontentloaded",
44    "wait_for_images": false,
45    "check_robots_txt": false,
46    "mean_delay": 0.1,
47    "max_range": 0.3,
48    "js_code": "",
49    "js_only": false,
50    "ignore_body_visibility": true,
51    "scan_full_page": false,
52    "scroll_delay": 0.2,
53    "process_iframes": false,
54    "override_navigator": false,
55    "magic": false,
56    "adjust_viewport_to_content": false,
57    "screenshot_wait_for": 0,
58    "screenshot_height_threshold": 20000,
59    "image_description_min_word_threshold": 50,
60    "image_score_threshold": 3,
61    "exclude_external_images": false,
62    "exclude_social_media_domains": [],
63    "exclude_external_links": false,
64    "exclude_social_media_links": false,
65    "exclude_domains": [],
66    "verbose": true,
67    "log_console": false,
68    "stream": false
69  },
70  "deepCrawlConfig": {
71    "max_pages": 100,
72    "max_depth": 3,
73    "include_external": false,
74    "score_threshold": 0.5,
75    "filter_chain": [],
76    "url_scorer": {}
77  },
78  "markdownConfig": {
79    "ignore_links": false,
80    "ignore_images": false,
81    "escape_html": true,
82    "skip_internal_links": false,
83    "include_sup_sub": false,
84    "citations": false,
85    "body_width": 80,
86    "fit_markdown": false
87  },
88  "contentFilterConfig": {
89    "type": "pruning",
90    "user_query": "",
91    "threshold": 0.45,
92    "min_word_threshold": 5,
93    "bm25_threshold": 1.2,
94    "apply_llm_filter": false,
95    "semantic_filter": "",
96    "word_count_threshold": 10,
97    "sim_threshold": 0.3,
98    "max_dist": 0.2,
99    "top_k": 3,
100    "linkage_method": "ward"
101  },
102  "userAgentConfig": {
103    "user_agent_mode": "random",
104    "device_type": "desktop",
105    "browser_type": "chrome",
106    "num_browsers": 3
107  },
108  "llmConfig": {
109    "provider": "groq/deepseek-r1-distill-llama-70b",
110    "api_token": "",
111    "instruction": "Summarize content in clean markdown.",
112    "base_url": "",
113    "chunk_token_threshold": 2048,
114    "apply_chunking": true,
115    "input_format": "markdown",
116    "temperature": 0.7,
117    "max_tokens": 4096
118  },
119  "extractionSchema": {
120    "name": "Custom Extraction",
121    "baseSelector": "div.article",
122    "fields": [
123      {
124        "name": "title",
125        "selector": "h1",
126        "type": "text"
127      },
128      {
129        "name": "link",
130        "selector": "a",
131        "type": "attribute",
132        "attribute": "href"
133      }
134    ]
135  }
136}
137EOF
138
139# Run the Actor using an HTTP API
140# See the full API reference at https://docs.apify.com/api/v2
141curl "https://api.apify.com/v2/acts/raizen~ai-web-scraper/runs?token=$API_TOKEN" \
142  -X POST \
143  -d @input.json \
144  -H 'Content-Type: application/json'