AI Web Scraper - Powered by Crawl4AI

1echo '{
2  "browserConfig": {
3    "browser_type": "chromium",
4    "headless": true,
5    "verbose_logging": false,
6    "ignore_https_errors": true,
7    "user_agent": "random",
8    "proxy": "",
9    "viewport_width": 1280,
10    "viewport_height": 720,
11    "accept_downloads": false,
12    "extra_headers": {}
13  },
14  "crawlerConfig": {
15    "cache_mode": "BYPASS",
16    "page_timeout": 20000,
17    "simulate_user": true,
18    "remove_overlay_elements": true,
19    "delay_before_return_html": 1,
20    "wait_for": "1",
21    "screenshot": false,
22    "pdf": false,
23    "enable_rate_limiting": false,
24    "semaphore_count": 10,
25    "memory_threshold_percent": 70,
26    "word_count_threshold": 200,
27    "css_selector": "",
28    "excluded_tags": [],
29    "excluded_selector": "",
30    "only_text": false,
31    "prettify": false,
32    "keep_data_attributes": false,
33    "remove_forms": false,
34    "bypass_cache": false,
35    "disable_cache": false,
36    "no_cache_read": false,
37    "no_cache_write": false,
38    "wait_until": "domcontentloaded",
39    "wait_for_images": false,
40    "check_robots_txt": false,
41    "mean_delay": 0.1,
42    "max_range": 0.3,
43    "js_code": "",
44    "js_only": false,
45    "ignore_body_visibility": true,
46    "scan_full_page": false,
47    "scroll_delay": 0.2,
48    "process_iframes": false,
49    "override_navigator": false,
50    "magic": false,
51    "adjust_viewport_to_content": false,
52    "screenshot_wait_for": 0,
53    "screenshot_height_threshold": 20000,
54    "image_description_min_word_threshold": 50,
55    "image_score_threshold": 3,
56    "exclude_external_images": false,
57    "exclude_social_media_domains": [],
58    "exclude_external_links": false,
59    "exclude_social_media_links": false,
60    "exclude_domains": [],
61    "verbose": true,
62    "log_console": false,
63    "stream": false
64  },
65  "deepCrawlConfig": {
66    "max_pages": 100,
67    "max_depth": 3,
68    "include_external": false,
69    "score_threshold": 0.5,
70    "filter_chain": [],
71    "url_scorer": {}
72  },
73  "markdownConfig": {
74    "ignore_links": false,
75    "ignore_images": false,
76    "escape_html": true,
77    "skip_internal_links": false,
78    "include_sup_sub": false,
79    "citations": false,
80    "body_width": 80,
81    "fit_markdown": false
82  },
83  "contentFilterConfig": {
84    "type": "pruning",
85    "user_query": "",
86    "threshold": 0.45,
87    "min_word_threshold": 5,
88    "bm25_threshold": 1.2,
89    "apply_llm_filter": false,
90    "semantic_filter": "",
91    "word_count_threshold": 10,
92    "sim_threshold": 0.3,
93    "max_dist": 0.2,
94    "top_k": 3,
95    "linkage_method": "ward"
96  },
97  "userAgentConfig": {
98    "user_agent_mode": "random",
99    "device_type": "desktop",
100    "browser_type": "chrome",
101    "num_browsers": 3
102  },
103  "llmConfig": {
104    "provider": "groq/deepseek-r1-distill-llama-70b",
105    "api_token": "",
106    "instruction": "Summarize content in clean markdown.",
107    "base_url": "",
108    "chunk_token_threshold": 2048,
109    "apply_chunking": true,
110    "input_format": "markdown",
111    "temperature": 0.7,
112    "max_tokens": 4096
113  },
114  "extractionSchema": {
115    "name": "Custom Extraction",
116    "baseSelector": "div.article",
117    "fields": [
118      {
119        "name": "title",
120        "selector": "h1",
121        "type": "text"
122      },
123      {
124        "name": "link",
125        "selector": "a",
126        "type": "attribute",
127        "attribute": "href"
128      }
129    ]
130  }
131}' |
132apify call raizen/ai-web-scraper --silent --output-dataset