AI Web Scraper - Powered by Crawl4AI

1import { ApifyClient } from 'apify-client';
2
3// Initialize the ApifyClient with your Apify API token
4// Replace the '<YOUR_API_TOKEN>' with your token
5const client = new ApifyClient({
6    token: '<YOUR_API_TOKEN>',
7});
8
9// Prepare Actor input
10const input = {
11    "browserConfig": {
12        "browser_type": "chromium",
13        "headless": true,
14        "verbose_logging": false,
15        "ignore_https_errors": true,
16        "user_agent": "random",
17        "proxy": "",
18        "viewport_width": 1280,
19        "viewport_height": 720,
20        "accept_downloads": false,
21        "extra_headers": {}
22    },
23    "crawlerConfig": {
24        "cache_mode": "BYPASS",
25        "page_timeout": 20000,
26        "simulate_user": true,
27        "remove_overlay_elements": true,
28        "delay_before_return_html": 1,
29        "wait_for": "1",
30        "screenshot": false,
31        "pdf": false,
32        "enable_rate_limiting": false,
33        "semaphore_count": 10,
34        "memory_threshold_percent": 70,
35        "word_count_threshold": 200,
36        "css_selector": "",
37        "excluded_tags": [],
38        "excluded_selector": "",
39        "only_text": false,
40        "prettify": false,
41        "keep_data_attributes": false,
42        "remove_forms": false,
43        "bypass_cache": false,
44        "disable_cache": false,
45        "no_cache_read": false,
46        "no_cache_write": false,
47        "wait_until": "domcontentloaded",
48        "wait_for_images": false,
49        "check_robots_txt": false,
50        "mean_delay": 0.1,
51        "max_range": 0.3,
52        "js_code": "",
53        "js_only": false,
54        "ignore_body_visibility": true,
55        "scan_full_page": false,
56        "scroll_delay": 0.2,
57        "process_iframes": false,
58        "override_navigator": false,
59        "magic": false,
60        "adjust_viewport_to_content": false,
61        "screenshot_wait_for": 0,
62        "screenshot_height_threshold": 20000,
63        "image_description_min_word_threshold": 50,
64        "image_score_threshold": 3,
65        "exclude_external_images": false,
66        "exclude_social_media_domains": [],
67        "exclude_external_links": false,
68        "exclude_social_media_links": false,
69        "exclude_domains": [],
70        "verbose": true,
71        "log_console": false,
72        "stream": false
73    },
74    "deepCrawlConfig": {
75        "max_pages": 100,
76        "max_depth": 3,
77        "include_external": false,
78        "score_threshold": 0.5,
79        "filter_chain": [],
80        "url_scorer": {}
81    },
82    "markdownConfig": {
83        "ignore_links": false,
84        "ignore_images": false,
85        "escape_html": true,
86        "skip_internal_links": false,
87        "include_sup_sub": false,
88        "citations": false,
89        "body_width": 80,
90        "fit_markdown": false
91    },
92    "contentFilterConfig": {
93        "type": "pruning",
94        "user_query": "",
95        "threshold": 0.45,
96        "min_word_threshold": 5,
97        "bm25_threshold": 1.2,
98        "apply_llm_filter": false,
99        "semantic_filter": "",
100        "word_count_threshold": 10,
101        "sim_threshold": 0.3,
102        "max_dist": 0.2,
103        "top_k": 3,
104        "linkage_method": "ward"
105    },
106    "userAgentConfig": {
107        "user_agent_mode": "random",
108        "device_type": "desktop",
109        "browser_type": "chrome",
110        "num_browsers": 3
111    },
112    "llmConfig": {
113        "provider": "groq/deepseek-r1-distill-llama-70b",
114        "api_token": "",
115        "instruction": "Summarize content in clean markdown.",
116        "base_url": "",
117        "chunk_token_threshold": 2048,
118        "apply_chunking": true,
119        "input_format": "markdown",
120        "temperature": 0.7,
121        "max_tokens": 4096
122    },
123    "extractionSchema": {
124        "name": "Custom Extraction",
125        "baseSelector": "div.article",
126        "fields": [
127            {
128                "name": "title",
129                "selector": "h1",
130                "type": "text"
131            },
132            {
133                "name": "link",
134                "selector": "a",
135                "type": "attribute",
136                "attribute": "href"
137            }
138        ]
139    }
140};
141
142// Run the Actor and wait for it to finish
143const run = await client.actor("raizen/ai-web-scraper").call(input);
144
145// Fetch and print Actor results from the run's dataset (if any)
146console.log('Results from dataset');
147console.log(`💾 Check your data here: https://console.apify.com/storage/datasets/${run.defaultDatasetId}`);
148const { items } = await client.dataset(run.defaultDatasetId).listItems();
149items.forEach((item) => {
150    console.dir(item);
151});
152
153// 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/js/docs