Web Scraper

1import { ApifyClient } from 'apify-client';
2
3// Initialize the ApifyClient with your Apify API token
4// Replace the '<YOUR_API_TOKEN>' with your token
5const client = new ApifyClient({
6    token: '<YOUR_API_TOKEN>',
7});
8
9// Prepare Actor input
10const input = {
11    "runMode": "DEVELOPMENT",
12    "startUrls": [
13        {
14            "url": "https://crawlee.dev"
15        }
16    ],
17    "linkSelector": "a[href]",
18    "globs": [
19        {
20            "glob": "https://crawlee.dev/*/*"
21        }
22    ],
23    "pseudoUrls": [],
24    "excludes": [
25        {
26            "glob": "/**/*.{png,jpg,jpeg,pdf}"
27        }
28    ],
29    "pageFunction": `// The function accepts a single argument: the "context" object.
30        // For a complete list of its properties and functions,
31        // see https://apify.com/apify/web-scraper#page-function 
32        async function pageFunction(context) {
33            // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!
34            // debugger; 
35        
36            // jQuery is handy for finding DOM elements and extracting data from them.
37            // To use it, make sure to enable the "Inject jQuery" option.
38            const $ = context.jQuery;
39            const pageTitle = $('title').first().text();
40            const h1 = $('h1').first().text();
41            const first_h2 = $('h2').first().text();
42            const random_text_from_the_page = $('p').first().text();
43        
44        
45            // Print some information to actor log
46            context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);
47        
48            // Manually add a new page to the queue for scraping.
49           await context.enqueueRequest({ url: 'http://www.example.com' });
50        
51            // Return an object with the data extracted from the page.
52            // It will be stored to the resulting dataset.
53            return {
54                url: context.request.url,
55                pageTitle,
56                h1,
57                first_h2,
58                random_text_from_the_page
59            };
60        }`,
61    "proxyConfiguration": {
62        "useApifyProxy": true
63    },
64    "initialCookies": [],
65    "waitUntil": [
66        "networkidle2"
67    ],
68    "preNavigationHooks": `// We need to return array of (possibly async) functions here.
69        // The functions accept two arguments: the "crawlingContext" object
70        // and "gotoOptions".
71        [
72            async (crawlingContext, gotoOptions) => {
73                // ...
74            },
75        ]`,
76    "postNavigationHooks": `// We need to return array of (possibly async) functions here.
77        // The functions accept a single argument: the "crawlingContext" object.
78        [
79            async (crawlingContext) => {
80                // ...
81            },
82        ]`,
83    "breakpointLocation": "NONE",
84    "customData": {}
85};
86
87// Run the Actor and wait for it to finish
88const run = await client.actor("apify/web-scraper").call(input);
89
90// Fetch and print Actor results from the run's dataset (if any)
91console.log('Results from dataset');
92console.log(`💾 Check your data here: https://console.apify.com/storage/datasets/${run.defaultDatasetId}`);
93const { items } = await client.dataset(run.defaultDatasetId).listItems();
94items.forEach((item) => {
95    console.dir(item);
96});
97
98// 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/js/docs