Web Scraper Task

1import { ApifyClient } from 'apify-client';
2
3// Initialize the ApifyClient with your Apify API token
4const client = new ApifyClient({
5    token: '<YOUR_API_TOKEN>',
6});
7
8// Prepare Actor input
9const input = {
10    "runMode": "DEVELOPMENT",
11    "startUrls": [
12        {
13            "url": "https://crawlee.dev"
14        }
15    ],
16    "linkSelector": "a[href]",
17    "globs": [
18        {
19            "glob": "https://crawlee.dev/*/*"
20        }
21    ],
22    "pseudoUrls": [],
23    "pageFunction": // The function accepts a single argument: the "context" object.
24    // For a complete list of its properties and functions,
25    // see https://apify.com/apify/web-scraper#page-function 
26    async function pageFunction(context) {
27        // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!
28        // debugger; 
29    
30        // jQuery is handy for finding DOM elements and extracting data from them.
31        // To use it, make sure to enable the "Inject jQuery" option.
32        const $ = context.jQuery;
33        const pageTitle = $('title').first().text();
34        const h1 = $('h1').first().text();
35        const first_h2 = $('h2').first().text();
36        const random_text_from_the_page = $('p').first().text();
37    
38    
39        // Print some information to actor log
40        context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);
41    
42        // Manually add a new page to the queue for scraping.
43       await context.enqueueRequest({ url: 'http://www.example.com' });
44    
45        // Return an object with the data extracted from the page.
46        // It will be stored to the resulting dataset.
47        return {
48            url: context.request.url,
49            pageTitle,
50            h1,
51            first_h2,
52            random_text_from_the_page
53        };
54    },
55    "proxyConfiguration": {
56        "useApifyProxy": true
57    },
58    "initialCookies": [],
59    "waitUntil": [
60        "networkidle2"
61    ],
62    "preNavigationHooks": `// We need to return array of (possibly async) functions here.
63        // The functions accept two arguments: the "crawlingContext" object
64        // and "gotoOptions".
65        [
66            async (crawlingContext, gotoOptions) => {
67                // ...
68            },
69        ]`,
70    "postNavigationHooks": `// We need to return array of (possibly async) functions here.
71        // The functions accept a single argument: the "crawlingContext" object.
72        [
73            async (crawlingContext) => {
74                // ...
75            },
76        ]`,
77    "breakpointLocation": "NONE",
78    "customData": {}
79};
80
81(async () => {
82    // Run the Actor and wait for it to finish
83    const run = await client.actor("undrtkr984/web-scraper-task").call(input);
84
85    // Fetch and print Actor results from the run's dataset (if any)
86    console.log('Results from dataset');
87    const { items } = await client.dataset(run.defaultDatasetId).listItems();
88    items.forEach((item) => {
89        console.dir(item);
90    });
91})();