JSDOM Scraper

1import { ApifyClient } from 'apify-client';
2
3// Initialize the ApifyClient with your Apify API token
4// Replace the '<YOUR_API_TOKEN>' with your token
5const client = new ApifyClient({
6    token: '<YOUR_API_TOKEN>',
7});
8
9// Prepare Actor input
10const input = {
11    "startUrls": [
12        {
13            "url": "https://crawlee.dev/js"
14        }
15    ],
16    "respectRobotsTxtFile": true,
17    "globs": [
18        {
19            "glob": "https://crawlee.dev/js/*/*"
20        }
21    ],
22    "pseudoUrls": [],
23    "excludes": [
24        {
25            "glob": "/**/*.{png,jpg,jpeg,pdf}"
26        }
27    ],
28    "linkSelector": "a[href]",
29    "pageFunction": async function pageFunction(context) {
30        const { window, request, log } = context;
31    
32        // The "window" property contains the JSDOM object which is useful
33        // for querying DOM elements and extracting data from them.
34        const pageTitle = window.document.title;
35    
36        // The "request" property contains various information about the web page loaded. 
37        const url = request.url;
38        
39        // Use "log" object to print information to Actor log.
40        log.info('Page scraped', { url, pageTitle });
41    
42        // Return an object with the data extracted from the page.
43        // It will be stored to the resulting dataset.
44        return {
45            url,
46            pageTitle
47        };
48    },
49    "proxyConfiguration": {
50        "useApifyProxy": true
51    },
52    "initialCookies": [],
53    "additionalMimeTypes": [],
54    "preNavigationHooks": `// We need to return array of (possibly async) functions here.
55        // The functions accept two arguments: the "crawlingContext" object
56        // and "requestAsBrowserOptions" which are passed to the `requestAsBrowser()`
57        // function the crawler calls to navigate..
58        [
59            async (crawlingContext, requestAsBrowserOptions) => {
60                // ...
61            }
62        ]`,
63    "postNavigationHooks": `// We need to return array of (possibly async) functions here.
64        // The functions accept a single argument: the "crawlingContext" object.
65        [
66            async (crawlingContext) => {
67                // ...
68            },
69        ]`,
70    "customData": {}
71};
72
73// Run the Actor and wait for it to finish
74const run = await client.actor("apify/jsdom-scraper").call(input);
75
76// Fetch and print Actor results from the run's dataset (if any)
77console.log('Results from dataset');
78console.log(`💾 Check your data here: https://console.apify.com/storage/datasets/${run.defaultDatasetId}`);
79const { items } = await client.dataset(run.defaultDatasetId).listItems();
80items.forEach((item) => {
81    console.dir(item);
82});
83
84// 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/js/docs