JSDOM Scraper

1import { ApifyClient } from 'apify-client';
2
3// Initialize the ApifyClient with your Apify API token
4const client = new ApifyClient({
5    token: '<YOUR_API_TOKEN>',
6});
7
8// Prepare Actor input
9const input = {
10    "startUrls": [
11        {
12            "url": "https://crawlee.dev"
13        }
14    ],
15    "globs": [
16        {
17            "glob": "https://crawlee.dev/*/*"
18        }
19    ],
20    "pseudoUrls": [],
21    "excludes": [
22        {
23            "glob": "/**/*.{png,jpg,jpeg,pdf}"
24        }
25    ],
26    "linkSelector": "a[href]",
27    "pageFunction": async function pageFunction(context) {
28        const { window, request, log } = context;
29    
30        // The "window" property contains the JSDOM object which is useful
31        // for querying DOM elements and extracting data from them.
32        const pageTitle = window.document.title;
33    
34        // The "request" property contains various information about the web page loaded. 
35        const url = request.url;
36        
37        // Use "log" object to print information to actor log.
38        log.info('Page scraped', { url, pageTitle });
39    
40        // Return an object with the data extracted from the page.
41        // It will be stored to the resulting dataset.
42        return {
43            url,
44            pageTitle
45        };
46    },
47    "proxyConfiguration": {
48        "useApifyProxy": true
49    },
50    "initialCookies": [],
51    "additionalMimeTypes": [],
52    "preNavigationHooks": `// We need to return array of (possibly async) functions here.
53        // The functions accept two arguments: the "crawlingContext" object
54        // and "requestAsBrowserOptions" which are passed to the `requestAsBrowser()`
55        // function the crawler calls to navigate..
56        [
57            async (crawlingContext, requestAsBrowserOptions) => {
58                // ...
59            }
60        ]`,
61    "postNavigationHooks": `// We need to return array of (possibly async) functions here.
62        // The functions accept a single argument: the "crawlingContext" object.
63        [
64            async (crawlingContext) => {
65                // ...
66            },
67        ]`,
68    "customData": {}
69};
70
71(async () => {
72    // Run the Actor and wait for it to finish
73    const run = await client.actor("apify/jsdom-scraper").call(input);
74
75    // Fetch and print Actor results from the run's dataset (if any)
76    console.log('Results from dataset');
77    console.log(`💾 Check your data here: https://console.apify.com/storage/datasets/${run.defaultDatasetId}`);
78    const { items } = await client.dataset(run.defaultDatasetId).listItems();
79    items.forEach((item) => {
80        console.dir(item);
81    });
82})();
83
84// 📚 Want to learn more 📖? Go to → https://docs.apify.com/api/client/js/docs