Web Scraper

1echo '{
2  "runMode": "DEVELOPMENT",
3  "startUrls": [
4    {
5      "url": "https://crawlee.dev"
6    }
7  ],
8  "linkSelector": "a[href]",
9  "globs": [
10    {
11      "glob": "https://crawlee.dev/*/*"
12    }
13  ],
14  "pseudoUrls": [],
15  "excludes": [
16    {
17      "glob": "/**/*.{png,jpg,jpeg,pdf}"
18    }
19  ],
20  "pageFunction": "// The function accepts a single argument: the \\"context\\" object.\\n// For a complete list of its properties and functions,\\n// see https://apify.com/apify/web-scraper#page-function \\nasync function pageFunction(context) {\\n    // This statement works as a breakpoint when you'\''re trying to debug your code. Works only with Run mode: DEVELOPMENT!\\n    // debugger; \\n\\n    // jQuery is handy for finding DOM elements and extracting data from them.\\n    // To use it, make sure to enable the \\"Inject jQuery\\" option.\\n    const $ = context.jQuery;\\n    const pageTitle = $('\''title'\'').first().text();\\n    const h1 = $('\''h1'\'').first().text();\\n    const first_h2 = $('\''h2'\'').first().text();\\n    const random_text_from_the_page = $('\''p'\'').first().text();\\n\\n\\n    // Print some information to actor log\\n    context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);\\n\\n    // Manually add a new page to the queue for scraping.\\n   await context.enqueueRequest({ url: '\''http://www.example.com'\'' });\\n\\n    // Return an object with the data extracted from the page.\\n    // It will be stored to the resulting dataset.\\n    return {\\n        url: context.request.url,\\n        pageTitle,\\n        h1,\\n        first_h2,\\n        random_text_from_the_page\\n    };\\n}",
21  "proxyConfiguration": {
22    "useApifyProxy": true
23  },
24  "initialCookies": [],
25  "waitUntil": [
26    "networkidle2"
27  ],
28  "preNavigationHooks": "// We need to return array of (possibly async) functions here.\\n// The functions accept two arguments: the \\"crawlingContext\\" object\\n// and \\"gotoOptions\\".\\n[\\n    async (crawlingContext, gotoOptions) => {\\n        // ...\\n    },\\n]\\n",
29  "postNavigationHooks": "// We need to return array of (possibly async) functions here.\\n// The functions accept a single argument: the \\"crawlingContext\\" object.\\n[\\n    async (crawlingContext) => {\\n        // ...\\n    },\\n]",
30  "breakpointLocation": "NONE",
31  "customData": {}
32}' |
33apify call apify/web-scraper --silent --output-dataset