Web Scraper

1# Set API token
2API_TOKEN=<YOUR_API_TOKEN>
3
4# Prepare Actor input
5cat > input.json << 'EOF'
6{
7  "runMode": "DEVELOPMENT",
8  "startUrls": [
9    {
10      "url": "https://crawlee.dev"
11    }
12  ],
13  "linkSelector": "a[href]",
14  "globs": [
15    {
16      "glob": "https://crawlee.dev/*/*"
17    }
18  ],
19  "pseudoUrls": [],
20  "excludes": [
21    {
22      "glob": "/**/*.{png,jpg,jpeg,pdf}"
23    }
24  ],
25  "pageFunction": "// The function accepts a single argument: the \"context\" object.\n// For a complete list of its properties and functions,\n// see https://apify.com/apify/web-scraper#page-function \nasync function pageFunction(context) {\n    // This statement works as a breakpoint when you're trying to debug your code. Works only with Run mode: DEVELOPMENT!\n    // debugger; \n\n    // jQuery is handy for finding DOM elements and extracting data from them.\n    // To use it, make sure to enable the \"Inject jQuery\" option.\n    const $ = context.jQuery;\n    const pageTitle = $('title').first().text();\n    const h1 = $('h1').first().text();\n    const first_h2 = $('h2').first().text();\n    const random_text_from_the_page = $('p').first().text();\n\n\n    // Print some information to actor log\n    context.log.info(`URL: ${context.request.url}, TITLE: ${pageTitle}`);\n\n    // Manually add a new page to the queue for scraping.\n   await context.enqueueRequest({ url: 'http://www.example.com' });\n\n    // Return an object with the data extracted from the page.\n    // It will be stored to the resulting dataset.\n    return {\n        url: context.request.url,\n        pageTitle,\n        h1,\n        first_h2,\n        random_text_from_the_page\n    };\n}",
26  "proxyConfiguration": {
27    "useApifyProxy": true
28  },
29  "initialCookies": [],
30  "waitUntil": [
31    "networkidle2"
32  ],
33  "preNavigationHooks": "// We need to return array of (possibly async) functions here.\n// The functions accept two arguments: the \"crawlingContext\" object\n// and \"gotoOptions\".\n[\n    async (crawlingContext, gotoOptions) => {\n        // ...\n    },\n]\n",
34  "postNavigationHooks": "// We need to return array of (possibly async) functions here.\n// The functions accept a single argument: the \"crawlingContext\" object.\n[\n    async (crawlingContext) => {\n        // ...\n    },\n]",
35  "breakpointLocation": "NONE",
36  "customData": {}
37}
38EOF
39
40# Run the Actor using an HTTP API
41# See the full API reference at https://docs.apify.com/api/v2
42curl "https://api.apify.com/v2/acts/apify~web-scraper/runs?token=$API_TOKEN" \
43  -X POST \
44  -d @input.json \
45  -H 'Content-Type: application/json'