Economist Category Scraper

1// This is the main Node.js source code file of your actor.
2// It is referenced from the "scripts" section of the package.json file.
3
4const Apify = require('apify');
5
6Apify.main(async () => {
7    // Get input of the actor. Input fields can be modified in INPUT_SCHEMA.json file.
8    // For more information, see https://apify.com/docs/actor/input-schema
9    const input = await Apify.getInput();
10    console.log('Input:');
11    console.dir(input);
12
13    // Here you can prepare your input for actor apify/web-scraper this input is based on a actor
14    // task you used as the starting point.
15    const metamorphInput = {
16        "startUrls": [
17            {
18                "url": `https://www.economist.com/${input.category}/?page=1`,
19                "method": "GET"
20            }
21        ],
22        "useRequestQueue": true,
23        "pseudoUrls": [
24            {
25                "purl": `https://www.economist.com/${input.category}/?page=[\\d+]`,
26                "method": "GET"
27            }
28        ],
29        "linkSelector": "a",
30        "pageFunction": async function pageFunction(context) {
31            // request is an instance of Apify.Request (https://sdk.apify.com/docs/api/request)
32            // $ is an instance of jQuery (http://jquery.com/)
33            const request = context.request;
34            const $ = context.jQuery;
35            const pageNum = parseInt(request.url.split('?page=').pop());
36        
37            context.log.info(`Scraping ${context.request.url}`);
38        
39            // Extract all articles.
40            const articles = [];
41            $('article').each((index, articleEl) => {
42                const $articleEl = $(articleEl);
43        
44                // H3 contains 2 child elements where first one is topic and second is article title.
45                const $h3El = $articleEl.find('h3');
46        
47                // Extract additonal info and push it to data object.
48                articles.push({
49                    pageNum,
50                    topic: $h3El.children().first().text(),
51                    title: $h3El.children().last().text(),
52                    url: $articleEl.find('a')[0].href,
53                    teaser: $articleEl.find('.teaser__text').text(),
54                });
55            });
56        
57            // Return results.
58            return articles;
59        },
60        "proxyConfiguration": {
61            "useApifyProxy": true
62        },
63        "debugLog": false,
64        "browserLog": false,
65        "injectJQuery": true,
66        "injectUnderscore": false,
67        "downloadMedia": false,
68        "downloadCss": false,
69        "ignoreSslErrors": false
70    };
71
72    // Now let's metamorph into actor apify/web-scraper using the created input.
73    await Apify.metamorph('apify/web-scraper', metamorphInput);
74});