Example Sitemap Cheerio

jancurn/example-sitemap-cheerio

No credit card required

An example actor that first downloads a sitemap in XML format and the crawls each page from the sitemap using the fast CheerioCrawler from Apify SDK.

Dockerfile

# This is a template for a Dockerfile used to run acts in Actor system.
# The base image name below is set during the act build, based on user settings.
# IMPORTANT: The base image must set a correct working directory, such as /usr/src/app or /home/user
FROM apify/actor-node

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Copy source code to container
# Do this in the last step, to have fast build if only the source code changed
COPY  . ./

# NOTE: The CMD is already defined by the base image.
# Uncomment this for local node inspector debugging:
# CMD [ "node", "--inspect=0.0.0.0:9229", "main.js" ]

main.js

const Apify = require('apify');
const cheerio = require('cheerio');

Apify.main(async () => {
    const input = await Apify.getInput();
    // Download sitemap
    const xml = await Apify.utils.requestAsBrowser({
        url: input?.url || 'http://beachwaver.com/sitemap_products_1.xml',
        headers: {
            'User-Agent': 'curl/7.54.0'
        }
    });
    
    // Parse sitemap and create RequestList from it
    const $ = cheerio.load(xml.toString());
    const sources = [];
    $('loc').each(function (val) {
        const url = $(this).text().trim();
        sources.push({
            url,
            headers: {
                // NOTE: Otherwise the target doesn't allow to download the page!
                'User-Agent': 'curl/7.54.0',
            }
        });
    });
    console.log(`Found ${sources.length} URLs in the sitemap`)
    const requestList = new Apify.RequestList({
        sources,
    });
    await requestList.initialize();
    
    // Crawl each page from sitemap
    const crawler = new Apify.CheerioCrawler({
        requestList,
        handlePageFunction: async ({ $, request }) => {
            console.log(`Processing ${request.url}...`);
            await Apify.pushData({
                url: request.url,
                title: $('title').text(),
            });
        },
    });

    await crawler.run();
    console.log('Done.');
});

package.json

{
    "name": "apify-project",
    "version": "0.0.1",
    "description": "",
    "author": "It's not you it's me",
    "license": "ISC",
    "dependencies": {
        "apify": "2.2.2",
        "cheerio": "latest"
    },
    "scripts": {
        "start": "node main.js"
    }
}