Metadata Extractor

jancurn/extract-metadata

No credit card required

A small efficient actor that loads a web page, parses its HTML using Cheerio library and extracts the following meta-data from the <HEAD> tag, such as page title, description, author etc.

Dockerfile

FROM apify/actor-node:16
COPY package*.json ./
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --only=prod --no-optional --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version
COPY . ./

INPUT_SCHEMA.json

{
    "title": "Schema for the jancurn/metadata-extractor actor",
    "type": "object",
    "schemaVersion": 1,
    "properties": {
        "urls": {
            "title": "Page URLs",
            "type": "array",
            "description": "URLs of the web pages to extract the metadata from. They must have either HTTP or HTTPS scheme.",
            "editor": "stringList",
            "prefill": ["https://www.apify.com/", "https://blog.apify.com"]
        },
        "proxy": {
            "title": "Proxy configuration",
            "type": "object",
            "description": "Select proxies to be used by your crawler.",
            "prefill": { "useApifyProxy": true },
            "editor": "proxy"
        }
    },
    "required": ["urls"]
}

main.js

const Apify = require('apify');

const { log } = Apify.utils;

Apify.main(async () => {
    const input = await Apify.getInput();
    const { urls = [], proxy = { useApifyProxy: false } } = input

    if (input.url) urls.push(input.url)

    const requests = [];
    for (const url of urls) {
        if (!new URL(url)) throw new Error('All URLs must be valid URLs!');
        requests.push({ url });
    }

    const requestList = await Apify.openRequestList('start-urls', requests);
    const proxyConfiguration = await Apify.createProxyConfiguration({ ...proxy });

    const crawler = new Apify.CheerioCrawler({
        requestList,
        proxyConfiguration,
        maxConcurrency: 50,
        handlePageFunction: async ({ $, request }) => {
            const meta = {};

            for (const tag of $('head meta')) {
                const name = $(tag).attr('name') || $(tag).attr('property') || $(tag).attr('http-equiv');
                const content = $(tag).attr('content');
                if (name) meta[name] = content ? content.trim() : null;
            }

            const result = {
                url: request.url,
                title: ($('head title').text() || '').trim(),
                meta,
            };

            return Apify.pushData(result);
        },
    });

    log.info('Starting the crawl...');
    await crawler.run();
    log.info('Scraping finished! Metadata for each site is available in "Results".');
});

package.json

{
	"name": "extract-metadata",
    "version": "0.0.2",
    "description": "Metadata extractor.",
	"dependencies": {
		"apify": "^2.0.7"
	},
    "scripts": {
        "start": "node main.js"
    },
    "author": "Jan Curn"
}