/* eslint-disable max-len */
/* eslint-disable no-shadow */
const Apify = require('apify');
const { gotScraping } = require('got-scraping');

const { utils: { log, sleep } } = Apify;

exports.handleStart = async ({ page, proxyInfo }, requestQueue) => {
    const { searchTerm } = await Apify.getInput();

    let pageNumber = 1;
    let response = null;
    const resp = await page.goto(`https://www.dnb.com/apps/dnb/servlets/CompanySearchServlet?pageNumber=${pageNumber}&pageSize=${global.PAGE_SIZE}&resourcePath=%2Fcontent%2Fdnb-us%2Fen%2Fhome%2Fsite-search-results%2Fjcr:content%2Fcontent-ipar-cta%2Fsinglepagesearch&returnNav=true&searchTerm=${searchTerm}`, {});
    const headers = resp.request().headers();

    await sleep(15000);

    const startUrls = [];
    do {
        response = await gotScraping({
            url: `https://www.dnb.com/apps/dnb/servlets/CompanySearchServlet?pageNumber=${pageNumber}&pageSize=${global.PAGE_SIZE}&resourcePath=%2Fcontent%2Fdnb-us%2Fen%2Fhome%2Fsite-search-results%2Fjcr:content%2Fcontent-ipar-cta%2Fsinglepagesearch&returnNav=true&searchTerm=${searchTerm}`,
            proxyUrl: proxyInfo.url,
            responseType: 'json',
            headers,
        });

        if (!response.body) {
            break;
        }

        let stopCrawling = false;
        const { totalMatchedResults, companies } = response.body;

        for (const company of companies) {
            const { urlSelector } = company;
            const detailUrl = `https://www.dnb.com/business-directory/company-profiles.${urlSelector}.html`;

            startUrls.push({
                url: detailUrl,
                userData: {
                    label: 'DETAIL',
                },
            });

            if (global.maxItems && startUrls.length >= global.maxItems) {
                log.info(`Max items reached: ${startUrls.length}`);
                stopCrawling = true;
                break;
            }
        }

        if (stopCrawling || pageNumber * global.PAGE_SIZE > totalMatchedResults) {
            break;
        }
        pageNumber += 1;
    } while (response.body);

    for (let index = 0; index < startUrls.length; index++) {
        await requestQueue.addRequest(startUrls[index]);
    }
};

exports.handleDetail = async ({ request, page }) => {
    await page.waitForFunction(() => document.querySelector('div > script[type="application/ld+json"]') !== null, { timeout: 60000 });
    await page.waitForFunction(() => document.querySelector('body > script[type="application/ld+json"]') !== null, {timeout: 60000 });

    let jsonText = await page.evaluate(() => {
        return document.querySelector('div > script[type="application/ld+json"]').textContent;
    });
    const { itemListElement } = JSON.parse(jsonText);
    const breadcrumb = [];
    for (const { item: { name } } of itemListElement) {
        breadcrumb.push(name);
    }

    jsonText = await page.evaluate(() => {
        return document.querySelector('body > script[type="application/ld+json"]').textContent;
    });
    const { name, description, url, numberofEmployees, telephone, employee,
        address: { addressLocality, addressRegion, postalCode, addressCountry, streetAddress } } = JSON.parse(jsonText);

    const principals = [];
    for (const emp of employee) {
        if (emp.name && emp.jobTitle) {
            principals.push({
                name: emp.name,
                position: emp.jobTitle,
            });
        }
    }

    const { industry, revenueText, yearStart, incorporated, type, role, fiscalYearEnd } = await page.evaluate(() => {
        // eslint-disable-next-line no-shadow
        const industry = [];
        const industryList = document.querySelectorAll('[name=industry_links] > span > span > a, [name=industry_links] > span > span > span');
        for (let index = 0; index < industryList.length; index++) {
            industry.push(industryList[index].textContent.trim().replace(/^,/, '')
                .trim());
        }

        const revenueText = document.querySelector('[name=revenue_in_us_dollar] > span') ? document.querySelector('[name=revenue_in_us_dollar] > span').textContent.trim() : null;

        const yearStart = document.querySelector('[name=year_started] > span') ? parseInt(document.querySelector('[name=year_started] > span').textContent.trim(), 10) : null;
        const incorporated = document.querySelector('[name=year_incorporated] > span') ? parseInt(document.querySelector('[name=year_incorporated] > span').textContent.trim(), 10) : null;

        const type = document.querySelector('.company-profile-comp.type') ? document.querySelector('.company-profile-comp.type').textContent.trim() : null;
        const role = document.querySelector('.company-profile-comp.role') ? document.querySelector('.company-profile-comp.role').textContent.trim() : null;

        const fiscalYearEnd = document.querySelector('[name=fiscal_year_end] > span') ? document.querySelector('[name=fiscal_year_end] > span').textContent.trim() : null;

        return {
            industry,
            revenueText,
            yearStart,
            incorporated,
            type,
            role,
            fiscalYearEnd,
        };
    })
    
    let revenue = null;
    if (!revenueText === false && revenueText.match(/[\d.,]+/)) {
        revenue = parseFloat(revenueText.match(/[\d.,]+/)[0].replace(/,/g, ''));
        if (revenueText.includes('million')) {
            revenue = Math.floor(revenue * 1000000);
        } else if (revenueText.includes('billion')) {
            revenue = Math.floor(revenue * 1000000000);
        }
    }

    const numberOfEmployees = numberofEmployees ? parseInt(numberofEmployees, 10) : null;

    function includes(str, arr) {
        const list = arr.split(',');
        for (const text of list) {
            if (str.includes(text.trim())) {
                return true;
            }
        }

        return false;
    }

    if (global.countryIn && (!addressCountry || !includes(addressCountry.toLowerCase(), global.countryIn.toLowerCase()))) {
        log.info(`Not match country criteria (${addressCountry} contains ${global.countryIn}): ${request.url}`);
        return;
    }

    if (global.industryIn && (!industry || !includes(industry.toString().toLowerCase(), global.industryIn.toLowerCase()))) {
        log.info(`Not match industry criteria (${industry.toString()} contains ${global.industryIn}): ${request.url}`);
        return;
    }

    if (global.revenueMin && (!revenue || revenue < global.revenueMin)) {
        log.info(`Not match revenue criteria (${revenue} >= ${global.revenueMin}): ${request.url}`);
        return;
    }

    if (global.numberOfEmployeesMin && (!numberOfEmployees || numberOfEmployees < global.numberOfEmployeesMin)) {
        log.info(`Not match number of employees criteria (${numberOfEmployees} >= ${global.numberOfEmployeesMin}): ${request.url}`);
        return;
    }

    if (global.yearStartFrom && (!yearStart || yearStart < global.yearStartFrom)) {
        log.info(`Not match year start criteria (${yearStart} >= ${global.yearStartFrom}): ${request.url}`);
        return;
    }

    await Apify.pushData({
        url: request.url,
        name,
        description,
        website: url,
        numberOfEmployees,
        telephone,
        addressLocality,
        addressCountry,
        addressRegion,
        streetAddress,
        postalCode,
        type,
        role,
        industry,
        principals,
        breadcrumb,
        revenue,
        fiscalYearEnd,
        yearStart,
        incorporated,
    });
};


src/routes.js

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

.eslintrc

# This file tells Git which files shouldn't be added to source control

.idea
node_modules


.gitignore

# First, specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-playwright-firefox:16

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --only=prod --no-optional --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Optionally, specify how to launch the source code of your actor.
# By default, Apify's base Docker images define the CMD instruction
# that runs the Node.js source code using the command specified
# in the "scripts.start" section of the package.json file.
# In short, the instruction looks something like this:
#
# CMD npm start


Dockerfile

Country in

Industry in

Max items

Filter number of employees from the search results.

Minimum number of employees

<strong>A proxy server is required to run this actor!</strong> Either use an Apify residential proxy, or provide your own proxy servers. Datacenter proxies will not work.

Proxy configuration

Minimum revenue

Search term as you would enter to Airbnb search.

Search term

Filter year start from the search results.

From year start

CheerioCrawler Template

INPUT_SCHEMA.json

{
    "env": { "npm_config_loglevel": "silent" }
}


apify.json

/* eslint-disable max-len */
const Apify = require('apify');
const playwright = require('playwright');
const { handleStart, handleDetail } = require('./src/routes');

const { utils: { log } } = Apify;

const PAGE_SIZE = 50;

Apify.main(async () => {
    const { startUrls = [], searchTerm, maxItems, revenueMin, numberOfEmployeesMin, yearStartFrom, countryIn, industryIn,
        proxyConfiguration } = await Apify.getInput();

    global.maxItems = maxItems;
    global.PAGE_SIZE = PAGE_SIZE;
    global.revenueMin = revenueMin;
    global.numberOfEmployeesMin = numberOfEmployeesMin;
    global.yearStartFrom = yearStartFrom;
    global.countryIn = countryIn;
    global.industryIn = industryIn;

    startUrls.push(`https://www.dnb.com/site-search-results.html#AllSearch=Tesla&CompanyProfilesPageNumber=1&CompanyProfilesSearch=Tesla&ContactProfilesPageNumber=1&DAndBMarketplacePageNumber=1&DAndBMarketplaceSearch=Tesla&IndustryPageNumber=1&SiteContentPageNumber=1&SiteContentSearch=Tesla&tab=Company%20Profiles`);
    const proxy = await Apify.createProxyConfiguration(proxyConfiguration);

    const requestList = await Apify.openRequestList('start-urls', startUrls);
    const requestQueue = await Apify.openRequestQueue();

    const crawler = new Apify.PlaywrightCrawler({
        requestList,
        requestQueue,
        proxyConfiguration: proxy,
        launchContext: {
            launcher: playwright.firefox,
            launchOptions: {
                headless: true,
            },
        },
        useSessionPool: true,
        navigationTimeoutSecs: 120,
        handlePageFunction: async (context) => {
            const { url, userData: { label } } = context.request;
            log.info('Page opened.', { label, url });

            switch (label) {
                case 'DETAIL':
                    return handleDetail(context);
                default:
                    return handleStart(context, requestQueue);
            }
        },
        preNavigationHooks: [
            async (crawlingContext) => {
                const { page } = crawlingContext;
                await page.route('**/*', (route) => {
                    return route.request().resourceType() === 'image'
                        ? route.abort()
                        : route.continue();
                });
            },
        ],
    });

    log.info('Starting the crawl.');
    await crawler.run();
    log.info('Crawl finished.');
});


main.js

dnb-companies-scraper

package.json

Filtering and extracting data about companies from https://www.dnb.com. Easily filtering results by different criteria such as revenue minimum, number of employees minimum, located in different countries, etc.

Filtering and extracting data about companies from https://www.dnb.com. Get structured data in JSON, XML, CSV, Excel.

DNB Companies Scraper

Where next?