# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 2
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

.eslintrc

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage

storage

.gitignore

Get HTML validity reports from various web pages using W3C HTML validator.

w3c-reporter

package.json

tsconfig.json

w3c-html-reporter

W3C HTML Validity Reporter

.actor/actor.json

Validity report

.actor/dataset_schema.json

# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome:16 AS builder

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json ./

# Install all dependencies. Don't audit to speed up the installation.
RUN npm install --include=dev --audit=false

# Next, copy the source files using the user set
# in the base image.
COPY --chown=myuser . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN npm run build

# Create final image
FROM apify/actor-node-puppeteer-chrome:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Copy built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY --chown=myuser . ./


# Run the image. If you know you won't need headful browsers,
# you can remove the XVFB start script for a micro perf gain.
CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent


.actor/Dockerfile

Debug Mode

You can edit this to use Apify proxy, or provide your own proxy servers.

Proxy configuration

URLs to get W3C HTML validity reports from.

Start URLs

W3C HTML Validity Reporter schema

.actor/input_schema.json

// For more information, see https://docs.apify.com/sdk/js
import { Actor, KeyValueStore, log, type ProxyConfigurationOptions } from 'apify';
// For more information, see https://crawlee.dev
import { PuppeteerCrawler } from 'crawlee';
import puppeteerVanilla from 'puppeteer';
import { addExtra } from 'puppeteer-extra';
import StealthPlugin from 'puppeteer-extra-plugin-stealth';
import AnonymizeUA from 'puppeteer-extra-plugin-anonymize-ua';

import { router } from './routes.js';

interface InputSchema {
  startUrls: string[];
  proxy?: ProxyConfigurationOptions & {
    useApifyProxy?: boolean;
  };
  debug?: boolean;
};

// Initialize the Apify SDK
await Actor.init();

// Configure puppeteer in stealth mode
const puppeteer = addExtra(puppeteerVanilla);
puppeteer.use(StealthPlugin());
puppeteer.use(AnonymizeUA());

const { startUrls = [], proxy: proxyConfig, debug } = await KeyValueStore.getInput<InputSchema>() ?? {};

if (debug) {
    log.setLevel(log.LEVELS.DEBUG);
}

const proxyConfiguration = await Actor.createProxyConfiguration(proxyConfig || {
    useApifyProxy: false,
    // groups: ['RESIDENTIAL'],
    // countryCode: 'US',
});

const crawler = new PuppeteerCrawler({
    launchContext: {
        launcher: puppeteer,
    },
    proxyConfiguration,
    requestHandler: router,
});

log.info('Starting the crawl.');
await crawler.run(startUrls.map((url: any) => {
    const urlToCrawl = url?.url ?? url;
    log.info(`Getting W3C validity report for ${urlToCrawl}...`);

    const searchParams = new URLSearchParams({ doc: urlToCrawl, out: 'json' });
    return {
        url: `https://validator.w3.org/nu/?${searchParams.toString()}`,
        method: 'GET',
    };
}));
log.info('Crawl finished.');

// Exit successfully
await Actor.exit();


src/main.ts

import { Dataset, createPuppeteerRouter } from 'crawlee';

export const router = createPuppeteerRouter();

router.addDefaultHandler(async ({ request, log, response }) => {
    log.info(`Processing ${request.url} ...`);

    if (!response || response.status() > 200) {
        log.error(`A problem occured while trying to fetch ${request.url}, you might need to use a proxy.`);
        return;
    }

    const { url, messages, language } = await response.json();

    log.info(`Found ${messages.length} messages for ${url} ...`);
    for (const {
        type: severity,
        subType,
        firstLine,
        lastLine,
        firstColumn,
        lastColumn,
        message,
        extract: markup,
        hiliteStart: highlightIndex,
        hiliteLength: highlightLength,
    } of messages) {
        await Dataset.pushData({
            url,
            language,
            severity: subType ?? severity,
            firstLine: firstLine ?? lastLine,
            lastLine,
            firstColumn,
            lastColumn,
            message,
            markup,
            highlightIndex,
            highlightLength,
        });
    }
});


W3C Html Reporter

Where next?