root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf
# editorconfig-tools is unable to ignore longs strings or urls
max_line_length = null


.editorconfig

.eslintrc

apify_storage
node_modules
package-lock.json


.gitignore

module.exports = {
    printWidth: 80,
    singleQuote: true,
    trailingComma: 'all',
};


.prettierrc.js

# First, specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Optionally, specify how to launch the source code of your actor.
# By default, Apify's base Docker images define the CMD instruction
# that runs the Node.js source code using the command specified
# in the "scripts.start" section of the package.json file.
# In short, the instruction looks something like this:
#
# CMD npm start


Dockerfile

An actor that scrapes ebooks from gutenberg.org

Function that takes a JQuery handle ($) as argument and returns data that will be merged with the default output

Extend output function

Select a language that Gutenberg.org provides

Language

Maximum number of items you want on your results

Max Items

Select proxies to be used by your crawler.

Proxy configuration

Keyword that you want to search in Gutenberg.org

Search keyword

URLs to start with. It should be category or product detail URLs

Start URLs

Gutenberg.org Scraper

INPUT_SCHEMA.json

gutenberg-scraper

apify.json

package.json

// Fetch offers from browse list page
const fetchEbooksFromBrowse = ($) => {
    return $('a[href]')
        .filter((i, el) => $(el).attr('href').match(/ebooks\/[0-9]/g))
        .map((i, el) => `https://www.gutenberg.org${$(el).attr('href')}`)
        .get();
};

// Fetch offers from search list page
const fetchEbooksFromSearch = ($) => {
    return $('.booklink a[href]')
        .filter((i, el) => $(el).attr('href').match(/ebooks\/[0-9]/g))
        .map((i, el) => `https://www.gutenberg.org${$(el).attr('href')}`)
        .get();
};

// Fetch ebook defaults
const fetchEbook = ($) => {
    return {
        author: $('a[itemprop="creator"]').text().trim(),
        title: $('h1[itemprop="name"]').text().trim(),
        language: $('tr[itemprop="inLanguage"]').text().trim().replace(/Language/, '')
            .trim(),
        htmlURL: $('.files td[property="dcterms:format"]')
            ? `https://www.gutenberg.org${$('.files td[property="dcterms:format"]')
                .filter((i, el) => $(el).attr('content').includes('text/html'))
                .find('a')
                .attr('href')}` : null,
        epubURL: $('.files td[content="application/epub+zip"] a').attr('href')
            ? `https://www.gutenberg.org${$('.files td[content="application/epub+zip"] a').attr('href')}`
            : null,
        kindleURL: $('.files td[content="application/x-mobipocket-ebook"] a').attr('href')
            ? `https://www.gutenberg.org${$('.files td[content="application/x-mobipocket-ebook"] a').attr('href')}`
            : null,
        plainTextURL: $('.files td[content="text/plain; charset=utf-8"] a').attr('href')
            ? `https://www.gutenberg.org${$('.files td[content="text/plain; charset=utf-8"] a').attr('href')}`
            : null,
    };
};

module.exports = {
    fetchEbooksFromBrowse,
    fetchEbooksFromSearch,
    fetchEbook,
};


src/extractors.js

src/main.js

// routes.js
const Apify = require('apify');
const extractors = require('./extractors');

const {
    utils: { log },
} = Apify;


// ItemCount
let itemCount = 0;

// BROWSE page crawler.
// Fetches the ebooks from the list
exports.BROWSE = async ({ $, request }, { requestQueue }) => {
    log.info(`CRAWLER -- Browse: Fetching ebooks from the url: ${request.url}`);


    // Check for error
    if ($('html').html().includes('There are too many')) {
        log.error(`There too many books on website. It doesn't show up: ${request.url}`);
        process.exit(0);
    }

    // Fetch all current ebooks listed
    const ebooks = extractors.fetchEbooksFromBrowse($);

    // Add them to request queue
    for (const ebook of ebooks) {
        await requestQueue.addRequest(
            {
                url: ebook,
                userData: {
                    label: 'EBOOK',
                },
            },
            { forefront: true },
        );
    }

    log.debug(`CRAWLER -- ${ebooks.length} ebooks added to queue`);
};

// SEARCH page crawler.
// Fetches the ebooks from search page and
// Adds next page to request queue
exports.SEARCH = async ({ $, request }, { requestQueue }) => {
    const { page, baseURL } = request.userData;
    log.info(`CRAWLER -- Search: Fetching ebooks from the url: ${request.url} page: ${page}`);

    // Fetch all current ebooks listed
    const ebooks = extractors.fetchEbooksFromSearch($);

    // Add them to request queue
    for (const ebook of ebooks) {
        await requestQueue.addRequest(
            {
                url: ebook,
                userData: {
                    label: 'EBOOK',
                },
            },
            { forefront: true },
        );
    }

    // Move to next page
    if (ebooks.length > 0) {
        await requestQueue.addRequest({
            url: `${baseURL}&start_index=${25 * page + 1}`,
            userData: {
                label: 'SEARCH',
                page: page + 1,
                baseURL,
            },
        });
    }

    log.debug(`CRAWLER -- ${ebooks.length} ebooks added to queue`);
};

// EBOOK page crawler.
// Gets details of current ebook
exports.EBOOK = async ({ $, request },{input}) => {
    const { extendOutputFunction, maxItems } = input;
    log.info(`CRAWLER -- Fetching ebook details from ${request.url}`);

    // Check for maxItems
    if (maxItems && maxItems <= itemCount) {
        process.exit(0);
    }


    // Fetch ebook details
    const ebook = extractors.fetchEbook($);


    // Push data
    await Apify.pushData({ ...ebook, ...(extendOutputFunction ? { ...eval(extendOutputFunction)($) } : {}) });
    itemCount += 1;

    log.debug(`CRAWLER -- ebook details fetched from ${request.url}`);
};


src/routes.js

src/tools.js

Explore and Download Free eBooks - Find and download a wide selection of free eBooks from Project Gutenberg. Search by keywords and language preferences. Discover literary gems in multiple formats.

Extract and download free eBooks from Project Gutenberg in all formats. Search by keywords and language preference. Explore a vast library of literary classics.

Free eBook Scraper

Where next?