# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage

# Added by Apify CLI
.venv


.gitignore

imdb-extractor

package.json

# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent


.actor/Dockerfile

Extracts data from IMDB.com. It takes a list of IMDB movie URLs as an input and returns a dataset with the extracted data.

IMDB Extractor

.actor/actor.json

genre

name

.actor/dataset_schema.json

Function that takes a JQuery handle ($) as argument and returns data that will be merged with the default output

Extend output function

Limit number of videos to be scraped. Must be greater than 0.

Max items

The best option is usually Automatic proxy. But you can also use your own proxies or no proxy

Proxy configuration

List of URLs that will be scraped or crawled. Go to https://www.imdb.com/search/title/ and choose filter you want. Then copy the URL and paste it here.

Start URLs

IMDB Extractor Input

.actor/input_schema.json

/* eslint-disable  */

import { Actor, log } from 'apify';
import { CheerioCrawler } from 'crawlee';
import { router } from './routes.js';

await Actor.init();

const input = await Actor.getInput();
const { startUrls, maxItems = 50, extendOutputFunction } = input;

const proxyConfiguration = await Actor.createProxyConfiguration({
    ...input.proxyConfiguration,
    countryCode: 'US',
});

const crawler = new CheerioCrawler({
    proxyConfiguration,
    requestHandler: router,
    maxConcurrency: 5,
    useSessionPool: true,
    persistCookiesPerSession: true,
    requestHandlerTimeoutSecs: 120,
});

const initializeStartUrls = startUrls.map((url) => {
    return {
        ...url,
        userData: {
            maxItems,
            extendOutputFunction,
        },
    };
});

await crawler.run(initializeStartUrls);

// Exit successfully
await Actor.exit();


src/main.js

/* eslint-disable */
import { Actor } from 'apify';
import { createCheerioRouter } from 'crawlee';
import { evalPageFunction, executeExtendOutputFn } from './utils.js';
export const router = createCheerioRouter();

// total items processed
let count = 0;

/**
 * Handler for start page. Enqueues detail pages and next page.
 * @param {CheerioCrawlerRequest} request
 */
router.addDefaultHandler(async ({ enqueueLinks, log, $, request }) => {
    const { maxItems } = request.userData;
    const paginationEle = $('.desc span');
    if (!paginationEle || paginationEle.text() === '') {
        return;
    }
    log.info(paginationEle.eq(0).text());

    // loop through items and enqueue detail page
    const { processedRequests } = await enqueueLinks({
        selector: '.lister-list .lister-item .lister-item-header a',
        label: 'detail',
        regexps: [/\/title\/tt\d+\//],
        userData: request.userData,
    });
    count += processedRequests.length;

    // if maxItems is reached, stop crawling
    if (maxItems && count >= maxItems) {
        log.info('maxItems reached');
        return;
    }

    // if maxItems is not reached, enqueue next page
    const nextPage = $('.desc a').last();
    if (nextPage && nextPage.text() === 'Next »') {
        const nextHref = nextPage.attr('href');
        const nextURL = new URL(nextHref, request.loadedUrl);
        log.info(`Enqueueing next page: ${nextURL.toString()}`);
        await enqueueLinks({
            urls: [nextURL.toString()],
            userData: request.userData,
        });
    }
});

// Handler for detail page
router.addHandler('detail', async ({ request, $, log }) => {
    const { extendOutputFunction } = request.userData;

    // parse ld+json script tag
    const ldJson = $('script[type="application/ld+json"]').html();
    const ldJsonObj = JSON.parse(ldJson);

    let userResult = {};
    if (extendOutputFunction) {
        const extendOutputFunctionEvaled =
            evalPageFunction(extendOutputFunction);
        userResult = await executeExtendOutputFn({
            fn: extendOutputFunctionEvaled,
            $,
            item: ldJsonObj,
        });
    }

    // push data to dataset
    Actor.pushData({ ...ldJsonObj, ...userResult });
});


src/routes.js

/* eslint-disable */
export const evalPageFunction = (extendOutputFunction) => {
    let extendOutputFunctionEvaled;
    if (extendOutputFunction) {
        try {
            extendOutputFunctionEvaled = eval(extendOutputFunction);
        } catch (e) {
            throw new Error(
                `extendOutputFunction is not a valid JavaScript! Error: ${e}`
            );
        }
        if (typeof extendOutputFunctionEvaled !== 'function') {
            throw new Error(
                `extendOutputFunction is not a function! Please fix it or use just default output!`
            );
        }
    }
    return extendOutputFunctionEvaled;
};

export const executeExtendOutputFn = async ({ fn, $, item }) => {
    const isObject = (val) =>
        typeof val === 'object' && val !== null && !Array.isArray(val);

    let userResult = {};

    if (!fn) {
        return userResult;
    }

    try {
        userResult = await fn($, item);
    } catch (e) {
        log.warning(
            `extendOutputFunction crashed! Pushing default output. Please fix your function if you want to update the output. Error: ${e}`
        );
        throw e;
    }

    if (!isObject(userResult)) {
        log.error('extendOutputFunction has to return an object!!!');
        process.exit(1);
    }
    return userResult;
};


src/utils.js

Extracts data from IMDB.com. It takes a list of IMDB movie URLs as input and returns a dataset with the extracted data.

This is an extractor that allows you to extract the following information from IMDB:

-   Title
-   Description
-   Year of release
-   Duration
-   Trailer URL
-   and more...

Limit number of videos to be scraped. Leave blank to scrape all videos.

Imdb Extractor

Imdb Extractor · Apify

Input · Imdb Extractor · Apify

API · Imdb Extractor · Apify

Related Actors · Imdb Extractor · Apify

Extracts data from IMDB.com. It takes a list of IMDB movie URLs as input and returns a dataset with the extracted data.

This is an extractor that allows you to extract the following information fr...

Where next?