# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 2
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage

storage

.gitignore

imdb-ratings

package.json

.actor/actor.json

# Specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent


.actor/Dockerfile

Save IDs, scrape info or ratings from IMDB charts. Popular movies example: 'moviemeter'

IMDb Charts

DATASET NAME

ID DATASET NAME

RATINGS DATASET NAME

Save movie data

Save rating data

Scraping limit number for IMDb Charts (popular). Default is 50.

Scrape limit

Array of movie IDs which wil be scraped. Example: ['tt2527338', 'tt1490017']

MOVIES IDs

Search query. It will save 5 IDs which can be scraped. Example: 'John Wick' or 'Star Wars'

Search title

Save IDs to dataset

Starting limit number for scraping charts. Default is 1. Set 100 and than it will scrape from 100.movie in IMDb Charts (example popular).

Starting limit

Specifies proxy servers that will be used by the scraper in order to hide its origin.<br><br>For details, see <a href='https://apify.com/apify/web-scraper#proxy-configuration' target='_blank' rel='noopener'>Proxy configuration</a> in README.

Proxy configuration

IMDB ID, data and ratings scraper.

.actor/INPUT_SCHEMA.json

# IMDB - Movie data and ratings scraper

This actor can scrape movie [info](https://www.imdb.com/title/tt10954600/) and movie ratings from [page](https://www.imdb.com/title/tt10954600/ratings/).

## Actor cost

Actor using cheerio scraper which is efficient. Recommended RAM usage is 256MB.

## Input config

1. Choose what you want scrape: `specific title IDs (MOVIE_IDS)` or `search keyword (QUERY) (limited to 5 titles per search)` or `scrape IMDB chart (CHART)`.
2. If you set CHART. You can set `LIMIT` and `START_LIMIT_FROM` (optional)
2. Do you want scrape title data `GET_MOVIES_DATA`, ratings `GET_RATINGS` or IDs `SAVE_IDs`?
3. Do you want save data to named dataset to have specific data in named dataset? Than set `DATASET_NAME` or `DATASET_NAME_RATINGS` or `DATASET_NAME_ID`

## Input

|Inpot value|Description|
|---|---|
|MOVIE_IDS|Scrape list of IDs from IMDB. ID can be find in url: https://www.imdb.com/title/`tt10954600`/|
|QUERY|Search any keyword|
|CHART|Scrape any IMDB chart. Like popular chart => "`moviemeter`".|
|START_LIMIT_FROM (optional)|Starting number of scraping movies. Default value is 1|
|LIMIT (optional)|Maximum number of movies that will be scraped. Default value is 50|
|GET_MOVIES_DATA|Set `true` if you want scrape movie data|
|GET_RATINGS|Set `true` if you want scrape ratings|
|SAVE_IDs (optional)|Set `true` if you want scrape IDs. Only for `QUERY` or `CHART`|
|DATASET_NAME (optional)|Save scraped data to named dataset. Run actor many times with same name but different IDs than this dataset will contains all data.|
|DATASET_NAME_RATINGS (optional)|Save scraped ratings to named dataset.|
|DATASET_NAME_ID (optional)|Save scraped IDs to named dataset.|
|proxyConfiguration (optional)|Select proxy to avoid IP address blocking.|

### Input example

Scrape ratings for 3 movies:

```
{
  "MOVIE_IDS": ["tt10954600", "tt12823454", "tt2911666"],
  "GET_RATINGS": true,
  "proxyConfiguration": {
    "useApifyProxy": false
  }
}
```

Scrape 5 movie data from search keyword "John Wick":

```
{
  "QUERY": "John Wick",
  "GET_MOVIES_DATA": true,
  "proxyConfiguration": {
    "useApifyProxy": false
  }
}
```

### Charts

IMDB has different chart.
Just set: `{ "CHART": "moviemeter", }`

|CHART|value|
|---|---|
|Popularity|"moviemeter"|
|A-Z|"alpha"|
|User Rating|"user_rating"|
|Number of Votes|"num_votes"|
|US Box Office|"boxoffice_gross_us"|
|Runtime|"runtime"|
|Year|"year"|
|Release Date|"release_date"|
|Date of Your Rating|"your_rating_date"|
|Your Rating|"sort=my_ratings"|

### Result for MOVIE DATA

Input:
`{ "GET_MOVIES_DATA": true }`

```
{
	"title": "John Wick",
	"releaseDate": "October 24, 2014 (United States)",
	"director": [
		"Chad Stahelski",
		"David Leitch"
	],
	"writers": [
		"Derek Kolstad"
	],
	"countryOfOrigin": [
		"United States",
		"United Kingdom",
		"China"
	],
	"rating": "7.4",
	"popularity": "119",
	"genres": [
		"Action",
		"Crime",
		"Thriller"
	],
	"officialSites": [
		"http://www.facebook.com/johnwickmovie",
		"http://www.johnwickthemovie.com/"
	],
	"language": "EnglishRussianHungarian",
	"locations": [
		"Calvary Cemetery, Woodside, Queens, New York City, New York, USA"
	],
	"ProductionCompanies": [
		"Summit Entertainment",
		"Thunder Road Pictures",
		"87Eleven"
	],
	"grossUSCanada": "$43,037,835",
	"openingWeekendUSCanada": "$14,415,922",
	"grossWorldwide": "$86,081,711",
	"runtime": "1 hour 41 minutes",
	"casts": [
		"Keanu Reeves @ John Wick",
		"Michael Nyqvist @ Viggo Tarasov",
		"Alfie Allen @ Iosef Tarasov",
		"Willem Dafoe @ Marcus",
		"Dean Winters @ Avi",
		"Adrianne Palicki @ Ms. Perkins",
		"Omer Barnea @ Gregori",
		"Toby Leonard Moore @ Victor",
		"Daniel Bernhardt @ Kirill",
		"Bridget Moynahan @ Helen",
		"John Leguizamo @ Aurelio",
		"Ian McShane @ Winston",
		"Bridget Regan @ Addy",
		"Lance Reddick @ Hotel Manager",
		"Keith Jardine @ Kuzma",
		"Tait Fletcher @ Nicholai",
		"Kazy Tauginas @ Ivan",
		"Alexander Frekey @ Alexander"
	],
	"url": "https://www.imdb.com/title/tt2911666"
}
```

### Result for ID's

Input:
`{ "SAVE_IDs": true }`

```
{
	"url": "https://www.imdb.com/find/?q=John+Wick",
	"titles": [
		{
			"title": "John Wick: Chapter 4",
			"id": "tt10366206"
		},
		{
			"title": "John Wick",
			"id": "tt2911666"
		},
		{
			"title": "John Wick: Chapter 2",
			"id": "tt4425200"
		},
		{
			"title": "John Wick: Chapter 3 - Parabellum",
			"id": "tt6146586"
		},
		{
			"title": "John Wick: Chapter 5",
			"id": "tt12844100"
		}
	]
}
```

### Result for ratings

Input:
`{ "GET_RATINGS": true }`

- Return ratings for all demographics:

`'imdb_users', 'males', 'females', 'aged_under_18', 'aged_18_29', 'aged_30_44', 'aged_45_plus', 'males_aged_under_18', 'males_aged_18_29', 'males_aged_30_44', 'males_aged_45_plus', 'females_aged_under_18', 'females_aged_18_29', 'females_aged_30_44', 'females_aged_45_plus'`

For all IMDB users:

```
{
	"date": "22-02-2023",
	"title": "John Wick (2014)",
	"demographic": "imdb_users",
	"averageVote": 7.4,
	"arithmeticVote": 7.6,
	"medianVote": 8,
	"totalVotes": 645487,
	"Votes_10": 79402,
	"Percent_10": 12.3,
	"Votes_9": 87601,
	"Percent_9": 13.6,
	"Votes_8": 188132,
	"Percent_8": 29.1,
	"Votes_7": 171542,
	"Percent_7": 26.6,
	"Votes_6": 68794,
	"Percent_6": 10.7,
	"Votes_5": 25724,
	"Percent_5": 4,
	"Votes_4": 10195,
	"Percent_4": 1.6,
	"Votes_3": 5509,
	"Percent_3": 0.9,
	"Votes_2": 3443,
	"Percent_2": 0.5,
	"Votes_1": 5145,
	"Percent_1": 0.8,
	"url": "https://www.imdb.com/title/tt2911666/ratings?demo=imdb_users"
}
```


.actor/README.md

export const baseUrl = 'https://www.imdb.com/title/';
export const searchUrl = 'https://www.imdb.com/search/title/';
export const findUrl = 'https://www.imdb.com/find/';
export const endUrl = '/ratings?demo=';
export const titleText = ' - User ratings - IMDbIMDb, an Amazon company';
// eslint-disable-next-line max-len
export const TYPES = ['imdb_users', 'males', 'females', 'aged_under_18', 'aged_18_29', 'aged_30_44', 'aged_45_plus', 'males_aged_under_18', 'males_aged_18_29', 'males_aged_30_44', 'males_aged_45_plus', 'females_aged_under_18', 'females_aged_18_29', 'females_aged_30_44', 'females_aged_45_plus'];
export const pageLimit = 50;


src/const.js

src/inputExample.json

import { Actor } from 'apify';
import { Dataset, CheerioCrawler, log } from 'crawlee';
import * as CONST from './const.js';

await Actor.init();
let { MOVIE_IDS, QUERY, GET_MOVIES_DATA, GET_RATINGS, CHART, SAVE_IDs, DEBUG, LIMIT, START_LIMIT_FROM, 
  DATASET_NAME, DATASET_NAME_ID, DATASET_NAME_RATINGS, proxyConfiguration } = await Actor.getInput();

(DEBUG) ? log.setLevel(log.LEVELS.DEBUG) : '';
(LIMIT) ? '' : LIMIT = 50;
if (!MOVIE_IDS && !QUERY  && !CHART) {
  log.error(`FAIL IN INPUT. Add to input one of: MOVIE_IDS=[""] or QUERY="" or CHART="".`);
}

// IDs scraper from querry search
if (QUERY && !MOVIE_IDS) {
  log.info('Getting 5 IDs for searching query: ' + QUERY);
  let IDs = [];
  const crawlerID = new CheerioCrawler({
    proxyConfiguration: await Actor.createProxyConfiguration(proxyConfiguration),
    minConcurrency: 1,
    maxConcurrency: 1,
    maxRequestRetries: 1,
    requestHandlerTimeoutSecs: 5,
    maxRequestsPerCrawl: 1,
    async requestHandler({ request, $ }) {
      log.debug(`Processing ${request.url}...`);
      const DATA = [];
      const titles = [];
      $('a.ipc-metadata-list-summary-item__t').each((index, el) => {
        if (index < 5) {
          let text = $(el).text();
          let link = $(el).attr('href').replace("/title/","").split("/");
          titles.push({
            title: text,
            id: link[0],
          });
          IDs.push(link[0]);
        }
      });

      if (SAVE_IDs) {
        DATA.push({url: request.url, CHART, titles});
        if (DATASET_NAME_ID && DATASET_NAME_ID.length > 0) {
          const dataset = await Dataset.open(DATASET_NAME_ID);
          await dataset.pushData(DATA);
        } else {
          await Actor.pushData(DATA);
        }
      }
    },
    failedRequestHandler({ request }) {
      log.debug(`Request ${request.url} failed twice.`);
    },
  });
  await crawlerID.run([`${CONST.findUrl}?q=${QUERY.replace(" ","+")}`]);  
  console.log('ID: '+ IDs);
  MOVIE_IDS = IDs;
}

// IDs scraper from sorting page
if (CHART && !MOVIE_IDS) {
  (START_LIMIT_FROM) ? START_LIMIT_FROM : START_LIMIT_FROM = 1;
  let scrapeTypeID = [];
  for (let i = 0; i < Math.ceil(LIMIT/CONST.pageLimit); i++) {
    let lim = CONST.pageLimit * i + START_LIMIT_FROM;
    scrapeTypeID.push(`${CONST.searchUrl}?title_type=feature&view=simple&sort=${CHART}&start=${lim}`);
  }

  log.info(`Getting ${LIMIT} IDs for feature chart: ` + CHART);
  let IDs = [];

  const crawlerType = new CheerioCrawler({
    proxyConfiguration: await Actor.createProxyConfiguration(proxyConfiguration),
    minConcurrency: 1,
    maxConcurrency: 2,
    maxRequestRetries: 1,
    requestHandlerTimeoutSecs: 20,
    async requestHandler({ request, $ }) {
      log.debug(`Processing ${request.url}...`);
      const DATA = [];
      const titles = [];
      $('div.lister-item div.lister-item-content').each((index, el) => {
        let number = Number($(el).find('span.lister-item-index').text().replace(".",""));
        let text = $(el).find(' > div > div > span > span > a').text();
        let link = $(el).find(' > div > div > span > span > a').attr('href').replace("/title/","").split("/");
        titles.push({
          index: number,
          title: text,
          id: link[0],
        });
        IDs.push(link[0]);
      });

      if (SAVE_IDs) {
        DATA.push({url: request.url, CHART, titles});
        if (DATASET_NAME_ID && DATASET_NAME_ID.length > 0) {
          const dataset = await Dataset.open(DATASET_NAME_ID);
          await dataset.pushData(DATA);
        } else {
          await Actor.pushData(DATA);
        }
      }
    },
    failedRequestHandler({ request }) {
      log.debug(`Request ${request.url} failed twice.`);
    },
  });
  await crawlerType.run(scrapeTypeID);  
  MOVIE_IDS = IDs;
}

// Movie data scraper
if (GET_MOVIES_DATA && MOVIE_IDS && MOVIE_IDS.length > 0) {
  log.info('Scraping all data from # ' + MOVIE_IDS.length + ' of movies.');
  let scrapeUrls = [];
  for (const ID of MOVIE_IDS) {
    scrapeUrls.push(CONST.baseUrl + ID);
  }
  const crawlerMOVIE = new CheerioCrawler({
    proxyConfiguration: await Actor.createProxyConfiguration(proxyConfiguration),
    maxRequestRetries: 1,
    requestHandlerTimeoutSecs: 10,
    async requestHandler({ request, $ }) {
      const title = $('h1').text();
      const rating = $('div[data-testid="hero-rating-bar__aggregate-rating"] span').first().text();
      const popularity = $('div[data-testid="hero-rating-bar__popularity__score"]').first().text();
      const casts = [];
      $('div[data-testid="title-cast-item"]').each((index, el) => {
        let text = $(el).find('[data-testid="title-cast-item__actor"]').text();
        let char = $(el).find('[data-testid="cast-item-characters-link"]').text();
        casts.push(`${text} @ ${char}`);
      });
      const genres = [];
      $('[data-testid="genres"] span').each((index, el) => {
        let text = $(el).text();
        genres.push(text);
      });
      const releaseDate = $('li[data-testid="title-details-releasedate"] > div').text();
      const countryOfOrigin = [];
      $('li[data-testid="title-details-origin"] ul li').each((index, el) => {
        let text = $(el).text();
        countryOfOrigin.push(text);
      });
      const officialSites = [];
      $('li[data-testid="details-officialsites"] > div a').each((index, el) => {
        let text = $(el).attr('href');
        officialSites.push(text);
      });
      const language = $('li[data-testid="title-details-languages"] > div').text();
      const director = [];
      $('[data-testid="title-pc-wide-screen"] li[data-testid="title-pc-principal-credit"]:nth-of-type(1) div a').each((index, el) => {
        let text = $(el).text();
        director.push(text);
      });
      const writers = [];
      $('[data-testid="title-pc-wide-screen"] li[data-testid="title-pc-principal-credit"]:nth-of-type(2) div a').each((index, el) => {
        let text = $(el).text();
        writers.push(text);
      });
      const locations = [];
      $('li[data-testid="title-details-filminglocations"] ul').each((index, el) => {
        let text = $(el).text();
        locations.push(text);
      });
      const ProductionCompanies = [];
      $('[data-testid="title-details-companies"] li').each((index, el) => {
        let text = $(el).text();
        ProductionCompanies.push(text);
      });
      const grossUSCanada = $('[data-testid="title-boxoffice-grossdomestic"] div ul li').first().text();
      const openingWeekendUSCanada = $('[data-testid="title-boxoffice-openingweekenddomestic"] div ul li').first().text();
      const grossWorldwide = $('[data-testid="title-boxoffice-cumulativeworldwidegross"] div ul li').first().text();
      const runtime = $('[data-testid="title-techspec_runtime"] div').first().text();

      let DATA = {
        title,
        releaseDate,
        director,
        writers,
        countryOfOrigin,
        rating,
        popularity,
        genres,
        officialSites,
        language,
        locations,
        ProductionCompanies,
        grossUSCanada,
        openingWeekendUSCanada,
        grossWorldwide,
        runtime,
        casts,
        url: request.url,
      };

      if (DATASET_NAME && DATASET_NAME.length > 0) {
        const dataset = await Dataset.open(DATASET_NAME);
        await dataset.pushData(DATA);
      } else {
        await Actor.pushData(DATA);
      }
    },
    failedRequestHandler({ request }) {
        log.debug(`Request ${request.url} failed twice.`);
    },
  });
  await crawlerMOVIE.run(scrapeUrls);
}

// Rating scraper
if (GET_RATINGS && MOVIE_IDS && MOVIE_IDS.length > 0) {
  let scrapeUrls = [];
  for (const ID of MOVIE_IDS) {
    for (const type of CONST.TYPES) {
      scrapeUrls.push(CONST.baseUrl + ID + CONST.endUrl + type);
    }
  }

  const crawler = new CheerioCrawler({
    proxyConfiguration: await Actor.createProxyConfiguration(proxyConfiguration),
    maxRequestRetries: 1,
    requestHandlerTimeoutSecs: 5,
    async requestHandler({ request, $ }) {
      log.debug(`Processing ${request.url}...`);
      const ratingExist = $("div.sectionHeading").text();
      if (ratingExist != "No Ratings Available") {
        const ratingValues = [];
        const topValues = [];
        const votesNumbers = [];
        const demographic = request.url.split(CONST.endUrl);
        const title = $('title').text().replace(CONST.titleText,"");
        let showedTotalVotes = Number($('td.Selected > div > a').text().trim().replace(",",""));
  
        console.log('Title:', title);
        console.log('Demographic:', demographic[1]);

        const averageVote = (($('div.title-ratings-sub-page > div.allText > div.allText').first().text().split(' / 10'))[0].split('average vote of'))[1];
        const votes = $('div.title-ratings-sub-page > div.allText > div.allText > div.allText').text();
        const arithmeticVote = Number(votes.split('Arithmetic mean = ')[1].split('\n')[0]);
        const medianVote = Number(votes.split('Median = ')[1]);

        $('div.rightAligned').each((index, el) => {
          if (index > 0) {
            let text = $(el).text().trim();
            ratingValues.push({
              rating: Number(text),
            });
          }
        });
  
        $('div.topAligned').each((index, el) => {
          let text = $(el).text().trim().replace("%","");
          topValues.push({
            percent: Number(text),
          });
        });
  
        $('div.leftAligned').each((index, el) => {
          if (index > 0 && index <= 10) {
            let text = $(el).text().trim().replace(",","");
            votesNumbers.push({
              votes: Number(text),
            });
          }
        });

        let totalVotesNumber = 0;
        for (const vote in votesNumbers) {
          totalVotesNumber += votesNumbers[vote].votes;
        }
        console.log('Control votes number:', totalVotesNumber);
  
        let totalPercent = 0;
        for (const percent in topValues) {
          totalPercent += topValues[percent].percent;
        }
        
        if (Math.round(totalPercent) != 100) {
          log.warning('Total percent is not correct:', Math.round(totalPercent));
          console.log(request.url);
        }
  
        let votesNumberCorrect = true;
        if (totalVotesNumber != showedTotalVotes) {
          votesNumberCorrect = showedTotalVotes;
          log.warning(`Total votes number is not corrent: ${totalVotesNumber} != ${showedTotalVotes}`);
          console.log(request.url);
        }
  
        console.log("Control percent is 100% =", Math.round(totalPercent),"%");
        log.debug('',ratingValues); 
        log.debug('',votesNumbers);
        log.debug('',topValues);

        let percentValues = [], increment = 0;
        for (const value in ratingValues) {
          if (votesNumbers[value].votes === 0) {
            increment+=1;
            log.debug('', {increment});
            percentValues.push(0);
          } else {
            percentValues.push(topValues[value-increment].percent);
          }
        }
        log.debug('Percent values: ', percentValues);

        let DATA = {
          date: await displayTime(),
          title,
          demographic: demographic[1],
          averageVote: Number(averageVote),
          arithmeticVote,
          medianVote,
          totalVotes : totalVotesNumber,
          Votes_10: votesNumbers[0].votes,
          Percent_10: percentValues[0],
          Votes_9: votesNumbers[1].votes,
          Percent_9: percentValues[1],
          Votes_8: votesNumbers[2].votes,
          Percent_8: percentValues[2],
          Votes_7: votesNumbers[3].votes,
          Percent_7: percentValues[3],
          Votes_6: votesNumbers[4].votes,
          Percent_6: percentValues[4],
          Votes_5: votesNumbers[5].votes,
          Percent_5: percentValues[5],
          Votes_4: votesNumbers[6].votes,
          Percent_4: percentValues[6],
          Votes_3: votesNumbers[7].votes,
          Percent_3: percentValues[7],
          Votes_2: votesNumbers[8].votes,
          Percent_2: percentValues[8],
          Votes_1: votesNumbers[9].votes,
          Percent_1: percentValues[9],
          url: request.url,
        };
  
        if (DATASET_NAME_RATINGS && DATASET_NAME_RATINGS.length > 0) {
          const dataset = await Dataset.open(DATASET_NAME_RATINGS);
          await dataset.pushData(DATA);
        } else {
          await Actor.pushData(DATA);
        }
      } else {
        log.warning("Rating does not exist for page: " + request.url);
      }
    },

    // This function is called if the page processing failed more than maxRequestRetries + 1 times.
    failedRequestHandler({ request }) {
      log.debug(`Request ${request.url} failed twice.`);
    },
  });

  await crawler.run(scrapeUrls);
  log.info('Crawler finished.');
}

async function displayTime() {
  let currentTime = new Date();
  let date = String(currentTime.getDate()).padStart(2, '0') + '-' + 
    String(currentTime.getMonth() + 1).padStart(2, '0') + '-' + 
    currentTime.getFullYear();
  return date;
}

await Actor.exit();

IMDB Ratings

Where next?