# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

.gitignore

package-lock.json

youtube-channel

package.json

Youtube-Channel

Youtube Channel Dataset

Youtube Channel Scraper

.actor/actor.json

# Specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM debian:latest

# Install necessary packages
RUN apt-get update && apt-get install -y wget gnupg2 curl unzip && rm -rf /var/lib/apt/lists/*

# Install Node.js and NPM
RUN curl -sL https://deb.nodesource.com/setup_16.x | bash -
RUN apt-get install -y nodejs

# Install Chrome
RUN curl -sSL https://dl.google.com/linux/linux_signing_key.pub | apt-key add - \
    && echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list \
    && apt-get update && apt-get install -y google-chrome-stable

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm install -g npm@latest \
    && npm --version \
    && rm -r ~/.npm
# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Run the image.
CMD npm start --silent





.actor/Dockerfile

Channel

Global timeout seconds (the base timeout is 300s and its calculated to add 300s for every 20 results)

The number of results you want from the channel.

Number of results  (no limit)

Add channel name

.actor/input-schema.json

# YouTube Scraper using Puppeteer

## Overview

This Apify actor uses Puppeteer to scrape YouTube videos for a specified channel and number of videos. The output of the actor includes details on each video scraped.

## Input

The actor takes in the following input:

- `channelName` (required) - The name of the channel to scrape videos from.
- `numberOfVideos` (required) - The number of videos to scrape.

## Output

The actor outputs an array of objects, where each object represents a single video. The object includes the following details for each video:

- `title` - The title of the video.
- `description` - The description of the video.
- `coverImage` - The URL of the Cover image.
- `videoUrl` - The URL of the video.
- `author` - The name of the author of the video.
- `viewCount` - The number of views the video has.
- `likeCount` - The number of likes the video has.
- `commentCount` - The number of comments the video has.
- `subscriberCount` - The number of subscribers the channel has.
- `publishedAt` - The date the video was published.

## Usage

To use this actor, you can set up a task on Apify and configure the input parameters as desired. Once the task is started, the actor will scrape the specified number of videos for the specified channel and output the details in the format described above.

### Example Input

```json
{
  "channelName": "National Geographic",
  "numberOfVideos": 10
}
```

### Example Output

```json
[
  {
    "title": "Video Title 1",
    "author": "Author Name",
    "description": "Video Description 1",
    "coverImage": "https://example.com/thumbnail-1.jpg",
    "videoUrl": "https://example.com/video-1.mp4",
    "commentCount": 12,
    "likeCount": 100,
    "viewCount": 1000,
    "subscriberCount": 1000000,
    "publishedAt": "2020-01-01T00:00:00.000Z"
  },
  {
    "title": "Video Title 2",
    "author": "Author Name",
    "description": "Video Description 2",
    "coverImage": "https://example.com/thumbnail-2.jpg",
    "videoUrl": "https://example.com/video-2.mp4",
    "commentCount": 12,
    "viewCount": 2000,
    "likeCount": 200,
    "subscriberCount": 1000000,
    "publishedAt": "2020-01-01T00:00:00.000Z"
  }
]
```

## Disclaimer

Please note that the use of this actor and any data obtained through its use is at your own risk. The developer of this actor is not responsible for any consequences resulting from the use of this actor and the data it generates. It is your responsibility to ensure that you comply with all applicable laws and regulations governing the use of such data.


.actor/README.md

// For more information, see https://sdk.apify.com/
import { Actor } from "apify";
import { scrape } from "./scrapper/youtube.js";

await Actor.init();

console.log("Loading input");
// Structure of input is defined in INPUT_SCHEMA.json.
const input = await Actor.getInput();
if (!input?.globalTimeOut) input.globalTimeOut = 300;

console.log(input);
console.log("Input loaded");
console.log(input?.channel);
console.log(input?.numberOfResults);
console.log(input?.globalTimeOut);

let channel = input?.channel;

// Check if the first character is '@', if not, add it
if (channel.charAt(0) !== "@") {
  channel = "@" + channel;
}

const result = await scrape(
  channel,
  input?.numberOfResults,
  input?.globalTimeOut
);

console.log("The result is: ", result);

// Structure of output is defined in .actor/actor.json
for (const item of result) {
  await Actor.pushData({
    title: item.title,
    author: item.author,
    videoUrl: item.videoUrl,
    coverImage: item.coverImage,
    subscriberCount: item.subscriberCount,
    likeCount: item.likeCount,
    description: item.description,
    viewCount: item.viewCount,
    commentCount: item.commentCount,
    publishedAt: item.publishedAt,
    id: item.id,
  });
}

await Actor.exit();


src/main.js

import puppeteer from "puppeteer-core";
import chromium from "chromium";
import findChrome from "chrome-finder";


async function scrape(channel, numberOfResults, globalTimeout) {
  const startTime = Date.now();
  if (!globalTimeout) globalTimeout = 300;
  if (globalTimeout === 300) {
    const additionalTimeout = Math.floor((numberOfResults - 1) / 20) * 300;
    globalTimeout += additionalTimeout;
  }
  globalTimeout = globalTimeout * 1000;
  console.log("Global timeout:", globalTimeout);

  // Helper function to check if the global timeout has been reached
  function checkGlobalTimeout() {
    if (Date.now() - startTime > globalTimeout) {
      throw new Error("Global timeout reached");
    }
  }

  try {
    console.log("Launching browser");
    // Launch the browser
    const browser = await puppeteer.launch({
      headless: true,
      executablePath: chromium.path || findChrome(),
      args: ["--no-sandbox", "--disable-setuid-sandbox"],
    });

    const page = await browser.newPage();
    const url = `https://www.youtube.com/${channel}/videos`;

    console.log("Navigating to YouTube channel");
    await page.goto(url, { waitUntil: "networkidle2" });

    // Handle consent prompt
    while (page.url().includes("consent.youtube.com")) {
      const popup = await page.$$("form button");
      if (popup.length > 0) {
        await popup[0].click();
      }
      await page.waitForNavigation();
    }

    if (!page.url().includes(`/${channel}/videos`)) {
      try {
        // Attempt to click the "Videos" tab
        await page.waitForSelector("#tabsContent", { timeout: 5000 });
        const videosTab = await page.$$("#tabsContent > tp-yt-paper-tab");

        await videosTab[1].click();
        await page.waitForNavigation({ waitUntil: "networkidle2" });
      } catch (error) {
        console.error("Loading");
      }
    }

    await page.waitForTimeout(3000);

    await page.waitForSelector("#subscriber-count", { timeout: 10000 });

    console.log("Waiting for subscriber count");
    const subscriberCount = await page.$eval("#subscriber-count", (el) =>
      el.textContent.trim()
    );

    console.log("Collecting video links and titles");
    const videos = await collectVideoLinksAndTitlesWithTimeout(
      page,
      numberOfResults,
      checkGlobalTimeout,
      subscriberCount,
      channel
    );

    console.log("Scraping video details");
    // Scrape video details
    await scrapeVideosDetails(videos, browser, checkGlobalTimeout);

    console.log("Closing browser");
    // Close the browser
    await browser.close();

    console.log("Returning video data");
    return videos;
  } catch (err) {
    console.log("Error occurred:", err);
  }
}
async function collectVideoLinksAndTitlesWithTimeout(
  page,
  numberOfResults,
  checkGlobalTimeout,
  subscriberCount,
  channel
) {
  const maxRetries = 3;
  const timeoutDuration = 60 * 1000; // 1 minute in milliseconds

  for (let retryCount = 0; retryCount < maxRetries; retryCount++) {
    try {
      const result = await Promise.race([
        collectVideoLinksAndTitles(
          page,
          numberOfResults,
          checkGlobalTimeout,
          subscriberCount,
          channel
        ),
        new Promise((_, reject) =>
          setTimeout(
            () => reject(new Error("Timeout exceeded")),
            timeoutDuration
          )
        ),
      ]);
      return result;
    } catch (error) {
      console.error(`Attempt ${retryCount + 1} failed: `, error);
      if (retryCount < maxRetries - 1) {
        console.log("Retrying...");
      } else {
        throw new Error(
          "Failed to collect video links and titles after retries"
        );
      }
    }
  }
}

async function collectVideoLinksAndTitles(
  page,
  numberOfResults,
  checkGlobalTimeout,
  subscriberCount,
  channel
) {
  let uniqueVideoLink = [];
  let titles = [];
  let uniqueCoverImages = [];

  while (
    uniqueVideoLink.length < numberOfResults ||
    titles.length < numberOfResults ||
    uniqueCoverImages.length < numberOfResults
  ) {
    checkGlobalTimeout();
    await page.evaluate(() => {
      window.scrollBy(0, window.innerHeight);
    });
    await page.waitForTimeout(1000);

    titles = await page.$$eval("#contents h3 > a #video-title", (titleLinks) =>
      titleLinks.map((link) => link.textContent.trim())
    );

    const videoLinks = await page.$$eval("#contents h3 > a", (videoLinks) =>
      videoLinks.map(
        (link) => "https://www.youtube.com" + link.getAttribute("href")
      )
    );
    const coverImages = await page.$$eval(
      "#contents #thumbnail img",
      (images) => images.map((image) => image.getAttribute("src"))
    );

    uniqueVideoLink = Array.from(new Set(videoLinks));
    titles = Array.from(new Set(titles));
    uniqueCoverImages = Array.from(new Set(coverImages));

    if (uniqueVideoLink.length >= numberOfResults) {
      uniqueVideoLink.splice(numberOfResults);
    }

    if (uniqueCoverImages.length >= numberOfResults) {
      uniqueCoverImages.splice(numberOfResults);
    }

    if (titles.length >= numberOfResults) {
      titles.splice(numberOfResults);
    }
  }

  console.log("Creating video objects");
  // Create video objects
  const videos = titles.map((title, i) => ({
    title: title,
    author: channel,
    videoUrl: uniqueVideoLink[i],
    coverImage: uniqueCoverImages[i],
    subscriberCount: parseSubscriberCount(subscriberCount),
    viewCount: null,
    description: null,
    commentCount: null,
    id: uniqueVideoLink[i].split("=")[1],

    likeCount: null,
  }));

  return videos;
}

function parseSubscriberCount(subscriberCount) {
  const countMatch = subscriberCount.match(/[\d.,]+/);
  const unitMatch = subscriberCount.match(/[kKmM]|&nbsp;/);

  if (!countMatch || !unitMatch) {
    return null;
  }

  const count = parseFloat(countMatch[0].replace(",", "."));
  let unit = unitMatch[0];

  // Check if the unit is a non-breaking space, and find the correct unit (k or M)
  if (unit === "&nbsp;") {
    if (subscriberCount.includes("k&nbsp;")) {
      unit = "k";
    } else if (subscriberCount.includes("M&nbsp;")) {
      unit = "M";
    }
  }

  switch (unit) {
    case "k":
    case "K":
      return count * 1_000;
    case "M":
    case "m":
      return count * 1_000_000;
    default:
      return null;
  }
}

// Scrape details for each video
async function scrapeVideosDetails(videos, browser, checkGlobalTimeout) {
  const batchSize = 4;
  const numberOfBatches = Math.ceil(videos.length / batchSize);

  for (let batchIndex = 0; batchIndex < numberOfBatches; batchIndex++) {
    checkGlobalTimeout();
    const startIndex = batchIndex * batchSize;
    const endIndex = startIndex + batchSize;
    const batch = videos.slice(startIndex, endIndex);

    console.log(
      `Scraping details for batch ${batchIndex + 1}/${numberOfBatches}`
    );
    const tasks = batch.map((video) =>
      scrapeVideoDetailsWithRetries(video, browser, checkGlobalTimeout)
    );

    console.log("Waiting for all tasks in batch to complete");
    await Promise.allSettled(tasks);
  }

  console.log("Removing duplicate videos");
  videos = removeDuplicateVideos(videos);
}

// Scrape video details with retries
async function scrapeVideoDetailsWithRetries(
  video,
  browser,
  checkGlobalTimeout
) {
  const maxRetries = 3;
  const page = await browser.newPage();
  let success = false;

  while (maxRetries > 0) {
    checkGlobalTimeout();
    try {
      await scrapeVideoDetails(video, page);
      success = true;
      break;
    } catch (error) {
      console.error(`Failed to scrape video ${video.videoUrl}`, error);
      maxRetries--;
      await page.waitForTimeout(2000);
    }
  }

  await page.close();
  return success;
}

// Scrape details for a single video
async function scrapeVideoDetails(video, page, maxRetries = 3) {
  await page.setRequestInterception(true);
  page.on("request", (request) => {
    if (
      request.resourceType() === "media" ||
      request.url().includes("videoplayback")
    ) {
      request.abort();
    } else {
      request.continue();
    }
  });
  await page.goto(video.videoUrl, {
    waitUntil: "domcontentloaded",
    timeout: 120000,
  });

  await page.waitForSelector("#expand");
  await page.evaluate(() => {
    window.scrollBy(0, window.innerHeight / 2);
  });
  await page.waitForTimeout(1000);

  await page.waitForSelector("#segmented-like-button", { timeout: 120000 });
  const likes = await page.$eval(
    "#segmented-like-button button",
    (el) => el.getAttribute("aria-label").match(/\d+/g)?.join("") || ""
  );
  video.likeCount = parseInt(likes);

  // Retry description scraping in case of failure
  let descriptionRetries = maxRetries;
  let descriptionSuccess = false;

  while (descriptionRetries > 0) {
    try {
      await page.waitForSelector("#expand", { timeout: 3000 });
      const expandBtn = await page.$("#expand");

      // Use page.$eval to check if the "hidden" attribute is present on the expandBtn element
      const isHidden = await page.$eval("#expand", (el) =>
        el.getAttribute("hidden")
      );

      if (expandBtn && !isHidden) {
        await expandBtn.click();
      }
      await page.waitForTimeout(1000);
      await page.waitForSelector("#description-inline-expander");
      const descriptionSelector = await page.$(
        "#description-inline-expander .yt-core-attributed-string"
      );
      if (!descriptionSelector) {
        video.description = "";
      } else {
        const description = await page.$eval(
          "#description-inline-expander .yt-core-attributed-string",
          (el) => (el ? el.textContent.replace(/\s+/g, " ").trim() : "")
        );
        video.description = description;
      }
      descriptionSuccess = true;

      const date = await page.$eval('meta[itemprop="datePublished"]', (el) =>
        el.getAttribute("content")
      );

      video.publishedAt = date;

      break;
    } catch (error) {
      descriptionRetries--;
      console.error(
        `Failed to scrape description for ${video.videoUrl}`,
        error
      );
      await page.waitForTimeout(2000);
    }
  }

  if (!descriptionSuccess) {
    throw new Error(
      `Failed to scrape description after ${maxRetries} retries.`
    );
  }

  await page.waitForSelector('meta[itemprop="interactionCount"]');
  const views = await page.$eval('meta[itemprop="interactionCount"]', (el) =>
    el.getAttribute("content")
  );
  video.viewCount = parseInt(views);
  await page.evaluate(() => {
    window.scrollBy(0, window.innerHeight * 6);
  });
  await page.waitForTimeout(1000);
  await page.waitForSelector("#title #count span");
  const commentsCount = await page.$eval(
    "#title #count span",
    (el) => el.textContent.match(/\d+/g)?.join("") || ""
  );
  video.commentCount = parseInt(commentsCount);
}

// Remove duplicate videos
function removeDuplicateVideos(videos) {
  const uniqueVideos = videos.reduce((accumulator, current) => {
    if (
      !accumulator.some(
        (video) => video.videoUrl && video.videoUrl === current.videoUrl
      )
    ) {
      accumulator.push(current);
    }
    return accumulator;
  }, []);

  return uniqueVideos;
}

export { scrape };


src/scrapper/youtube.js

Unlock valuable insights with our tool that extracts crucial video data such as title, description, cover image URL, video URL, author, view count, like count, comment count, and channel subscriber count. Enhance your marketing and content creation strategies with informed, data-driven decisions.

Youtube Channel

Youtube Channel · Apify

API · Youtube Channel · Apify

Input · Youtube Channel · Apify

Related actors · Youtube Channel · Apify

Unlock valuable insights with our tool that extracts crucial video data such as title, description, cover image URL, video URL, author, view count, like count, comment count, and channel subscriber...

YouTube Scraper using Puppeteer

Overview

Input

Output

Usage

Example Input

Example Output

Disclaimer

Where next?