# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage

# Added by Apify CLI
.venv


.gitignore

youtube-subtitles

package.json

Extract subtitles (captions) from one or multiple YouTube videos. Download YouTube video subtitles (captions) as a HTML, JSON, CSV, Excel, or XML doc.

youtube-subtitles-scraper

YouTube Video Subtitles (captions) Scraper

.actor/actor.json

Overview

.actor/dataset_schema.json

# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent


.actor/Dockerfile

Subtitles language code. E.g. en | es | fr | de

Language

Start URLs

Fetch auto-generated captions if no user submitted

Fetch auto-generated

Youtube video subtitles (caption) scraper

.actor/input_schema.json

/**
 * This template is a production ready boilerplate for developing with `CheerioCrawler`.
 * Use this to bootstrap your projects using the most up-to-date code.
 * If you're looking for examples or want to learn more, see README.
 */

// For more information, see https://docs.apify.com/sdk/js
import { Actor } from 'apify';
// For more information, see https://crawlee.dev
import { CheerioCrawler } from 'crawlee';
import { router } from './routes.js';

// Initialize the Apify SDK
await Actor.init();

const startUrls = [];

const input = await Actor.getInput();

for (const video of input.startUrls) {
    startUrls.push({
        url: video.url,
        userData: { ...input },
    });
}

// const proxyConfiguration = await Actor.createProxyConfiguration();

const crawler = new CheerioCrawler({
    // proxyConfiguration,
    requestHandler: router,
});

await crawler.run(startUrls);

// Exit successfully
await Actor.exit();


src/main.js

import { Dataset, createCheerioRouter } from 'crawlee';
import he from 'he';
import striptags from 'striptags';

export const router = createCheerioRouter();

router.addDefaultHandler(async ({ request, crawler, $, log }) => {
    const input = request.userData;
    const title = $('title').text().replace(' - YouTube', '');
    log.info(`Crawling video: ${title}`, { url: request.loadedUrl });
    const scripts = $('body > script').get();
    const scriptData = await getScriptData(scripts);

    const videoData = scriptData.videoDetails;
    videoData.videoUrl = request.loadedUrl;

    const captionUrl = await getLangCaprionUrl(scriptData.captions.playerCaptionsTracklistRenderer.captionTracks, input.language, input.useAsr);
    if (captionUrl) {
        await crawler.requestQueue.addRequest({
            url: captionUrl,
            userData: {
                label: 'captions',
                videoData,
            },
        });
    } else {
        log.error(`No subtitles found for for video ${title} with language '${input.language}' and ASR '${input.useAsr}'`);
        log.error(`Try repeat with other INPUT parameters`);
    }
});

router.addHandler('captions', async ({ body, request }) => {
    const lines = await getCaption(body, request.userData.videoData);
    await Dataset.pushData(lines);
});

async function getScriptData(scripts) {
    for (const script of scripts) {
        for (const scriptChildern of script.children) {
            const scriptData = scriptChildern.data;
            if (scriptData) {
                if (scriptData.includes('captionTracks')) {
                    const regex = /var ytInitialPlayerResponse = ({.*?});/;
                    const match = regex.exec(scriptData);
                    return JSON.parse(match[1]);
                }
            }
        }
    }
}

async function getLangCaprionUrl(captions, lang, takeAsr) {
    const langCaptions = [];
    for (const caption of captions) {
        if (!lang) {
            lang = 'en';
        }
        if (caption.languageCode === lang) {
            langCaptions.push(caption);
        }
    }
    if (!langCaptions.length) {
        return null;
    }
    if (langCaptions.length >= 1) {
        for (const langCaption of langCaptions) {
            if (!langCaption.hasOwnProperty('kind')) {
                return langCaption.baseUrl;
            }
        }
    }
    if (takeAsr) {
        return langCaptions[0].baseUrl;
    }
    if (!takeAsr) {
        if (langCaptions[0].kind !== 'asr') {
            return langCaptions[0].baseUrl;
        }
        return null;
    }
}

async function getCaption(transcript, videoData) {
    const lines = transcript
        .replace('<?xml version="1.0" encoding="utf-8" ?><transcript>', '')
        .replace('</transcript>', '')
        .split('</text>')
        .filter((line) => line && line.trim())
        .map((line) => {
            const startRegex = /start="([\d.]+)"/;
            const durRegex = /dur="([\d.]+)"/;

            const [, start] = startRegex.exec(line);
            const [, duration] = durRegex.exec(line);

            const htmlText = line
                .replace(/<text.+>/, '')
                .replace(/&amp;/gi, '&')
                .replace(/<\/?[^>]+(>|$)/g, '');

            const decodedText = he.decode(htmlText);
            const text = striptags(decodedText);

            return {
                videoId: videoData.videoId,
                videoUrl: videoData.videoUrl,
                videoTitle: videoData.title,
                videoLength: videoData.lengthSeconds,
                videoDescription: videoData.shortDescription,
                videoKeywords: videoData.keywords,
                author: videoData.author,
                start,
                duration,
                text,
            };
        });

    return lines;
}


Where next?