root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

.eslintrc

# This file tells Git which files shouldn't be added to source control

.idea
node_modules


.gitignore

# First, specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:18-beta

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --only=prod --no-optional --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Optionally, specify how to launch the source code of your actor.
# By default, Apify's base Docker images define the CMD instruction
# that runs the Node.js source code using the command specified
# in the "scripts.start" section of the package.json file.
# In short, the instruction looks something like this:
#
# CMD npm start


Dockerfile

How many times the request should be retried before giving up

Maximum request retries

Proxy

Video urls that contains video publicly available

Video URLs

Input schema for the apify_project actor.

INPUT_SCHEMA.json

# Youtube video downloader

Download multiple videos embedded on public posts

## Supported URL formats

* `https://www.youtube.com/watch?v=`...
* `https://youtu.be/...`
* `https://m.youtube.com/...`

## Custom data

If you need to pass custom data to the output, set the `userData` object of the request in the `startUrls` array and it will be appended to the output

```json
{
    "startUrls": [{
        "url": "https://www.youtube.com/...",
        "userData": {
            "userid": "6236572396729"
        }
    }]
}
```

## Proxies

To be able to access Youtube, RESIDENTIAL proxy group is required. You can provide your own proxy for the download itself.
It can take from 2 to 360 seconds to download depending on the size of the video and the quality of the proxy provided.

README.md

{
    "env": { "npm_config_loglevel": "silent" }
}


apify.json

import { Actor } from 'apify';
import { BasicCrawler, downloadListOfUrls, log, utils, CriticalError } from 'crawlee';
import { gotScraping } from 'got-scraping';
import { createHash } from 'crypto';
import { Readable, PassThrough, Writable, pipeline, Stream } from 'stream';
import simpleGet from 'simple-get';
import ProxyAgent from 'proxy-agent';

const createValidKVNameFromUrl = (url) => createHash('sha256').update(url).digest('hex').slice(0, -2);

const deferred = () => {
    let _reject;
    let _resolve;

    const promise = new Promise((r1, r2) => {
        _resolve = (e) => {
            setTimeout(() => r1(e));
        };
        _reject = (e) => {
            setTimeout(() => r2(e));
        };
    });

    return {
        promise,
        resolve(v) {
            _resolve?.(v);
        },
        reject(e) {
            _reject?.(e);
        }
    }
}

/**
 * A passthrough stream that aborts when nothing is
 * happening for a while, or the stream is too slow
 *
 * @param {number} timeoutMillis
 */
const passthroughAbortStream = (timeoutMillis) => {
    let lastActivity = Date.now();
    let finished = false;

    const passthrough = new PassThrough({
        autoDestroy: true,
        emitClose: true,
        // highWaterMark: 1024 * 64,
        transform(chunk, _encoding, callback) {
            lastActivity = Date.now();
            callback(null, chunk);
        },
    });

    const f = () => {
        if (Date.now() - lastActivity >= timeoutMillis) {
            clear();
            passthrough.destroy(new Error(`Aborted slow/stuck stream after ${timeoutMillis}ms`));
        } else if (!finished) {
            setTimeout(f, 1000);
        }
    };

    setTimeout(f);

    const clear = () => {
        finished = true;
    };

    passthrough
        .once('close', clear)
        .once('end', clear)
        .once('error', clear);

    return passthrough;
};

const getInputUrls = async (requests, proxy, requestQueue) => {
    const isValid = (u) => {
        return (u.includes('youtube.') && u.includes('?v=')) || u.includes('youtu.be');
    }

    for (const { url, requestsFromUrl, userData } of requests) {
        const urls = [];
        
        if (requestsFromUrl) {
            urls.push(
                ...await downloadListOfUrls({
                    url: requestsFromUrl, 
                    proxyUrl: await proxy.newUrl()
                }),
            );
        } else {
            urls.push(url);
        }

        await requestQueue.addRequests(urls.filter(isValid).map((url) => ({ url, userData })));
    }
};

const extractUrl = (body) => {
    const matches = body.match(/"formats":(\[[^\]]+\])/);
    let output = '';

    if (matches?.[1]) {
        output = matches[1];
    }

    if (output) {
        output = JSON.parse(output);
    }

    if (output?.length) {
        // console.dir(output, { depth: Infinity });
        return output.reduce((out, format) => {
            if (
                format?.height > out.height && 
                format?.mimeType?.includes?.('video/mp4') &&
                format?.audioChannels >= 2
            ) {
                out.height = format.height;
                out.url = format.url;
            }
            return out;
        }, { height: -Infinity, url: undefined })?.url;
    }
}

await Actor.init();

const { 
    startUrls = [], 
    proxy, 
    maxRequestRetries = 10,
} = await Actor.getInput();

const videoProxy = await Actor.createProxyConfiguration(proxy);
const requestQueue = await Actor.openRequestQueue();

await getInputUrls(
    startUrls, 
    videoProxy, 
    requestQueue,
);

if (await requestQueue.isEmpty()) {
    throw new Error('You need to provide valid urls to scrape');
}

const pageProxy = await Actor.createProxyConfiguration({
    groups: ['RESIDENTIAL'],
    countryCode: 'US'
});

const getVideo = async (url, proxyUrl) => {
    return new Promise((resolve, reject) => {
        simpleGet({
            url,
            timeout: 595 * 1000,
            agent: new ProxyAgent(proxyUrl),
        }, (err, res) => {
            if (err) {
                return reject(err);
            }
            resolve(res);
        });
    });
    // return new Promise((resolve, reject) => {
    //     simpleGet.concat({
    //         url,
    //         timeout: 595 * 1000,
    //         agent: new ProxyAgent(proxyUrl),
    //     }, (err, res, data) => {
    //         if (err) {
    //             return reject(err);
    //         }
    //         resolve(data);
    //     });
    // });
}

const abortController = new AbortController();

Actor.on('migrating', () => abortController.abort());
Actor.on('aborting', () => abortController.abort());

const crawler = new BasicCrawler({
    requestQueue,
    sessionPoolOptions: {
        sessionOptions: {
            maxErrorScore: 0.5,
            maxUsageCount: 1,
        }
    },
    maxRequestRetries,
    requestHandlerTimeoutSecs: 800,
    async requestHandler({ request, session }) {
        const { video = false, originalUrl, sessionId, ...rest } = request.userData;

        if (!video) {
            const response = await gotScraping({
                url: request.url,
                useHeaderGenerator: true,
                proxyUrl: await (video ? pageProxy : videoProxy).newUrl(session.id),
                isStream: video,
                timeout: {
                    request: 350 * 1000,
                    socket: 30000,
                    connect: 5000,
                    secureConnect: 5000,
                    response: 10000,
                },
                retry: {
                    limit: 0
                },
            });
            
            const { body } = response;
            const videoUrl = extractUrl(body);

            if (!videoUrl) {
                session.retire();
                throw new Error(`Couldn't find a video in the provided URL`);
            }

            await crawler.addRequests([{
                url: videoUrl,
                userData: {
                    ...rest,
                    originalUrl: request.url, 
                    sessionId: session.id,
                    video: true,
                },
            }]);
        } else {
            const hash = createValidKVNameFromUrl(originalUrl);
            const { defaultKeyValueStoreId } = Actor.getEnv();

            const videoUrl = `https://api.apify.com/v2/key-value-stores/${defaultKeyValueStoreId}/records/${hash}`;

            log.info(`Downloading from ${originalUrl}`, { hash });

            const response = await getVideo(request.url, await videoProxy?.newUrl(sessionId ?? session.id));

            const startTime = Date.now();
            const { promise, reject } = deferred();
            
            response.on('error', reject);
            
            const passthrough = passthroughAbortStream(20000);
            passthrough.on('error', reject);

            const stream = pipeline(
                response,
                passthrough,
                (err) => {
                    if (err) {
                        reject(err);
                    }
                }
            );

            await Promise.race([
                promise,
                Actor.setValue(hash, stream/*response*/, { contentType: 'video/mp4' }),
            ]);

            await Actor.pushData({
                url: originalUrl,
                videoUrl,
                userData: rest,
                error: false,
            });

            log.info(`Download finished in ${(Date.now() - startTime) / 1000} seconds`, { videoUrl, originalUrl });
        }
    },
    async failedRequestHandler({ request }) {
        const { originalUrl, video, ...rest } = request.userData;
        
        await Actor.pushData({
            url: originalUrl,
            videoUrl: null,
            userData: { ...rest }, 
            error: true,
        });
    }
});

const totalCount = (await Actor.getValue('TOTAL_COUNT')) || requestQueue.assumedTotalCount;  

await Actor.setValue('TOTAL_COUNT', totalCount);

log.info(`Got ${totalCount} links`);

await crawler.run();

if (abortController.signal.aborted) {
    await new Promise(() => {}); // wait for migration
}

await utils.sleep(10000);

await Actor.exit();


Youtube video downloader

Supported URL formats

Custom data

Proxies

Industries

Where next?