# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

.env.copy

.gitignore

actor

package.json

# Specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-playwright-chrome:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent


.actor/Dockerfile

Vimeo Apify Actor
=================

# description

Get from a Vimeo page :

    title
    video_count
    video_count_collected
    pagination_max
    status
    timestamp
    
and an array of videos :

    id
    url
    image

# input

Url input format is : https://vimeo.com/channel

pagination_max : Number of load more sections to load

Set No timout (or >=240) and Memory 4096 MB. 

# result

    [
        {
            title: 'page',
            video_count: 359,
            video_count_collected: 50,
            pagination_max: 4,
            status: false,
            timestamp: 1679600102334,
            videos: [
                {
                    id: '11111111',
                    url: 'https://www.vimeo.com/11111111',
                    image: 'https://i.vimeocdn.com/id/11111111'
                },
                {
                    id: '11111112',
                    url: 'https://www.vimeo.com/11111112',
                    image: 'https://i.vimeocdn.com/id/11111112'
                },
                {
                    id: '11111113',
                    url: 'https://www.vimeo.com/11111113',
                    image: 'https://i.vimeocdn.com/id/11111113'
                },
                {
                    id: '11111114',
                    url: 'https://www.vimeo.com/11111114',
                    image: 'https://i.vimeocdn.com/id/11111114'
                },

                ...
            ]
        }
    ]

# Legal Mentions and Terms of Use

This actor is provided 'as is' and without warranties of any kind, either express or implied. There is no guarantee for the accuracy, completeness, reliability, or suitability of this actor for any particular purpose. You use this actor at your own risk.


.actor/README.md

vimeo

Vimeo

.actor/actor.json

Vimeo Page

.actor/dataset_schema.json

pagination_max

Vimeo page url, format : https://www.vimeo.com/page

data

.actor/input_schema.json

import dotenv, { parse } from 'dotenv'
dotenv.config()
let dev = typeof (process.env.APP_ENVIRONMENT) !== 'undefined' ? process.env.APP_ENVIRONMENT : false

import { Actor } from 'apify';
import { PlaywrightCrawler, Configuration, Dataset } from "crawlee";

await Actor.init()

// For more information, see https://sdk.apify.com/

const input = await Actor.getInput();

let url, pagination_max
url = typeof (input?.url) == 'undefined' ? "https://vimeo.com/wearepixelartworks" : input?.url
pagination_max = typeof (input?.pagination_max) == 'undefined' ? 10 : input?.pagination_max
//console.log('---- ' + pagination_max);
if (dev) console.log('---- ' + pagination_max);
if (dev) console.log('url: ', url);
if (dev) console.log('----');

if (dev) Configuration.set('headless', false); // dev

const crawler = new PlaywrightCrawler({
    requestHandlerTimeoutSecs: 240,
    requestHandler: async ({ request, page, infiniteScroll, enqueueLinks }) => {

        if (dev) page.setViewportSize({ width: 1400, height: 900 }); // dev

        if (dev) await page.waitForTimeout(1000);
        else await page.waitForTimeout(1000);

        // onetrust-accept-btn-handler
        if (dev) await page.getByRole('button', { name: 'Accept All Cookies' }).click();

        let title = await page.title()

        /**
         * build//test
         */
        //dev = true//false

        if (dev) console.log(title);
        if (dev) console.log('----');

        async function getByText(page, text) {
            try {
                await page.waitForSelector(`text=${text}`, { state: 'visible', timeout: 500 });
                return true;
            } catch (e) {
                return false;
            }
        }

        /**
         * infiniteScroll
         * 
         * until links.length == video_count
         * 
         */
        let blocks = 0, hit = 0, timer = 500
        await infiniteScroll({
            stopScrollCallback: async () => {
                await page.waitForTimeout(timer)
                timer = timer + Math.floor(Math.random() * 10)
                const elementExists = await getByText(page, 'Load more');
                //console.log(elementExists)
                if (elementExists || (elementExists && (blocks < pagination_max))) {
                    const links = await page.getByText('Load more');
                    console.log('--> ' + blocks)
                    if (links) {
                        await links.click()
                        blocks++
                        return blocks > pagination_max
                    }
                } else {
                    //console.log('exit!!')
                    return true
                }
            },
        })

        if (dev) console.log(`
        
>>>>>>>>>>>>>>>>>> parse
        
`);

        const video_count_block = await page.locator('text=/\\d+ video/').first()
        let video_count = await video_count_block.textContent();
        video_count = parseInt(video_count.replace('videos', '').replace('video', '').trim())

        if (dev) console.log(`----------- video_count

    ${video_count}

    `)

        const blocks_master = await page.$$('article');

        if (dev) console.log(`
        
======================= blocks_master > ${blocks_master.length}
        
        `)

        let img, srcset, id, ids = [], data = [], obj = {}

        for (const blocks_scrollable of blocks_master) {

            obj = {}
            img = null
            id = null

            const video_image = await blocks_scrollable.$$('img')

            for (const video_image_src of video_image) {
                img = await video_image_src.getAttribute('src');
                //srcset = await video_image_src.getAttribute('srcset');
                //console.log(srcset)
                obj['image'] = img
            }

            const video_link = await blocks_scrollable.$$('a')

            for (const video_link_a of video_link) {
                id = await video_link_a.getAttribute('href');
                id = id.replace('/', '')

                if (id.includes('http')) {
                    id = null
                } else {
                    obj['id'] = id
                    obj['url'] = 'https://vimeo.com/' + id
                }
            }

            if (img && id && !ids.includes(id)) {
                ids.push(id)
                data.push(obj)
            }


        }

        let status = false
        if (video_count == data.length) { status = true }

        let actor_data = {
            title: title,
            // channel_name: channel_name,
            // channel_handle: channel_handle,
            // tagline: tagline,
            // subscribers: count,
            video_count: video_count,
            video_count_collected: data.length,
            pagination_max: pagination_max,
            status: status,
            timestamp: Date.now(),
            videos: data,
        }

        if (dev) console.log(actor_data)

        await Actor.pushData(actor_data);

    }

})

await crawler.run([url]);
await Actor.exit();


Vimeo Apify Actor

description

input

result

Legal Mentions and Terms of Use

Where next?