const { log } = require('crawlee');

exports.handleStart = async ({ request, page }) => {
    // Handle Start URLs
};

exports.handleList = async ({ request, page }) => {
    // Handle pagination
};

exports.handleDetail = async ({ request, page }) => {
    // Handle details
};


src/routes.js

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.idea
node_modules


.gitignore

# First, specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-playwright-chrome:16

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --only=prod --no-optional --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Optionally, specify how to launch the source code of your actor.
# By default, Apify's base Docker images define the CMD instruction
# that runs the Node.js source code using the command specified
# in the "scripts.start" section of the package.json file.
# In short, the instruction looks something like this:
#
# CMD npm start


Dockerfile

Max results to return; use this to limit long-running search scrapes

Max Results

Search Terms

Twitch Search Scraper

INPUT_SCHEMA.json

{
     "env": { "npm_config_loglevel": "silent" }
}


apify.json

// This is the main Node.js source code file of your actor.

// Import Apify SDK. For more information, see https://sdk.apify.com/
const { Actor } = require('apify');
const { chromium } = require('playwright');  // Or 'firefox' or 'webkit'.


async function scrollLastInQueryIntoView(page, selector) {
    await page.evaluate((s) => {
        // Special bonus for those using jQuery
        let elements = [...document.querySelectorAll(s)];

        elements[elements.length-1].scrollIntoView();
    },selector)
}
function sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
}

async function scrapeVideos(page, scrapeOptions){
    let videoData = await page.evaluate(() => {
        let videoCards = [...document.querySelectorAll('div.search-results [data-a-target="search-result-video"]:not([data-apify-scraped])')];
        return videoCards.map((el) => {
            el.setAttribute('data-apify-scraped','true');
            let videoObject = {
                title: el.querySelector('[data-test-selector="search-result-video__title"]')?.innerText,
                channel: el.querySelector('[data-test-selector="search-result-video__name"]')?.innerText,
                category: el.querySelector('[data-test-selector="search-result-video__category"]')?.innerText,
                views: el.querySelector('[data-test-selector="search-result-video__viewer-count"]')?.innerText,
                //tags: [...el.querySelectorAll('[data-test-selector="search-result-video__tags"] .tw-tag')].map((tag) => tag.innerText)
            };

            return videoObject
        });
    });

    for (var i = 0; i < videoData.length; i++){
        if(scrapeOptions.currentResultCount >= scrapeOptions.maxResults){
            break;
        }
        Actor.pushData(videoData[i]);
        scrapeOptions.currentResultCount = scrapeOptions.currentResultCount + 1;
    }
}

// Start function
Actor.main(async () => {
    const { maxResults, searchTerms } = await Actor.getInput();
    let scrapeOptions = {
        maxResults: maxResults,
        currentResultCount: 0
    };
    const browser = await chromium.launch({ headless: false });
    
    console.log('launched!');
    
    // Open a new page
    const page = await browser.newPage();
    
    // Navigate
    const videoSearchUrl = `https://www.twitch.tv/search?type=videos&term=${encodeURIComponent(searchTerms)}`;
    await page.goto(videoSearchUrl);

    await Promise.any([
        page.waitForSelector('div.search-results [data-a-target="search-result-video"]'), 
        page.waitForSelector('[data-test-selector="search-error-title"]')
    ]);

    const searchError = await page.$('[data-test-selector="search-error-title"]');
    if (searchError) {
        console.log('No results retrieved.');
        // TODO
    }

    // Scrape Current View
    scrapeVideos(page, scrapeOptions);
    let lastResultCount = scrapeOptions.currentResultCount;
    const maxRetries = 30;
    let retries = maxRetries;
    while(retries > 0 && scrapeOptions.currentResultCount < scrapeOptions.maxResults){
        await scrollLastInQueryIntoView (page, 'div.search-results [data-a-target="search-result-video"]')

        scrapeVideos(page, scrapeOptions);
        if(lastResultCount == scrapeOptions.currentResultCount){
            retries--;
            await sleep(100);
        }
        
        lastResultCount = scrapeOptions.currentResultCount;
        retries = maxRetries;
    }

    await browser.close();
});


Twitch Video Search Scraper

Where next?