# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-playwright-chrome:16 AS builder

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json ./

# Install all dependencies. Don't audit to speed up the installation.
RUN npm install --include=dev --audit=false

# Next, copy the source files using the user set
# in the base image.
COPY --chown=myuser . ./

# Install all dependencies and build the project.
# Don't audit to speed up the installation.
RUN npm run build

# Create final image
FROM apify/actor-node-playwright-chrome:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Copy built JS files from builder image
COPY --from=builder --chown=myuser /home/myuser/dist ./dist

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY --chown=myuser . ./


# Run the image. If you know you won't need headful browsers,
# you can remove the XVFB start script for a micro perf gain.
CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent


.actor/Dockerfile

Crawlee and Playwright project in typescript.

web-traffic-generator

Web traffic generator

.actor/actor.json

🧑🏻‍🤝‍🧑🏻 Simultaneous visitors

Crawl website for give visits to all pages.

🕸️ Crawl website for give visits to all pages

Country of proxies to be used. You need to enable datacenter or residential proxies to use this option.

🇧🇷 Proxies country

🔗 Start URLs

If you want to use desktop devices, set this to true.

🖥️ Use desktop devices

If you want to use mobile devices, set this to true.

📱 Use mobile devices

If you want to use residential proxies, set this to true.

🌐🏠 Use residential proxies

If you want to use tablet devices, set this to true.

📟 Use tablet devices

#️⃣ Number of visitors

The amount of time the bot will stay on the page.

⌛ The amount of time it takes for the bot to stay scrolled on the page

PlaywrightCrawler Template

.actor/input_schema.json

import { CheerioCrawler } from 'crawlee';
import { UrlsManager } from './urls-manager.js';
import { LoggerType, getLogger } from './logger.js';

// export const getSiteAllUrls = async (url: string): Promise<UrlsManager> => {
//   const crawledUrls = UrlsManager.new([url]);

//   const crawler = new CheerioCrawler({
//     async requestHandler({ request, enqueueLinks }) {
//       const currentUrl = request.url;
//       crawledUrls.pushUrl(currentUrl);
//       await enqueueLinks({ baseUrl: currentUrl });
//     },
//     maxRequestsPerCrawl: 99, //  TO CHANGE TO INPUT PARAM
//   });

//   // Run the crawler
//   await crawler.run([url]);

//   return crawledUrls;
// };

export class Crawler {
  private crawler: CheerioCrawler;
  protected log: LoggerType;

  public urls: UrlsManager;

  constructor(
    private baseUrls: string[],
    private maxRequestsPerCrawl: number,
  ) {
    this.log = getLogger(`[CRAWLER]`);
    this.urls = UrlsManager.new(this.baseUrls);
    this.crawler = new CheerioCrawler({
      requestHandler: async ({ request, enqueueLinks }) => {
        const currentUrl = request.url;
        this.log.info(`crawling ${currentUrl}`);
        this.urls.pushUrl(currentUrl);
        await enqueueLinks({ baseUrl: currentUrl });
      },
      maxRequestsPerCrawl: this.maxRequestsPerCrawl,
    });
  }

  static new = (urls: string[], maxRequestsPerCrawl: number) => {
    return new Crawler(urls, maxRequestsPerCrawl);
  };

  async run() {
    this.log.info(`running crawler`);
    await this.crawler.run(this.urls.urls);
    this.log.info(`crawler finished : ${this.urls.length} urls crawled`);
    this.urls.show();
    return this.urls;
  }
}


src/crawler.ts

import { Logger, ILogObj } from 'tslog';

export type LoggerType = Logger<ILogObj>;

export const getLogger = (prefix: string): Logger<ILogObj> => new Logger({
  prettyLogTemplate: '{{hh}}:{{MM}}:{{ss}} {{logLevelName}}\t[{{fileNameWithLine}}{{name}}]\t',
  stylePrettyLogs: true,
  minLevel: 0,

  prefix: [prefix],
});

export function testLog() {
  const logger = getLogger('test');
  logger.silly('Silly log');
  logger.debug('Debug log');
  logger.error('Error log');
  logger.fatal('Fatal log');
  logger.info('Info log');
  logger.trace('Trace log');
  logger.warn('Warn log');
}


src/logger.ts

// /**
//  * This template is a production ready boilerplate for developing with `PlaywrightCrawler`.
//  * Use this to bootstrap your projects using the most up-to-date code.
//  * If you're looking for examples or want to learn more, see README.
//  */

// // For more information, see https://docs.apify.com/sdk/js
// import { Actor } from 'apify';
// // For more information, see https://crawlee.dev
// import { ProxyConfiguration, launchPlaywright } from 'crawlee';
// import { devices } from 'playwright';
// import { Utils } from './utils.js';

// // Initialize the Apify SDK
// await Actor.init();

// type Input = {
//     startUrls: { url: string}[];
//     chunkSize: number;
//     visitsNumber: number;
//     waitSecondsInterval: number;
// }

// const input = await Actor.getInput<Input>();

// try {
//   // const proxyConfiguration = await Actor.createProxyConfiguration({
//   //     groups: ['RESIDENTIAL'],
//   // }) as ProxyConfiguration;
//   // const proxyUrl = await proxyConfiguration.newUrl('ggggg');
//   // https://api.ipify.org?format=json

//   const getPage = async (duration: number) => {
//     console.log('new page');

//     const randomDevice = Object.values(devices)[Math.floor(Math.random() * Object.values(devices).length)];
//     const browser = await launchPlaywright({
//       launchOptions: { headless: false },
//       userAgent: randomDevice.userAgent,
//       // proxyUrl: await proxyConfiguration.newUrl(),
//     });

//     try {
//       const page = await browser.newPage();
//       // https://iwpft.cariz.fr
//       await page.goto(input?.startUrls?.[0]?.url || 'https://iwpft.cariz.fr');

//       // wait page load
//       await page.waitForLoadState('domcontentloaded');
//       console.log('page loaded');

//       // await page.waitForURL('https://monordo.com');
//       // await page.waitForTimeout(10000);

//       const startNavigation = Date.now();
//       const maxEndTime = startNavigation + duration;
//       const clickableElements = await page.$$(
//         'a, button, input[type="button"], input[type="submit"], [role="button"]',
//       );
//       const links = await Promise.all(clickableElements.map(async (el) => ({ el, href: await el.evaluate((node) => node.getAttribute('href')) })));
//       const toClick = links.find((l) => l.href && l.href.startsWith('https://iwpft.cariz.fr'))
//             || links.find((l) => l.href && l.href.startsWith('#')); ;

//       await Utils.scrollPageWithTimeAndReverse(page, Date.now() + 5000);

//       if (toClick) {
//         console.log('clicking', toClick.href);
//         await toClick.el.scrollIntoViewIfNeeded();
//         await toClick.el.click();
//       }

//       const endNavigation = Date.now();
//       if (endNavigation < maxEndTime) {
//         await Utils.scrollPageWithTimeAndReverse(page, maxEndTime - endNavigation);
//       }
//     } catch (error) {
//       console.log(error);
//     } finally {
//       try {
//         await browser.close();
//       } catch (error) {
//         console.log(error);
//       }
//     }
//   };

//   // const pages = Array.from({ length: 100 }, () => getPage(10000));

//   // const chunked = _.chunk(pages, 3);

//   const getPromises = (length: number) => Array.from({ length }, () => getPage((input?.waitSecondsInterval || 1) * 1000));

//   const visits = input?.visitsNumber || 10;
//   const chunkSize = input?.chunkSize || 1;

//   let count = 0;
//   for (let i = 0; i < visits / chunkSize; i += 1) {
//     const pagesNb = count + chunkSize > visits ? visits - count : chunkSize;
//     count += pagesNb;
//     await Promise.all(getPromises(pagesNb));
//   }
// } catch (error) {
//   console.log(error);
// }

// // Exit successfully
// await Actor.exit();


src/main copy.ts

import { Actor, ProxyConfiguration } from 'apify';
import { Crawler } from './crawler.js';
import { getLogger } from './logger.js';
import { Visitor } from './visitor.js';
import { UrlsManager } from './urls-manager.js';

const isLocal = process.env.ACTOR_ENV === 'local';

await Actor.init();

type Input = {
  startUrls: string[];
  chunkSize: number;
  visitsNumber: number;
  waitSecondsInterval: number;
  crawl: boolean;
  useResidentialProxies: boolean;
  proxiesCountry: string;
  useMobileDevices: boolean;
  useTabletDevices: boolean;
  useDesktopDevices: boolean;
};

const logger = getLogger('[MAIN_PROCESS]');
const {
  startUrls,
  chunkSize,
  visitsNumber: desiredVisits,
  waitSecondsInterval,
  useResidentialProxies,
  proxiesCountry,
  crawl,
  useMobileDevices,
  useTabletDevices,
  useDesktopDevices,
} = await Actor.getInput<Input>() as Input;

const getAllowedDevices = (): ('desktop' | 'mobile' | 'tablet')[] => {
  const allowedDevices: ('desktop' | 'mobile' | 'tablet')[] = [];
  if (useMobileDevices) allowedDevices.push('mobile');
  if (useTabletDevices) allowedDevices.push('tablet');
  if (useDesktopDevices) allowedDevices.push('desktop');
  return allowedDevices.length ? allowedDevices : ['desktop'];
};

try {
  // const input = await Actor.getInput<Input>();
  const urls = crawl ? await Crawler.new(startUrls, 1000).run() : UrlsManager.new(startUrls);
  const allowedDevices = getAllowedDevices();
  const proxyConfiguration = isLocal ? undefined : await Actor.createProxyConfiguration({
    groups: useResidentialProxies ? ['RESIDENTIAL'] : undefined,
    countryCode: (proxiesCountry && proxiesCountry !== 'all') ? proxiesCountry : undefined,
  }) as ProxyConfiguration;

  let count = 0;
  for (let i = 0; i < desiredVisits / chunkSize; i += 1) {
    const pagesNb = count + chunkSize > desiredVisits ? desiredVisits - count : chunkSize;
    const visitors = Visitor.getVisitChunks(
      urls.getNextUrls(pagesNb),
      waitSecondsInterval * 1000,
      allowedDevices,
      proxyConfiguration,
    );
    const visits = await Promise.all(visitors);
    count += visits.map((v) => (v.success ? 1 : 0)).reduce((a: number, b: number) => a + b, 0);
    await Actor.pushData(visits);
  }
} catch (error) {
  logger.fatal((error as Error).message);
}

await Actor.exit();


src/main.ts

// import { Dataset, createPlaywrightRouter } from 'crawlee';

// export const router = createPlaywrightRouter();

// router.addDefaultHandler(async ({ enqueueLinks, log }) => {
//     log.info(`enqueueing new URLs`);
//     await enqueueLinks({
//         globs: ['https://apify.com/*'],
//         label: 'detail',
//     });
// });

// router.addHandler('detail', async ({ request, page, log }) => {
//     const title = await page.title();
//     log.info(`${title}`, { url: request.loadedUrl });

//     await Dataset.pushData({
//         url: request.loadedUrl,
//         title,
//     });
// });


src/routes.ts

export class UrlsManager {
  static index = 0;

  constructor(
    public urls: string[],
  ) {}

  public get length() {
    return this.urls.length;
  }

  static new(urls: string[]) {
    return new UrlsManager(urls);
  }

  getNextUrl() {
    const url = this.urls[UrlsManager.index];
    UrlsManager.index = (UrlsManager.index + 1) % this.urls.length;
    return url;
  }

  getNextUrls(number: number) {
    return Array.from({ length: number }).map(() => this.getNextUrl());
  }

  pushUrl(url?: string) {
    if (!url) return;
    this.urls = Array.from(new Set([...this.urls, url]));
  }

  show() {
    this.urls.forEach((element) => console.log(element));
  }
}


src/urls-manager.ts

import { Page, devices } from 'playwright';

export class Utils {
  static scrollPageWithTimeAndReverse = async (page: Page, endTime: number) => {
    await page.evaluate(async (_endTime) => {
      await new Promise((resolve) => {
        let totalHeight = 0;
        const distance = 600;
        const timer = setInterval(() => {
          const { scrollHeight } = document.body;
          // eslint-disable-next-line no-nested-ternary
          const direction = totalHeight >= scrollHeight
            ? -1 : totalHeight <= 10 ? 1 : 1;
          window.scrollBy(0, distance * direction);
          totalHeight += distance * direction;
          if (Date.now() >= _endTime) {
            clearInterval(timer);
            resolve(null);
          }
        }, 1000);
      });
    }, endTime);
  };

  static getRandomTabletDevice = () => Object.values(devices)
    .filter((e) => e.hasTouch && !e.isMobile)[Math.floor(Math.random() * Object.values(devices).length)];

  static getRandomDesktopDevice = () => Object.values(devices)
    .filter((e) => !e.hasTouch && !e.isMobile)[Math.floor(Math.random() * Object.values(devices).length)];

  static getRandomMobileDevice = () => Object.values(devices)
    .filter((e) => e.isMobile)[Math.floor(Math.random() * Object.values(devices).length)];

  static getRandomDevice = (
    types: ('tablet' | 'mobile' | 'desktop')[] = ['desktop', 'mobile', 'tablet'],
    repeatMax = 3,
  ): typeof devices[''] => {
    const rep = {
      tablet: Utils.getRandomTabletDevice,
      mobile: Utils.getRandomMobileDevice,
      desktop: Utils.getRandomDesktopDevice,
    };
    const tmp = rep[types[Math.floor(Math.random() * types.length)]]();
    if (!tmp && repeatMax > 0) {
      return Utils.getRandomDevice(types, repeatMax - 1);
    }
    return tmp ?? devices['Galaxy Tab S4'];
  };

  static getDomainFromURL(url: string): string {
    const parsedURL = new URL(url);
    return parsedURL.hostname;
  }
}


src/utils.ts

/* eslint-disable indent */
import { ProxyConfiguration } from 'apify';
import { launchPlaywright } from 'crawlee';
import { Browser, ElementHandle, Page, devices } from 'playwright';
import { createId } from '@paralleldrive/cuid2';
import { Utils } from './utils.js';
import { getLogger, LoggerType } from './logger.js';

export type PageLink = {
  el: ElementHandle<SVGElement | HTMLElement>;
  href: string | null;
};

export type Visit = {
  url: string;
  duration: number;
  startTime: number;
  endTime: number;
  success: boolean;
  device?: typeof devices[''];
  proxyUrl?: string;
}

export class Visitor {
  protected _browser?: Browser;

  protected _page?: Page;

  protected _startVisitTime?: number;

  protected _proxyUrl?: string;

  protected _device?: typeof devices[''];

  public id = createId();

  log: LoggerType;

  public get browser() {
    if (!this._browser) {
      throw new Error('Browser not initialized');
    }
    return this._browser;
  }

  public set browser(browser: Browser) {
    this._browser = browser;
  }

  public get page() {
    if (!this._page) {
      throw new Error('Page not initialized');
    }
    return this._page;
  }

  public set page(page: Page) {
    this._page = page;
  }

  constructor(
    protected readonly allowedDevices: ('desktop' | 'mobile' | 'tablet')[],
    protected readonly proxies?: ProxyConfiguration,
  ) {
    this.log = getLogger(`[VISITOR_${this.id}]`);
  }

  public async init() {
    this.log.info('initializing visitor');
    this._proxyUrl = this.proxies ? await this.proxies.newUrl() : undefined;
    this._device = Utils.getRandomDevice(this.allowedDevices);
    this.browser = await launchPlaywright({
      launchOptions: { headless: true },
      userAgent: this._device.userAgent,
      proxyUrl: this._proxyUrl,
    });
    this.page = await this.browser.newPage();
    return this;
  }

  getEndVisitTime = (duration: number) => (this._startVisitTime || Date.now()) + duration;

  public async goto(url: string) {
    await this.page.goto(url);
    await this.page.waitForLoadState('domcontentloaded');
    this._startVisitTime = Date.now();
  }

  getPageClickableElements = async (): Promise<PageLink[]> => {
    const clickableElements = await this.page.$$(
      'a, button, input[type="button"], input[type="submit"], [role="button"]',
    );
    const links = await Promise.all(clickableElements.map(async (el) => ({ el, href: await el.evaluate((node) => node.getAttribute('href')) })));
    return links;
  };

  tryToClick = async (links: PageLink): Promise<boolean> => {
    try {
      this.log.info(`Trying to click on ${links.href}`);
      await links.el.scrollIntoViewIfNeeded();
      await links.el.click();
      this.log.info(`Clicked on ${links.href}`);
      return true;
    } catch (e) {
      return false;
    }
  };

  orderLinksByClickability = (links: PageLink[]): PageLink[] => {
    const baseUrl = this.page.url();
    const baseDomain = Utils.getDomainFromURL(baseUrl);
    const linksWithSameDomain = links.filter((l) => l.href && l.href.indexOf(baseDomain) !== -1);
    const linksWithSameBaseUrl = linksWithSameDomain.filter((l) => l.href && l.href.indexOf(baseUrl) !== -1);
    const linksStartingWithHash = links.filter((l) => l.href && l.href.startsWith('#'));
    const linksWithDifferentBaseUrl = linksWithSameDomain.filter((l) => l.href && l.href.indexOf(baseUrl) === -1);
    const linksWithoutHref = links.filter((l) => !l.href);

    return Array.from(new Set([
      ...linksWithSameDomain,
      ...linksWithDifferentBaseUrl,
      ...linksStartingWithHash,
      ...linksWithSameBaseUrl,
      ...linksWithoutHref,
    ]));
  };

  clickOnLinks = async (input: PageLink[]): Promise<void> => {
    const links = this.orderLinksByClickability(input);
    for (let i = 0; i < links.length; i += 1) {
      const clicked = await this.tryToClick(links[i]);
      if (clicked) {
        break;
      }
    }
  };

  visit = async (url: string, duration: number): Promise<Visit> => {
    this.log.info(`Visiting ${url}`);
    let success = false;
    await this.init();
    try {
      await this.goto(url);
      const links = await this.getPageClickableElements();
      await this.clickOnLinks(links);
      if (this.getEndVisitTime(duration) > Date.now()) {
        await Utils.scrollPageWithTimeAndReverse(this.page, this.getEndVisitTime(duration) - Date.now());
      }
      success = true;
    } catch (e) {
      this.log.error(e);
    }
    await this.browser.close();
    return {
      url,
      duration,
      startTime: this._startVisitTime || Date.now(),
      endTime: Date.now(),
      success,
      device: this._device,
      proxyUrl: this._proxyUrl,
    };
  };

  static getVisitChunks = (
    urls: string[],
    duration: number,
    allowedDevices: ('desktop' | 'mobile' | 'tablet')[],
    proxies?: ProxyConfiguration,
    ) => {
    return urls.map((url) => new Visitor(allowedDevices, proxies).visit(url, duration));
  };
}


src/visitor.ts

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

.eslintrc

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage

# Added by Apify CLI
.venv


.gitignore

test-web-traffic

package.json

pnpm-lock.yaml

tsconfig.json

🚀💥 Introducing WebRocket! 💥 Supercharge your website 📈, deep crawling 🕸️, and robust error handling 🤖. Blast off with start URLs 🚀, choose simultaneous visitors 🧑🏻‍🤝‍🧑🏻, and set visit numbers #️⃣. Customize the stay duration ⌛, pick device types 📱🖥️📟, and use residential proxies 🌍🏠

🔥 Web Traffic Generator | 🚀 WebRocket 🚀

🔥 Web Traffic Generator | 🚀 WebRocket 🚀 · Apify

Input · 🔥 Web Traffic Generator | 🚀 WebRocket 🚀 · Apify

API · 🔥 Web Traffic Generator | 🚀 WebRocket 🚀 · Apify

Related Actors · 🔥 Web Traffic Generator | 🚀 WebRocket 🚀 · Apify

🚀💥 Introducing WebRocket! 💥 Supercharge your website 📈, deep crawling 🕸️, and robust error handling 🤖. Blast off with start URLs 🚀, choose simultaneous visitors 🧑🏻‍🤝‍🧑🏻, and set visit n...

Where next?