Example Selenium

1const Apify = require('apify');
2const { Capabilities, Builder, logging } = require('selenium-webdriver');
3const chrome = require('selenium-webdriver/chrome');
4const proxy = require('selenium-webdriver/proxy');
5const { anonymizeProxy } = require('proxy-chain');
6
7const launchChromeWebdriver = async (options) => {
8    let anonymizedProxyUrl = null;
9
10    // logging.installConsoleHandler();
11    // logging.getLogger('webdriver.http').setLevel(logging.Level.ALL);
12
13    // See https://github.com/SeleniumHQ/selenium/wiki/DesiredCapabilities for reference.
14    const capabilities = new Capabilities();
15    capabilities.set('browserName', 'chrome');
16
17    // Chrome-specific options
18    // By default, Selenium already defines a long list of command-line options
19    // to enable browser automation, here we add a few other ones
20    // (inspired by Lighthouse, see lighthouse/lighthouse-cli/chrome-launcher)
21    const chromeOptions = new chrome.Options();
22    chromeOptions.addArguments('--disable-translate');
23    chromeOptions.addArguments('--safebrowsing-disable-auto-update');
24
25    if (options.headless) {
26        chromeOptions.addArguments('--headless', '--no-sandbox');
27    }
28
29    if (options.userAgent) {
30        chromeOptions.addArguments(`--user-agent=${options.userAgent}`);
31    } 
32
33    if (options.extraChromeArguments) {
34        chromeOptions.addArguments(options.extraChromeArguments);
35    }
36
37    const builder = new Builder();
38
39    // For proxy servers with authentication, this class starts a local proxy server
40    // NOTE: to view effective proxy settings in Chrome, open chrome://net-internals/#proxy
41    if (options.proxyUrl) {
42        const anonymizedProxyUrl = await anonymizeProxy(options.proxyUrl)
43        chromeOptions.addArguments(`--proxy-server=${anonymizedProxyUrl}`);
44    }
45
46    const webDriver = builder
47        .setChromeOptions(chromeOptions)
48        .withCapabilities(capabilities)
49        .build();
50
51    return webDriver;
52};
53
54
55Apify.main(async () => {
56    const input = await Apify.getInput();
57    console.log('Input:');
58    console.dir(input);
59
60    // Prepare proxy URL
61    let proxyUrl = '';
62    if (input.proxy) {
63        const { useApifyProxy, apifyProxyGroups, proxyUrls } = input.proxy;
64        if (useApifyProxy) {
65            proxyUrl = Apify.getApifyProxyUrl({ groups: apifyProxyGroups });
66        } else if ((proxyUrls || []).length) {
67            proxyUrl = proxyUrls[Math.floor(Math.random * proxyUrls.length)];
68        }
69    }
70
71    // Prepare user agent
72    let userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36';
73    if (input.userAgent) userAgent = input.userAgent;
74
75    const options = {
76        proxyUrl,
77        userAgent,
78        headless: true,
79    }
80
81    console.log('Launching Selenium WebDriver...');
82    const webDriver = await launchChromeWebdriver(options);
83
84    console.log(`Opening ${input.url}...`);
85    await webDriver.get(input.url);
86
87    const currentUrl = await webDriver.getCurrentUrl();
88    console.log(`Current url ${currentUrl}`);
89                   
90    console.log('Getting the page title...');
91    const pageTitle = await webDriver.executeScript(() => {
92        return document.title;
93    });
94    console.log(`Page title: ${pageTitle}`);
95                   
96    console.log('Taking screenshot...');
97    const screenshot = await webDriver.takeScreenshot();
98    await Apify.setValue('screenshot.png', Buffer.from(screenshot, 'base64'), { contentType: 'image/png' });
99
100    console.log('Getting the page html code...');
101    const source = await webDriver.getPageSource();
102    await Apify.setValue('source.html', source, { contentType: 'text/html'});
103    
104    console.log('Saving output...');
105    await Apify.setValue('OUTPUT', {
106        url: input.url,
107        pageTitle, 
108    });
109    
110    console.log('Done.')
111});