Selenium with wait

1"""
2This module serves as the entry point for executing the Apify Actor. It handles the configuration of logging
3settings. The `main()` coroutine is then executed using `asyncio.run()`.
4
5Feel free to modify this file to suit your specific needs.
6"""
7
8import asyncio
9import logging
10
11from apify.log import ActorLogFormatter
12
13from .main import main
14
15# Configure loggers
16handler = logging.StreamHandler()
17handler.setFormatter(ActorLogFormatter())
18
19apify_client_logger = logging.getLogger('apify_client')
20apify_client_logger.setLevel(logging.INFO)
21apify_client_logger.addHandler(handler)
22
23apify_logger = logging.getLogger('apify')
24apify_logger.setLevel(logging.DEBUG)
25apify_logger.addHandler(handler)
26
27# Execute the Actor main coroutine
28asyncio.run(main())

1"""
2This module defines the `main()` coroutine for the Apify Actor, executed from the `__main__.py` file.
3
4Feel free to modify this file to suit your specific needs.
5
6To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:
7https://docs.apify.com/sdk/python
8"""
9
10from urllib.parse import urljoin
11from time import sleep
12
13from selenium import webdriver
14from selenium.webdriver.chrome.options import Options as ChromeOptions
15from selenium.webdriver.common.by import By
16
17from apify import Actor
18
19# To run this Actor locally, you need to have the Selenium Chromedriver installed.
20# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/
21# When running on the Apify platform, it is already included in the Actor's Docker image.
22
23
24async def main() -> None:
25    """
26    The main coroutine is being executed using `asyncio.run()`, so do not attempt to make a normal function
27    out of it, it will not work. Asynchronous execution is required for communication with Apify platform,
28    and it also enhances performance in the field of web scraping significantly.
29    """
30    
31    async with Actor:
32        # Read the Actor input
33        actor_input = await Actor.get_input() or {}
34        start_urls = actor_input.get('start_urls', [
35            {'url': 'https://knowledge.alteryx.com/index/s/article/How-to-restart-Alteryx-Service-remotely-when-RDP-is-not-available'}
36            ])
37        max_depth = actor_input.get('max_depth', 1)
38        wait_time = actor_input.get('wait_time_in_seconds', 5)
39        if not isinstance(wait_time, int):
40            Actor.log.info('Sleep time is not a number. Setting default...')
41            wait_time = 5
42
43        if not start_urls:
44            Actor.log.info('No start URLs specified in actor input, exiting...')
45            await Actor.exit()
46
47        # Enqueue the starting URLs in the default request queue
48        default_queue = await Actor.open_request_queue()
49        for start_url in start_urls:
50            url = start_url.get('url')
51            Actor.log.info(f'Enqueuing {url} ...')
52            await default_queue.add_request({'url': url, 'userData': {'depth': 0}})
53
54        # Launch a new Selenium Chrome WebDriver
55        Actor.log.info('Launching Chrome WebDriver...')
56        chrome_options = ChromeOptions()
57        if Actor.config.headless:
58            chrome_options.add_argument('--headless')
59        chrome_options.add_argument('--no-sandbox')
60        chrome_options.add_argument('--disable-dev-shm-usage')
61        driver = webdriver.Chrome(options=chrome_options)
62
63        # TODO: Maybe i can comment this ones?
64        driver.get('http://www.example.com')
65        assert driver.title == 'Example Domain'
66
67        # Process the requests in the queue one by one
68        while request := await default_queue.fetch_next_request():
69            url = request['url']
70            depth = request['userData']['depth']
71            Actor.log.info(f'Scraping {url} ...')
72
73            try:
74                # Open the URL in the Selenium WebDriver
75                driver.get(url)
76                Actor.log.info(f'Sleeping for this much time: {wait_time} seconds ...')
77                sleep(wait_time)
78
79                # Push the title of the page into the default dataset
80                title = driver.title
81                html_content = driver.page_source
82                Actor.log.info("Got title and html content")
83                await Actor.push_data({'url': url, 'title': title, 'text': html_content})
84            except Exception:
85                Actor.log.exception(f'Cannot extract data from {url}.')
86            finally:
87                await default_queue.mark_request_as_handled(request)
88
89        driver.quit()

1# Feel free to add your Python dependencies below. For formatting guidelines, see:
2# https://pip.pypa.io/en/latest/reference/requirements-file-format/
3
4apify ~= 1.7.0
5selenium ~= 4.14.0