Parsel crawler

1"""Module defines the main entry point for the Apify Actor.
2
3Feel free to modify this file to suit your specific needs.
4
5To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:
6https://docs.apify.com/sdk/python
7"""
8
9from __future__ import annotations
10
11from apify import Actor
12from crawlee.crawlers import ParselCrawler
13
14from .routes import router
15
16
17async def main() -> None:
18    """Define a main entry point for the Apify Actor.
19
20    This coroutine is executed using `asyncio.run()`, so it must remain an asynchronous function for proper execution.
21    Asynchronous execution is required for communication with Apify platform, and it also enhances performance in
22    the field of web scraping significantly.
23    """
24    # Enter the context of the Actor.
25    async with Actor:
26        # Retrieve the Actor input, and use default values if not provided.
27        actor_input = await Actor.get_input() or {}
28        start_urls = [
29            url.get('url')
30            for url in actor_input.get(
31                'start_urls',
32                [{'url': 'https://apify.com'}],
33            )
34        ]
35
36        # Exit if no start URLs are provided.
37        if not start_urls:
38            Actor.log.info('No start URLs specified in Actor input, exiting...')
39            await Actor.exit()
40
41        # Create a crawler.
42        crawler = ParselCrawler(
43            # Limit the crawl to max requests. Remove or increase it for crawling all links.
44            max_requests_per_crawl=10,
45            # Set the request handler to the request router defined in routes.py.
46            request_handler=router,
47        )
48
49        # Run the crawler with the starting requests.
50        await crawler.run(start_urls)