Crawlee + Playwright + Camoufox

1from apify import Actor
2from camoufox import AsyncNewBrowser
3from crawlee.browsers import BrowserPool, PlaywrightBrowserController, PlaywrightBrowserPlugin
4from crawlee.crawlers import PlaywrightCrawler
5from typing_extensions import override
6
7from .routes import router
8
9
10class CamoufoxPlugin(PlaywrightBrowserPlugin):
11    """Browser plugin that uses Camoufox Browser, but otherwise keeps the functionality of PlaywrightBrowserPlugin."""
12
13    @override
14    async def new_browser(self) -> PlaywrightBrowserController:
15        if not self._playwright:
16            raise RuntimeError('Playwright browser plugin is not initialized.')
17
18        return PlaywrightBrowserController(
19            browser=await AsyncNewBrowser(self._playwright, headless=True),
20            max_open_pages_per_browser=1,  #  Increase, if camoufox can handle it in your use case.
21            header_generator=None,  #  This turns off the crawlee header_generation. Camoufox has its own.
22        )
23
24
25async def main() -> None:
26    """Define a main entry point for the Apify Actor.
27
28    This coroutine is executed using `asyncio.run()`, so it must remain an asynchronous function for proper execution.
29    Asynchronous execution is required for communication with Apify platform, and it also enhances performance in
30    the field of web scraping significantly.
31    """
32    # Enter the context of the Actor.
33    async with Actor:
34        # Retrieve the Actor input, and use default values if not provided.
35        actor_input = await Actor.get_input() or {}
36        start_urls = [
37            url.get('url')
38            for url in actor_input.get(
39                'start_urls',
40                [{'url': 'https://apify.com'}],
41            )
42        ]
43
44        # Exit if no start URLs are provided.
45        if not start_urls:
46            Actor.log.info('No start URLs specified in Actor input, exiting...')
47            await Actor.exit()
48
49        # Create a crawler.
50        crawler = PlaywrightCrawler(
51            # Limit the crawl to max requests. Remove or increase it for crawling all links.
52            max_requests_per_crawl=10,
53            browser_pool=BrowserPool(plugins=[CamoufoxPlugin()]),
54            # Set the request handler to the request router defined in routes.py.
55            request_handler=router,
56        )
57
58        # Run the crawler with the starting requests.
59        await crawler.run(start_urls)