Start with Python

1"""
2This module defines the `main()` coroutine for the Apify Actor, executed from the `__main__.py` file.
3
4Feel free to modify this file to suit your specific needs.
5
6To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:
7https://docs.apify.com/sdk/python
8"""
9
10# Beautiful Soup - library for pulling data out of HTML and XML files, read more at
11# https://www.crummy.com/software/BeautifulSoup/bs4/doc
12from bs4 import BeautifulSoup
13
14# HTTPX - library for making asynchronous HTTP requests in Python, read more at https://www.python-httpx.org/
15from httpx import AsyncClient
16
17# Apify SDK - toolkit for building Apify Actors, read more at https://docs.apify.com/sdk/python
18from apify import Actor
19
20
21async def main() -> None:
22    """
23    The main coroutine is being executed using `asyncio.run()`, so do not attempt to make a normal function
24    out of it, it will not work. Asynchronous execution is required for communication with Apify platform,
25    and it also enhances performance in the field of web scraping significantly.
26    """
27    async with Actor:
28        # Structure of input is defined in input_schema.json
29        actor_input = await Actor.get_input() or {}
30        url = actor_input.get('url')
31
32        # Create an asynchronous HTTPX client
33        async with AsyncClient() as client:
34            # Fetch the HTML content of the page.
35            response = await client.get(url, follow_redirects=True)
36
37        # Parse the HTML content using Beautiful Soup
38        soup = BeautifulSoup(response.content, 'html.parser')
39
40        # Extract all headings from the page (tag name and text)
41        headings = []
42        for heading in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
43            heading_object = {'level': heading.name, 'text': heading.text}
44            Actor.log.info(f'Extracted heading: {heading_object}')
45            headings.append(heading_object)
46
47        # Save headings to Dataset - a table-like storage
48        await Actor.push_data(headings)