.actor

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python:3.11

# Second, copy just requirements.txt into the actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Specify how to launch the source code of your actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]


.actor/Dockerfile

{
    "actorSpecification": 1,
    "name": "getting-started-python-beautifulsoup",
    "title": "Getting started with Python and BeautifulSoup",
    "description": "Scrapes titles of websites using BeautifulSoup.",
    "version": "0.0",
    "meta": {
         "templateId": "python-beautifulsoup"
    },
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile",
    "storages": {
        "dataset": {
            "actorSpecification": 1,
            "title": "URLs and their titles",
            "views": {
                "titles": {
                    "title": "URLs and their titles",
                    "transformation": {
                        "fields": [
                            "full_name",
                            "agent_rating",
                            "description",
                            "phones",
                        
                        ]
                    },
                    "display": {
                        "component": "table",
                        "properties": {
                            "Agent full_name": {
                                "label": "Agent full_name",
                                "format": "text"
                            },
                            "agent_rating": {
                                "label": "agent_rating",
                                "format": "text"
                            },
                            "license_number": {
                                "label": "license_number",
                                "format": "text"
                            },
                            "description": {
                                "label": "description",
                                "format": "text"
                            },
                            "agent-group": {
                                "label": "agent-group",
                                "format": "text"
                            }
                        }
                    }
                }
            }
        }
    }
}


.actor/actor.json

Maximum depth

Start URLs

Python BeautifulSoup Scraper

.actor/input_schema.json

src/__init__.py

import asyncio
import logging

from apify.log import ActorLogFormatter

from .main import main

handler = logging.StreamHandler()
handler.setFormatter(ActorLogFormatter())

apify_client_logger = logging.getLogger('apify_client')
apify_client_logger.setLevel(logging.INFO)
apify_client_logger.addHandler(handler)

apify_logger = logging.getLogger('apify')
apify_logger.setLevel(logging.DEBUG)
apify_logger.addHandler(handler)

asyncio.run(main())


src/__main__.py

from urllib.parse import urljoin

import requests as r
from apify import Actor
from bs4 import BeautifulSoup
import json, time, random, math
import pandas as pd

async def main():
    async with Actor:
        # Read the Actor input
        actor_input = await Actor.get_input() or {}
        start_urls = actor_input.get('start_urls', [{ 'url': 'https://apify.com' }])
        max_depth = actor_input.get('max_depth', 1)
        print("------------------------------**************-------------------------------")

        new_url = start_urls[0]["url"].split("pg-")[0]
        print(new_url)

        if new_url.endswith("/"):
            pass
        else:
            new_url = new_url +"/"

        if not start_urls:
            Actor.log.info('No start URLs specified in actor input, exiting...')
            await Actor.exit()
        headers = {
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36"
        }
        
        # Enqueue the starting URLs in the default request queue
        for page in range(1,9999):
            url = new_url + "pg-"+ str(page)
            res = r.get(url,headers=headers)
            soup = BeautifulSoup(res.content, "html.parser")
            # Get JSON data
            js_d = soup.find("script", {"id":"__NEXT_DATA__","type":"application/json"})
            data = json.loads(js_d.contents[0])

            matching_rows = data["props"]["pageProps"]["pageData"]["matching_rows"]
            for d in data["props"]["pageProps"]["pageData"]["agents"]:
                
                await Actor.push_data(d)


            #stop when reached the end   
            if page == math.ceil(matching_rows / 20):
                print("done")
                break

        


src/main.py

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
.venv

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

# This file tells Git which files shouldn't be added to source control

.idea
.DS_Store

apify_storage
storage

.venv/
.env/
__pypackages__
dist/
build/
*.egg-info/
*.egg

__pycache__

.mypy_cache
.dmypy.json
dmypy.json
.pytest_cache

.scrapy
*.log


.gitignore

# Add your dependencies here.
# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
# for how to format them
apify ~= 1.0.0
beautifulsoup4 ~= 4.9.0
requests ~= 2.28.2
pandas

requirements.txt

Our Realtor scraper Pro tool helps overcome data retrieval challenges from Realtor, which lacks an API. Benefit from a range of features for your real estate business.
Agent details: You can easily scrape details on agent profiles, such as contact information, recent listings, and client reviews.

realtor-agents-scraper-pro

Realtor Agents Scraper Pro

Realtor Agents Scraper Pro · Apify

Input · Realtor Agents Scraper Pro · Apify

API · Realtor Agents Scraper Pro · Apify

Related Actors · Realtor Agents Scraper Pro · Apify

Our Realtor scraper Pro tool helps overcome data retrieval challenges from Realtor, which lacks an API. Benefit from a range of features for your real estate business.
Agent details: You can easily...

Where next?