.actor

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python:3.11

# Second, copy just requirements.txt into the actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Specify how to launch the source code of your actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]


.actor/Dockerfile

Scrapes titles of websites using BeautifulSoup.

getting-started-python-beautifulsoup

URLs and their titles

Getting started with Python and BeautifulSoup

.actor/actor.json

Maximum depth

Start URLs

Redfin agent reviews scraper

.actor/input_schema.json

src/__init__.py

import asyncio
import logging

from apify.log import ActorLogFormatter

from .main import main

handler = logging.StreamHandler()
handler.setFormatter(ActorLogFormatter())

apify_client_logger = logging.getLogger('apify_client')
apify_client_logger.setLevel(logging.INFO)
apify_client_logger.addHandler(handler)

apify_logger = logging.getLogger('apify')
apify_logger.setLevel(logging.DEBUG)
apify_logger.addHandler(handler)

asyncio.run(main())


src/__main__.py

from urllib.parse import urljoin

import requests,time
from apify import Actor
from bs4 import BeautifulSoup

async def main():
    async with Actor:
        # Read the Actor input
        actor_input = await Actor.get_input() or {}
        start_urls = actor_input.get('start_urls', [{ 'url': 'https://www.redfin.com/real-estate-agents/brentjon/reviews' }])
        print(start_urls)
        print("try: ",start_urls[0]["url"])
        max_depth = actor_input.get('max_depth', 1)
        
        new_url = start_urls[0]["url"]
        print(new_url)
        if not start_urls:
            Actor.log.info('No querry specified in actor input, exiting...')
            await Actor.exit()
        print("starting...")

        # Enqueue the starting URLs in the default request queue
        headers = {
            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36"
        }

        for urlx in start_urls:
            response = requests.get(urlx["url"], headers=headers)
            time.sleep(2)
            soup = BeautifulSoup(response.text, "html.parser")  
            all_reviews = soup.find_all("div", class_="agent-review-column big-column")
    
            for rev in all_reviews:
                
                try:
                    StarRating = int(float(rev.find("div", class_="stars")["aria-label"].split("Rating ")[-1].split()[0]))
                except:
                    StarRating = ""
                try:
                    comment = rev.find("div", class_="review").text
                except:
                    comment= ""
                    
                try:
                    clienttype = rev.find("span", class_="client-type").text
                except:
                    clienttype=""
                
                try:
                    closedate = rev.find("span", class_="close-date").text.replace("Closed  ","").replace("'","20").strip()
                except:
                    closedate=""
                    
                    
                try:
                    Details = rev.find("span", class_="prop-info").find("p").text
                except:
                    Details = ""
                    
                    
                try:
                    ListingURL= "https://www.redfin.com" + rev.find("span", class_="prop-info").find("a")["href"]
                except:
                    ListingURL=""
                    
                print(closedate)
                await Actor.push_data({
                    "Source":"Redfin",
                    "Star Rating":StarRating,
                    "Comment":comment,
                    "Client Type":clienttype,
                    "Close Date":closedate,
                    "Details":Details,
                    "Listing URL":ListingURL
                })
                #liste.append(item)

src/main.py

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
.venv

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

# This file tells Git which files shouldn't be added to source control

.idea
.DS_Store

apify_storage
storage

.venv/
.env/
__pypackages__
dist/
build/
*.egg-info/
*.egg

__pycache__

.mypy_cache
.dmypy.json
dmypy.json
.pytest_cache

.scrapy
*.log


.gitignore

# Add your dependencies here.
# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
# for how to format them
apify ~= 1.0.0
beautifulsoup4 ~= 4.9.0
requests ~= 2.28.2


Where next?