.actor

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python:3.11

# Second, copy just requirements.txt into the actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Specify how to launch the source code of your actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]


.actor/Dockerfile

houzz

URLs and their titles

Extract Eventbrite online events

.actor/actor.json

Maximum depth

Start URLs

.actor/input_schema.json

src/__init__.py

import asyncio
import logging

from apify.log import ActorLogFormatter

from .main import main

handler = logging.StreamHandler()
handler.setFormatter(ActorLogFormatter())

apify_client_logger = logging.getLogger('apify_client')
apify_client_logger.setLevel(logging.INFO)
apify_client_logger.addHandler(handler)

apify_logger = logging.getLogger('apify')
apify_logger.setLevel(logging.DEBUG)
apify_logger.addHandler(handler)

asyncio.run(main())


src/__main__.py

from urllib.parse import urljoin
import math
import requests
from apify import Actor
from bs4 import BeautifulSoup
import json
import time, random

async def main():
    async with Actor:
        # Read the Actor input
        actor_input = await Actor.get_input() or {}
        start_urls = actor_input.get('start_urls', [{ 'url': 'https://www.houzz.com/professionals/home-remodeling/highlight-license-probr1-bo~t_34257~ps_4002' }])

        max_depth = actor_input.get('max_depth', 1)
        
        if "?fi=" not in  start_urls[0]["url"]:
            
            new_url = start_urls[0]["url"]+"?fi="
        else:
            new_url = start_urls[0]["url"]


        print(new_url)
        if not start_urls:
            Actor.log.info('No querry specified in actor input, exiting...')
            await Actor.exit()
        
        page_no = 0
        while True:
            url = new_url+ str(page_no)

            headers = {
                'authority': 'www.houzz.com',
                'accept': '*/*',
                'accept-language': 'en-US,en;q=0.9,tr;q=0.8,fr;q=0.7',
                'referer': f'{url}',
                'sec-ch-ua': '"Not.A/Brand";v="8", "Chromium";v="114", "Microsoft Edge";v="114"',
                'sec-ch-ua-mobile': '?0',
                'sec-ch-ua-platform': '"Windows"',
                'sec-fetch-dest': 'empty',
                'sec-fetch-mode': 'cors',
                'sec-fetch-site': 'same-origin',
                'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36 Edg/116.0.1938.54',
                'x-hz-request': 'true',
                'x-hz-spf-request': 'true',
                'x-requested-with': 'XMLHttpRequest',
                'x-spf-previous': f'{url}',
                'x-spf-referer': f'{url}',
            }
            
            try:
                response = requests.get(url, headers=headers)
                time.sleep(random.randint(1,3))
                soup = BeautifulSoup(response.text, "html.parser")
                print("*************************************************")
                script_content = str(soup.find("script", {"id":"hz-ctx", "type":"application/json"}))
                script_content = script_content.replace('<script id="hz-ctx" type="application/json">',"").replace("</script>","")
                print(script_content)
                print("**************script_content***********")
                json_data = json.loads(script_content)["data"]["stores"]["data"]["ProfessionalStore"]["data"]
                
                for x in json_data:
                    await Actor.push_data(json_data[x])


                print(page_no)
                if (len(json_data)) == 0 or (len(json_data))<1:
                    break

            except:
                print("except")
                page_no -=15
                time.sleep(20)
            page_no +=15



src/main.py

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
.venv

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

# This file tells Git which files shouldn't be added to source control

.idea
.DS_Store

apify_storage
storage

.venv/
.env/
__pypackages__
dist/
build/
*.egg-info/
*.egg

__pycache__

.mypy_cache
.dmypy.json
dmypy.json
.pytest_cache

.scrapy
*.log


.gitignore

# Add your dependencies here.
# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
# for how to format them
apify ~= 1.0.0
beautifulsoup4 ~= 4.9.0
requests ~= 2.28.2


requirements.txt

Houzz is a well-known platform for discovering and exploring home design and remodeling ideas but it lacks an API for directly accessing comprehensive project details. Thankfully,with the help of Apify script users can effortlessly extract the latest project information without any coding knowledge"

houzz · Apify

Input · houzz · Apify

API · houzz · Apify

Related Actors · houzz · Apify

Houzz is a well-known platform for discovering and exploring home design and remodeling ideas but it lacks an API for directly accessing comprehensive project details. Thankfully,with the help of A...

Where next?