# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
.venv

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

# This file tells Git which files shouldn't be added to source control

.idea
.DS_Store

apify_storage
storage

.venv/
.env/
__pypackages__
dist/
build/
*.egg-info/
*.egg

__pycache__

.mypy_cache
.dmypy.json
dmypy.json
.pytest_cache

.scrapy
*.log

# Added by Apify CLI
node_modules
.venv


.gitignore

# Add your dependencies here.
# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
# for how to format them
apify ~= 1.1.1
nest-asyncio ~= 1.5.6
scrapy ~= 2.9.0


requirements.txt

[settings]
default = src.settings

[deploy]
project = src


scrapy.cfg

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python:3.11

# Second, copy just requirements.txt into the actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Specify how to launch the source code of your actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]


.actor/Dockerfile

amh-crawler

URLs and their titles

Getting started with Python and Scrapy

.actor/actor.json

Maximum number of pages to scrape concurrently

Maximum concurrency

Maximum depth

Maximum pages

Start URLs

Name of the state to store the request queue in

State name

Amh Scraper

.actor/input_schema.json

src/__init__.py

# This file is executed only when the project is run as an Apify Actor,
# not when you run it locally using `scrapy crawl`.

# We need to configure the logging first before we import anything else,
# so that nothing else imports `scrapy.utils.log` before we patch it.
import logging
from apify.log import ActorLogFormatter
import scrapy.utils.log

handler = logging.StreamHandler()
handler.setFormatter(ActorLogFormatter())

apify_logger = logging.getLogger('apify')
apify_logger.setLevel(logging.DEBUG)
apify_logger.addHandler(handler)

apify_client_logger = logging.getLogger('apify_client')
apify_client_logger.setLevel(logging.DEBUG)
apify_client_logger.addHandler(handler)


# We can't attach our log handler to the loggers normally,
# because Scrapy would remove them in the `configure_logging` call here:
# https://github.com/scrapy/scrapy/blob/a5c1ef82762c6c0910abea00c0a6249c40005e44/scrapy/utils/log.py#L95
# (even though `disable_existing_loggers` is set to False :facepalm:).
# We need to monkeypatch Scrapy's `configure_logging` method like this,
# so that our handler is attached right after Scrapy calls the `configure_logging` method,
# because otherwise we would lose some log messages.
old_configure_logging = scrapy.utils.log.configure_logging

def new_configure_logging(*args, **kwargs):
    old_configure_logging(*args, **kwargs)

    # Scrapy uses these four main loggers: https://github.com/scrapy/scrapy/blob/a5c1ef82762c6c0910abea00c0a6249c40005e44/scrapy/utils/log.py#L44-L61
    logging.getLogger('scrapy').addHandler(handler)
    logging.getLogger('twisted').addHandler(handler)
    logging.getLogger('filelock').addHandler(handler)
    logging.getLogger('hpack').addHandler(handler)

scrapy.utils.log.configure_logging = new_configure_logging


# Now we can do the rest of the setup
import asyncio
import os

import nest_asyncio

from scrapy.utils.reactor import install_reactor
from .main import main

# This is necessary so that twisted and asyncio work well together
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
nest_asyncio.apply()

os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings'

asyncio.run(main())


src/__main__.py

# Define the models for your scraped items here
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TitlebotItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


src/items.py

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from apify import Actor

from .pipelines import ActorDatasetPushPipeline
from .spiders.amh_spider import AmhSpider


async def main():
    async with Actor:
        actor_input = await Actor.get_input() or {}
        state_name = actor_input.get('state_name', 'Tennessee')

        settings = get_project_settings()
        settings['ITEM_PIPELINES'] = { ActorDatasetPushPipeline: 1 }

        process = CrawlerProcess(settings, install_root_handler=False)

        # If you want to run multiple spiders, call `process.crawl` for each of them here
        process.crawl(AmhSpider, state_name=state_name)

        process.start()


src/main.py

# Define your item pipelines here
#
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

from apify import Actor
from itemadapter import ItemAdapter


# Used to output the items into the actor's default dataset
# Enabled only when the project is run as an actor
class ActorDatasetPushPipeline:
    async def process_item(self, item, spider):
        item_dict = ItemAdapter(item).asdict()
        await Actor.push_data(item_dict)
        return item


src/pipelines.py

# Scrapy settings for this project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html


BOT_NAME = 'titlebot'

SPIDER_MODULES = ['src.spiders']
NEWSPIDER_MODULE = 'src.spiders'

REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False


src/settings.py

# This package will contain the spiders of your Scrapy project
#
# Please refer to the Scrapy documentation
# for information on how to create and manage your spiders.


src/spiders/__init__.py

import scrapy
from urllib.parse import quote
import re

class AmhSpider(scrapy.Spider):
    name = 'amh'
    custom_settings = {
        "FEEDS": {
            "Tennessee_states.csv": {"format": "csv", "encoding": "utf8", "overwrite": True},
        },


    }

    def __init__(self, state_name) -> None:
        self.next_id = 'rzjXXqOBD9uOUh22abk5J'
        self.base_url = f'https://www.amh.com/_next/data/{self.next_id}/query.json'
        self.state_name = state_name
        super().__init__()

        
        
    # parse "buildId":"NRHrR6nsUb8rTYxcR_eEZ"
    def start_requests(self):


        url = f"https://www.amh.com/query?criteria={self.state_name}"
        yield scrapy.Request(url=url, callback=self.get_next_id)

    def get_next_id(self, response):
        self.next_id = re.findall(r'buildId":"(.*?)"', response.text)[0]
        self.base_url = f'https://www.amh.com/_next/data/{self.next_id}/query.json'
        url = f"{self.base_url}?criteria={quote(self.state_name)}&page=1&pageSize=100"  
        yield scrapy.Request(url=url, callback=self.parse)

    def parse(self, response):

        json_file = response.json()
        results = json_file.get('pageProps', {}).get('results', [])
        for result in results:
            # result.pop('id').pop('accessCodeAvailableTo').pop('accessCodeAvailableFrom').pop('canGetAccessCode').pop('bannerFeature')
            # result.pop('id')
            result.pop('accessCodeAvailableTo')
            result.pop('accessCodeAvailableFrom')
            result.pop('canGetAccessCode')
            # result.pop('bannerFeatureName')
            images = result.get('images', [])
            good_images = [x.get('url_600w') for x in images]
            result['images'] = good_images
            yield result

        # downloading images in scrapy
            has_result = json_file.get(
                'pageProps', {}).get('hasResults', False)
            if has_result:
                page_number = json_file.get('pageProps', {}).get(
                    'query', {}).get('page', 1)
                state_name = json_file.get('pageProps', {}).get(
                    'query', {}).get('criteria', '')
                url = f'{self.base_url}?criteria={quote(state_name)}&page={int(page_number)+1}'
                self.logger.info(f'Next page: {url}')
                yield scrapy.Request(url=url)


Amh Crawler

Where next?