# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python:3.11

# Second, copy just requirements.txt into the actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Specify how to launch the source code of your actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]


.actor/Dockerfile

Scrapes the California Court Interpreter Directory for personal details of interpreters

ca-court-interpreter

California Court Interpreter Personal Details

.actor/actor.json

Maximum depth

Start URLs

Apify Scraper

.actor/input_schema.json

# This package will contain the spiders of your Scrapy project
#
# Please refer to the Scrapy documentation
# for information on how to create and manage your spiders.


src/spiders/__init__.py

import scrapy


class CourtsCASpider(scrapy.Spider):
    name = 'courts_ca'
    start_urls = [
        'https://www.courts.ca.gov/cms/courtinterpreters/newctintdb.cfm']

    def parse(self, response):
        options_value = response.xpath(
            '//select[@name="searchLanguage"]/option/@value').extract()
        for code in options_value:
            if code == '0':
                continue
            formdata = {
                'searchLanguage': code,
                'searchCounty': '60',
                'submit': '+Search+'
            }
            yield scrapy.FormRequest.from_response(
                response,
                formdata=formdata,
                callback=self.parse_language,
            )

    def parse_language(self, response):

        for datas in response.xpath("//tr/td"):
            details = {}
            for data in datas.xpath('./strong'):
                details[data.xpath('./text()').extract_first()] = data.xpath(
                    './following-sibling::text()').extract_first()

            yield details


src/spiders/ca_court_spider.py

src/__init__.py

# This file is executed only when the project is run as an Apify Actor,
# not when you run it locally using `scrapy crawl`.

# We need to configure the logging first before we import anything else,
# so that nothing else imports `scrapy.utils.log` before we patch it.
import logging
from apify.log import ActorLogFormatter
import scrapy.utils.log

handler = logging.StreamHandler()
handler.setFormatter(ActorLogFormatter())

apify_logger = logging.getLogger('apify')
apify_logger.setLevel(logging.DEBUG)
apify_logger.addHandler(handler)

apify_client_logger = logging.getLogger('apify_client')
apify_client_logger.setLevel(logging.DEBUG)
apify_client_logger.addHandler(handler)


# We can't attach our log handler to the loggers normally,
# because Scrapy would remove them in the `configure_logging` call here:
# https://github.com/scrapy/scrapy/blob/a5c1ef82762c6c0910abea00c0a6249c40005e44/scrapy/utils/log.py#L95
# (even though `disable_existing_loggers` is set to False :facepalm:).
# We need to monkeypatch Scrapy's `configure_logging` method like this,
# so that our handler is attached right after Scrapy calls the `configure_logging` method,
# because otherwise we would lose some log messages.
old_configure_logging = scrapy.utils.log.configure_logging

def new_configure_logging(*args, **kwargs):
    old_configure_logging(*args, **kwargs)

    # Scrapy uses these four main loggers: https://github.com/scrapy/scrapy/blob/a5c1ef82762c6c0910abea00c0a6249c40005e44/scrapy/utils/log.py#L44-L61
    logging.getLogger('scrapy').addHandler(handler)
    logging.getLogger('twisted').addHandler(handler)
    logging.getLogger('filelock').addHandler(handler)
    logging.getLogger('hpack').addHandler(handler)

scrapy.utils.log.configure_logging = new_configure_logging


# Now we can do the rest of the setup
import asyncio
import os

import nest_asyncio

from scrapy.utils.reactor import install_reactor
from .main import main

# This is necessary so that twisted and asyncio work well together
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
nest_asyncio.apply()

os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings'

asyncio.run(main())


src/__main__.py

# Define the models for your scraped items here
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TitlebotItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


src/items.py

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from apify import Actor

from .pipelines import ActorDatasetPushPipeline
from .spiders.ca_court_spider import CourtsCASpider


async def main():
    async with Actor:
        actor_input = await Actor.get_input() or {}
        start_urls = [start_url.get('url') for start_url in actor_input.get('start_urls', [{ 'url': 'https://www.courts.ca.gov/cms/courtinterpreters/newctintdb.cfm' }])]

        settings = get_project_settings()
        settings['ITEM_PIPELINES'] = { ActorDatasetPushPipeline: 1 }

        process = CrawlerProcess(settings, install_root_handler=False)

        # If you want to run multiple spiders, call `process.crawl` for each of them here
        process.crawl(CourtsCASpider, start_urls=start_urls)

        process.start()


src/main.py

# Define your item pipelines here
#
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

from apify import Actor
from itemadapter import ItemAdapter


# Used to output the items into the actor's default dataset
# Enabled only when the project is run as an actor
class ActorDatasetPushPipeline:
    async def process_item(self, item, spider):
        item_dict = ItemAdapter(item).asdict()
        await Actor.push_data(item_dict)
        return item


src/pipelines.py

# Scrapy settings for this project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html


BOT_NAME = 'titlebot'

SPIDER_MODULES = ['src.spiders']
NEWSPIDER_MODULE = 'src.spiders'

REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False
CONCURRENT_REQUESTS = 32


src/settings.py

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
.venv

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

# This file tells Git which files shouldn't be added to source control

.idea
.DS_Store

apify_storage
storage

.venv/
.env/
__pypackages__
dist/
build/
*.egg-info/
*.egg

__pycache__

.mypy_cache
.dmypy.json
dmypy.json
.pytest_cache

.scrapy
*.log

# Added by Apify CLI
node_modules
.venv


.gitignore

# Add your dependencies here.
# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
# for how to format them
apify ~= 1.1.1
nest-asyncio ~= 1.5.6
scrapy ~= 2.9.0


requirements.txt

[settings]
default = src.settings

[deploy]
project = src


scrapy.cfg

It is a powerful web scraping tool designed to extract key information about court interpreters from the California Courts website. It collects details such as name, language proficiency, counties of operation, contact info, and employment status, making it valuable for legal and language research.

Ca Court Interpreter

Ca Court Interpreter · Apify

Input · Ca Court Interpreter · Apify

API · Ca Court Interpreter · Apify

Related Actors · Ca Court Interpreter · Apify

It is a powerful web scraping tool designed to extract key information about court interpreters from the California Courts website. It collects details such as name, language proficiency, counties ...

Where next?