FPAK API

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python:3.11

# Second, copy just requirements.txt into the Actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Use compileall to ensure the runnability of the Actor Python code.
RUN python3 -m compileall -q .

# Specify how to launch the source code of your Actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]

{
    "actorSpecification": 1,
    "name": "my-actor-3",
    "title": "Getting started with Python and Scrapy",
    "description": "Scrapes titles of websites using Scrapy.",
    "version": "0.0",
    "meta": {
        "templateId": "python-scrapy"
    },
    "input": "./input_schema.json",
    "dockerfile": "./Dockerfile"
}

{
    "title": "Python Scrapy Scraper",
    "type": "object",
    "schemaVersion": 1,
    "properties": {
        "startUrls": {
            "title": "Start URLs",
            "type": "array",
            "description": "URLs to start with",
            "prefill": [
                { "url": "https://apify.com" }
            ],
            "editor": "requestListSources"
        },
        "proxyConfiguration": {
            "sectionCaption": "Proxy and HTTP configuration",
            "title": "Proxy configuration",
            "type": "object",
            "description": "Specifies proxy servers that will be used by the scraper in order to hide its origin.",
            "editor": "proxy",
            "prefill": { "useApifyProxy": true },
            "default": { "useApifyProxy": true }
        }
    },
    "required": ["startUrls"]
}

# Git folder
.git

# IDE
.idea/
.vscode/
.DS_Store

# Apify storage folder
storage/

# Python virtual environment
.venv/
.env/

# Python (and python tools) cache files
__pycache__/
*.pyc
.ruff_cache/
.mypy_cache/
.pytest_cache/

# Python build files
__pypackages__/
dist/
build/
*.egg-info/
*.egg

# log files
*.log

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf

# IDE
.idea/
.vscode/
.DS_Store

# Apify storage folder
storage/

# Python virtual environment
.venv/
.env/

# Python (and python tools) cache files
__pycache__/
*.pyc
.ruff_cache/
.mypy_cache/
.pytest_cache/

# Python build files
__pypackages__/
dist/
build/
*.egg-info/
*.egg

# log files
*.log

1# Feel free to add your Python dependencies below. For formatting guidelines, see:
2# https://pip.pypa.io/en/latest/reference/requirements-file-format/
3
4apify[scrapy] ~= 1.7.0
5nest-asyncio ~= 1.5.8
6scrapy ~= 2.11.1

[settings]
default = src.settings

[deploy]
project = src

1"""
2This module transforms a Scrapy project into an Apify Actor, handling the configuration of logging, patching Scrapy's
3logging system, and establishing the required environment to run the Scrapy spider within the Apify platform.
4
5This file is specifically designed to be executed when the project is run as an Apify Actor using `apify run` locally
6or being run on the Apify platform. It is not being executed when running the project as a Scrapy project using
7`scrapy crawl title_spider`.
8
9We recommend you do not modify this file unless you really know what you are doing.
10"""
11
12# We need to configure the logging first before we import anything else, so that nothing else imports
13# `scrapy.utils.log` before we patch it.
14from __future__ import annotations
15from logging import StreamHandler, getLogger
16from typing import Any
17from scrapy.utils import log as scrapy_logging
18from scrapy.utils.project import get_project_settings
19from apify.log import ActorLogFormatter
20
21# Define names of the loggers.
22MAIN_LOGGER_NAMES = ['apify', 'apify_client', 'scrapy']
23OTHER_LOGGER_NAMES = ['filelock', 'hpack', 'httpcore', 'httpx', 'protego', 'twisted']
24ALL_LOGGER_NAMES = MAIN_LOGGER_NAMES + OTHER_LOGGER_NAMES
25
26# To change the logging level, modify the `LOG_LEVEL` field in `settings.py`. If the field is not present in the file,
27# Scrapy will default to `DEBUG`. This setting applies to all loggers. If you wish to change the logging level for
28# a specific logger, do it in this file.
29settings = get_project_settings()
30LOGGING_LEVEL = settings['LOG_LEVEL']
31
32# Define a logging handler which will be used for the loggers.
33apify_handler = StreamHandler()
34apify_handler.setFormatter(ActorLogFormatter(include_logger_name=True))
35
36
37def configure_logger(logger_name: str | None, log_level: str, *handlers: StreamHandler) -> None:
38    """
39    Configure a logger with the specified settings.
40
41    Args:
42        logger_name: The name of the logger to be configured.
43        log_level: The desired logging level ('DEBUG', 'INFO', 'WARNING', 'ERROR', ...).
44        handlers: Optional list of logging handlers.
45    """
46    logger = getLogger(logger_name)
47    logger.setLevel(log_level)
48    logger.handlers = []
49
50    for handler in handlers:
51        logger.addHandler(handler)
52
53
54# Apify loggers have to be set up here and in the `new_configure_logging` as well to be able to use them both from
55# the `main.py` and Scrapy components.
56for logger_name in MAIN_LOGGER_NAMES:
57    configure_logger(logger_name, LOGGING_LEVEL, apify_handler)
58
59# We can't attach our log handler to the loggers normally, because Scrapy would remove them in the `configure_logging`
60# call here: https://github.com/scrapy/scrapy/blob/2.11.0/scrapy/utils/log.py#L113 (even though
61# `disable_existing_loggers` is set to False :facepalm:). We need to monkeypatch Scrapy's `configure_logging` method
62# like this, so that our handler is attached right after Scrapy calls the `configure_logging` method, because
63# otherwise we would lose some log messages.
64old_configure_logging = scrapy_logging.configure_logging
65
66
67def new_configure_logging(*args: Any, **kwargs: Any) -> None:
68    """
69    We need to manually configure both the root logger and all Scrapy-associated loggers. Configuring only the root
70    logger is not sufficient, as Scrapy will override it with its own settings. Scrapy uses these four primary
71    loggers - https://github.com/scrapy/scrapy/blob/2.11.0/scrapy/utils/log.py#L60:L77. Therefore, we configure here
72    these four loggers and the root logger.
73    """
74    old_configure_logging(*args, **kwargs)
75
76    # We modify the root (None) logger to ensure proper display of logs from spiders when using the `self.logger`
77    # property within spiders. See details in the Spider logger property:
78    # https://github.com/scrapy/scrapy/blob/2.11.0/scrapy/spiders/__init__.py#L43:L46.
79    configure_logger(None, LOGGING_LEVEL, apify_handler)
80
81    # We modify other loggers only by setting up their log level. A custom log handler is added
82    # only to the root logger to avoid duplicate log messages.
83    for logger_name in ALL_LOGGER_NAMES:
84        configure_logger(logger_name, LOGGING_LEVEL)
85
86    # Set the HTTPX logger explicitly to the WARNING level, because it is too verbose and spams the logs with useless
87    # messages, especially when running on the platform.
88    configure_logger('httpx', 'WARNING')
89
90
91scrapy_logging.configure_logging = new_configure_logging
92
93# Now we can do the rest of the setup
94import asyncio
95import os
96import nest_asyncio
97from scrapy.utils.reactor import install_reactor
98from .main import main
99
100# To ensure seamless compatibility between asynchronous libraries Twisted (used by Scrapy) and AsyncIO (used by Apify),
101# it is highly recommended to use AsyncioSelectorReactor as the Twisted reactor
102# The reactor installation must be done manually before calling `nest_asyncio.apply()`,
103# otherwise, it will not work correctly on Windows.
104install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
105nest_asyncio.apply()
106
107# Specify the path to the Scrapy project settings module
108os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings'
109
110# Run the Apify main coroutine
111asyncio.run(main())

1"""
2Scrapy item models module
3
4This module defines Scrapy item models for scraped data. Items represent structured data
5extracted by spiders.
6
7For detailed information on creating and utilizing items, refer to the official documentation:
8https://docs.scrapy.org/en/latest/topics/items.html
9"""
10
11from scrapy import Field, Item
12
13
14class TitleItem(Item):
15    """
16    Represents a title item scraped from a web page.
17    """
18
19    url = Field()
20    title = Field()

1"""
2This module defines the main coroutine for the Apify Scrapy Actor, executed from the __main__.py file. The coroutine
3processes the Actor's input and executes the Scrapy spider. Additionally, it updates Scrapy project settings by
4applying Apify-related settings. Which includes adding a custom scheduler, retry middleware, and an item pipeline
5for pushing data to the Apify dataset.
6
7Customization:
8--------------
9
10Feel free to customize this file to add specific functionality to the Actor, such as incorporating your own Scrapy
11components like spiders and handling Actor input. However, make sure you have a clear understanding of your
12modifications. For instance, removing `apply_apify_settings` break the integration between Scrapy and Apify.
13
14Documentation:
15--------------
16
17For an in-depth description of the Apify-Scrapy integration process, our Scrapy components, known limitations and
18other stuff, please refer to the following documentation page: https://docs.apify.com/cli/docs/integrating-scrapy.
19"""
20
21from __future__ import annotations
22
23from scrapy.crawler import CrawlerProcess
24
25from apify import Actor
26from apify.scrapy.utils import apply_apify_settings
27
28# Import your Scrapy spider here
29from .spiders.title import TitleSpider as Spider
30
31# Default input values for local execution using `apify run`
32LOCAL_DEFAULT_START_URLS = [{'url': 'https://apify.com'}]
33
34
35async def main() -> None:
36    """
37    Apify Actor main coroutine for executing the Scrapy spider.
38    """
39    async with Actor:
40        Actor.log.info('Actor is being executed...')
41
42        # Process Actor input
43        actor_input = await Actor.get_input() or {}
44        start_urls = actor_input.get('startUrls', LOCAL_DEFAULT_START_URLS)
45        proxy_config = actor_input.get('proxyConfiguration')
46
47        # Add start URLs to the request queue
48        rq = await Actor.open_request_queue()
49        for start_url in start_urls:
50            url = start_url.get('url')
51            await rq.add_request(request={'url': url, 'method': 'GET'})
52
53        # Apply Apify settings, it will override the Scrapy project settings
54        settings = apply_apify_settings(proxy_config=proxy_config)
55
56        # Execute the spider using Scrapy CrawlerProcess
57        process = CrawlerProcess(settings, install_root_handler=False)
58        process.crawl(Spider)
59        process.start()

1"""
2Scrapy middlewares module
3
4This module defines Scrapy middlewares. Middlewares are processing components that handle requests and
5responses, typically used for adding custom headers, retrying requests, and handling exceptions.
6
7There are 2 types of middlewares: spider middlewares and downloader middlewares. For detailed information
8on creating and utilizing them, refer to the official documentation:
9https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
10https://docs.scrapy.org/en/latest/topics/spider-middleware.html
11"""
12
13from __future__ import annotations
14from typing import Generator, Iterable
15
16from scrapy import Request, Spider, signals
17from scrapy.crawler import Crawler
18from scrapy.http import Response
19
20# useful for handling different item types with a single interface
21from itemadapter import is_item, ItemAdapter
22
23
24class TitleSpiderMiddleware:
25    # Not all methods need to be defined. If a method is not defined,
26    # scrapy acts as if the spider middleware does not modify the
27    # passed objects.
28
29    @classmethod
30    def from_crawler(cls, crawler: Crawler) -> TitleSpiderMiddleware:
31        # This method is used by Scrapy to create your spiders.
32        s = cls()
33        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
34        return s
35
36    def process_spider_input(self, response: Response, spider: Spider) -> None:
37        # Called for each response that goes through the spider
38        # middleware and into the spider.
39
40        # Should return None or raise an exception.
41        return None
42
43    def process_spider_output(
44        self,
45        response: Response,
46        result: Iterable,
47        spider: Spider,
48    ) -> Generator[Iterable[Request] | None, None, None]:
49        # Called with the results returned from the Spider, after
50        # it has processed the response.
51
52        # Must return an iterable of Request, or item objects.
53        for i in result:
54            yield i
55
56    def process_spider_exception(
57        self,
58        response: Response,
59        exception: BaseException,
60        spider: Spider,
61    ) -> Iterable[Request] | None:
62        # Called when a spider or process_spider_input() method
63        # (from other spider middleware) raises an exception.
64
65        # Should return either None or an iterable of Request or item objects.
66        pass
67
68    def process_start_requests(
69        self, start_requests: Iterable[Request], spider: Spider
70    ) -> Iterable[Request]:  # Called with the start requests of the spider, and works
71        # similarly to the process_spider_output() method, except
72        # that it doesn’t have a response associated.
73
74        # Must return only requests (not items).
75        for r in start_requests:
76            yield r
77
78    def spider_opened(self, spider: Spider) -> None:
79        pass
80
81
82class TitleDownloaderMiddleware:
83    # Not all methods need to be defined. If a method is not defined,
84    # scrapy acts as if the downloader middleware does not modify the
85    # passed objects.
86
87    @classmethod
88    def from_crawler(cls, crawler: Crawler) -> TitleDownloaderMiddleware:
89        # This method is used by Scrapy to create your spiders.
90        s = cls()
91        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
92        return s
93
94    def process_request(self, request: Request, spider: Spider) -> Request | Response | None:
95        # Called for each request that goes through the downloader
96        # middleware.
97
98        # Must either:
99        # - return None: continue processing this request
100        # - or return a Response object
101        # - or return a Request object
102        # - or raise IgnoreRequest: process_exception() methods of
103        #   installed downloader middleware will be called
104        return None
105
106    def process_response(self, request: Request, response: Response, spider: Spider) -> Request | Response:
107        # Called with the response returned from the downloader.
108
109        # Must either;
110        # - return a Response object
111        # - return a Request object
112        # - or raise IgnoreRequest
113        return response
114
115    def process_exception(self, request: Request, exception: BaseException, spider: Spider) -> Response | None:
116        # Called when a download handler or a process_request()
117        # (from other downloader middleware) raises an exception.
118
119        # Must either:
120        # - return None: continue processing this exception
121        # - return a Response object: stops process_exception() chain
122        # - return a Request object: stops process_exception() chain
123        pass
124
125    def spider_opened(self, spider: Spider) -> None:
126        pass

1"""
2Scrapy item pipelines module
3
4This module defines Scrapy item pipelines for scraped data. Item pipelines are processing components
5that handle the scraped items, typically used for cleaning, validating, and persisting data.
6
7For detailed information on creating and utilizing item pipelines, refer to the official documentation:
8http://doc.scrapy.org/en/latest/topics/item-pipeline.html
9"""
10
11from scrapy import Spider
12
13from .items import TitleItem
14
15
16class TitleItemPipeline:
17    """
18    This item pipeline defines processing steps for TitleItem objects scraped by spiders.
19    """
20
21    def process_item(self, item: TitleItem, spider: Spider) -> TitleItem:
22        # Do something with the item here, such as cleaning it or persisting it to a database
23        return item

1"""
2Scrapy settings module
3
4This module contains Scrapy settings for the project, defining various configurations and options.
5
6For more comprehensive details on Scrapy settings, refer to the official documentation:
7http://doc.scrapy.org/en/latest/topics/settings.html
8"""
9
10# You can update these options and add new ones
11BOT_NAME = 'titlebot'
12DEPTH_LIMIT = 1
13LOG_LEVEL = 'INFO'
14NEWSPIDER_MODULE = 'src.spiders'
15REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
16ROBOTSTXT_OBEY = True
17SPIDER_MODULES = ['src.spiders']
18ITEM_PIPELINES = {
19    'src.pipelines.TitleItemPipeline': 123,
20}
21SPIDER_MIDDLEWARES = {
22    'src.middlewares.TitleSpiderMiddleware': 543,
23}
24DOWNLOADER_MIDDLEWARES = {
25    'src.middlewares.TitleDownloaderMiddleware': 543,
26}

1"""
2Scrapy spiders package
3
4This package contains the spiders for your Scrapy project. Spiders are the classes that define how to scrape
5and process data from websites.
6
7For detailed information on creating and utilizing spiders, refer to the official documentation:
8https://docs.scrapy.org/en/latest/topics/spiders.html
9"""

1from __future__ import annotations
2
3from typing import Generator
4from urllib.parse import urljoin
5
6from scrapy import Request, Spider
7from scrapy.responsetypes import Response
8
9from ..items import TitleItem
10
11
12class TitleSpider(Spider):
13    """
14    Scrapes title pages and enqueues all links found on the page.
15    """
16
17    name = 'title_spider'
18
19    # The `start_urls` specified in this class will be merged with the `start_urls` value from your Actor input
20    # when the project is executed using Apify.
21    start_urls = ['https://apify.com/']
22
23    def parse(self, response: Response) -> Generator[TitleItem | Request, None, None]:
24        """
25        Parse the web page response.
26
27        Args:
28            response: The web page response.
29
30        Yields:
31            Yields scraped TitleItem and Requests for links.
32        """
33        self.logger.info('TitleSpider is parsing %s...', response)
34
35        # Extract and yield the TitleItem
36        url = response.url
37        title = response.css('title::text').extract_first()
38        yield TitleItem(url=url, title=title)
39
40        # Extract all links from the page, create Requests out of them, and yield them
41        for link_href in response.css('a::attr("href")'):
42            link_url = urljoin(response.url, link_href.get())
43            if link_url.startswith(('http://', 'https://')):
44                yield Request(link_url)