# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python:3.11

# Second, copy just requirements.txt into the actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Specify how to launch the source code of your actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]


.actor/Dockerfile

airbnb-actor

URLs and their titles

Airbnb actor

.actor/actor.json

Location

Python Scrapy Scraper

.actor/input_schema.json

# This package will contain the spiders of your Scrapy project
#
# Please refer to the Scrapy documentation
# for information on how to create and manage your spiders.


src/spiders/__init__.py

import scrapy
import json
import re
import base64
import urllib.parse


class AirbnbSpider(scrapy.Spider):
    name = 'airbnb'

    def __init__(self, location):
        location = location.replace('-', '~').replace(' ', '-')
        self.location = location

    def start_requests(self):
        self.url = f'https://www.airbnb.com/s/{self.location}/homes'
        yield scrapy.Request(self.url)

    def encode_room_id(self, room_id):
        '''Encodes using base 64
        '''

        text = f"StayListing:{room_id}"
        encoded_text = base64.b64encode(text.encode())

        room_id_encoded = str(encoded_text, "utf-8")
        return room_id_encoded

    def parse(self, response):

        # Extract the json_file
        data_file_text = response.xpath(
            '//script[@id="data-deferred-state"]/text()').get()
        property_json = json.loads(data_file_text)
        try:
            property_details = property_json['niobeMinimalClientData'][0][1]['data']['presentation'][
                'explore']['sections']['sectionIndependentData']['staysSearch']['searchResults']
        except Exception as e:
            self.logger.info("Error while processing the json file", e)
        for property_detail in property_details:
            room_id = property_detail.get("listing", {}).get("id")
            if room_id:
                url = f'https://www.airbnb.com/rooms/{room_id}'
                yield scrapy.Request(url=url, callback=self.parse_room,
                                     meta={
                                         'room_id': room_id,
                                     })

        # Next pagination
        try:
            cursor = property_json['niobeMinimalClientData'][0][1]['data']['presentation']['explore'][
                'sections']['sectionIndependentData']['staysSearch']['paginationInfo']['nextPageCursor']
        except:
            cursor = False
        if cursor:
            next_page = self.url + f'?cursor={cursor}'
            print("Next pages:", next_page)

            yield scrapy.Request(next_page, callback=self.parse)

    def parse_room(self, response):
        data_file_text = response.xpath(
            '//script[@id="data-deferred-state"]/text()').get()
        # with open('data_airbnb.json' ,'w') as f:
        #     f.write(data_file_text)
        if not data_file_text:
            max_try = response.meta.get('max_try', 0)
            if max_try < 3:
                max_try = max_try + 1
                print(f"Trying {max_try} again for crawling", response.url)
                with open('error_rooms.txt', 'a') as err_room:
                    err_room.write(f'{response.url} \n')

                # wait until networkidle and page query slector is not loaded fully

                yield scrapy.Request(url=response.url, callback=self.parse_room,
                                     meta={
                                         'max_try': max_try,




                                     })
        try:
            property_json = json.loads(data_file_text)
        except Exception as e:
            self.logger.info(
                f"Error while processing the json file {response.meta.get('room_id')}")
            if f'/rooms/plus/{response.meta.get("room_id")}' in response.text:
                room_plus_url = f'https://www.airbnb.com/rooms/plus/{response.meta.get("room_id")}'
                yield scrapy.Request(url=room_plus_url, callback=self.parse_room, meta={'room_id': response.meta.get('room_id')})
        property_sections = property_json['niobeMinimalClientData'][0][1]['data'][
            'presentation']['stayProductDetailPage']['sections']['sections']
        ammenities = [
            x for x in property_sections if x['sectionComponentType'] == 'AMENITIES_DEFAULT']
        if ammenities:
            ammenities_list = [k.get('title') for k in ammenities[0].get(
                'section').get('seeAllAmenitiesGroups')]
            ammenties_parse = re.findall("'title': '(.*?)'", str(ammenities))
        else:
            ammenities_parse = ''
        location_pdf = [
            x for x in property_sections if x['sectionComponentType'] == 'LOCATION_PDP']
        if location_pdf:
            location = location_pdf[0].get('section').get('subtitle')
        try:
            description = [x for x in property_sections if x['sectionComponentType']
                           == 'DESCRIPTION_DEFAULT'][0]['section']['htmlDescription']['htmlText']
        except:
            description = ""

        # OVERVIEW_DEFAULT
        overview_details = [
            x for x in property_sections if x['sectionComponentType'] == 'OVERVIEW_DEFAULT']
        if overview_details:
            overview_list = [x.get('title') for x in overview_details[0].get(
                'section').get('detailItems')]
        else:
            overview_list = []
        property_dict = property_json['niobeMinimalClientData'][0][1]['data'][
            'presentation']['stayProductDetailPage']['sections']['metadata']
        data_file_text = response.xpath(
            '//script[@id="data-state"]/text()').get()
        property_title = property_dict.get('sharingConfig').get('title')

        # TODO delete the below code if not required
        property_location = property_dict.get('sharingConfig').get('location')
        guests = property_dict.get('sharingConfig', {}).get('personCapacity')

        # reviews_ratings = response.xpath('//span[contains(text(),"reviews")]/text()').extract()
        images = property_dict.get('sharingConfig').get('imageUrl')
        property_type = property_dict.get('sharingConfig').get('propertyType')
        per_night = response.xpath(
            '//span[contains(text(),"per night")]/text()').get()
        review = property_dict.get('sharingConfig').get('reviewCount')
        rating = property_dict.get('sharingConfig').get('starRating')
        room_encoded = self.encode_room_id(response.meta.get('room_id'))
        extensions = {
            "persistedQuery": {
                "version": 1,
                "sha256Hash": "2a34e10a85daf07208acea811c266701e3ffa7b8143ffee084c739d1db0ec197"
            }
        }
        variables = {"id": room_encoded, "pdpSectionsRequest": {"adults": "1", "bypassTargetings": False, "categoryTag": None, "causeId": None, "children": None, "disasterId": None, "discountedGuestFeeVersion": None, "displayExtensions": None, "federatedSearchId": None, "forceBoostPriorityMessageType": None, "infants": None, "interactionType": None, "layouts": [
            "SIDEBAR", "SINGLE_COLUMN"], "pets": 0, "pdpTypeOverride": None, "preview": False, "previousStateCheckIn": None, "previousStateCheckOut": None, "priceDropSource": None, "privateBooking": False, "promotionUuid": None, "relaxedAmenityIds": None, "searchId": None, "selectedCancellationPolicyId": None, "selectedRatePlanId": None, "splitStays": None, "staysBookingMigrationEnabled": False, "translateUgc": None, "useNewSectionWrapperApi": False, "sectionIds": ["BOOK_IT_CALENDAR_SHEET", "POLICIES_DEFAULT", "BOOK_IT_SIDEBAR", "URGENCY_COMMITMENT_SIDEBAR", "BOOK_IT_NAV", "BOOK_IT_FLOATING_FOOTER", "EDUCATION_FOOTER_BANNER", "URGENCY_COMMITMENT", "CANCELLATION_POLICY_PICKER_MODAL", "EDUCATION_FOOTER_BANNER_MODAL"], "checkIn": None, "checkOut": None, "p3ImpressionId": ""}}
        extensions = urllib.parse.quote(json.dumps(extensions))
        variables = urllib.parse.quote(json.dumps(variables))
        # urlencoder

        new_url = f'https://www.airbnb.com/api/v3/StaysPdpSections?operationName=StaysPdpSections&locale=en&currency=USD&variables={variables}&extensions={extensions}'
        payload = {}
        headers = {
            'x-airbnb-api-key': 'd306zoyjsyarp7ifhu67rjxn52tv0t20',
            'x-airbnb-graphql-platform-client': 'minimalist-niobe',
            'x-airbnb-supports-airlock-v2': 'true',
        }

        items = {
            'title': property_title,
            'per_night': per_night,
            'description': description,
            'amenities': ammenties_parse,
            'url': response.url,
            'location': property_location,
            'overview': overview_list,
            'review': review,
            'rating': rating,
            'url': response.url,
            'property_id': response.meta['room_id'],
            'property_type': property_type,
            'guests': guests,
            'images': images,


        }

        yield scrapy.Request(url=new_url, headers=headers, callback=self.parse_price, meta={'items': items})

        # TODO price need another request

    def parse_price(self, response):

        # section_id BOOK_IT_CALENDAR_SHEET

        try:
            price = response.json()[
                'data']['presentation']['stayProductDetailPage']['sections']['sections']
            # [0]['section']['structuredDisplayPrice']['primaryLine']['price']
            price_section = [x for x in price if x['sectionId']
                             == 'BOOK_IT_CALENDAR_SHEET'][0]
            price = price_section['section']['structuredDisplayPrice']['primaryLine']['price']
            self.logger.info(price)
            if price:
                price = price.replace('$', '')
                price = price.replace(',', '')
                price = int(price)
        except:
            price = 'N/A'

        items = response.meta['items']
        items['per_night'] = price
        items['title'] = items['title'].split('·')[0]
        yield items


src/spiders/airbnb_spider.py

src/__init__.py

# This file is executed only when the project is run as an Apify Actor,
# not when you run it locally using `scrapy crawl`.

# We need to configure the logging first before we import anything else,
# so that nothing else imports `scrapy.utils.log` before we patch it.
import logging
from apify.log import ActorLogFormatter
import scrapy.utils.log

handler = logging.StreamHandler()
handler.setFormatter(ActorLogFormatter())

apify_logger = logging.getLogger('apify')
apify_logger.setLevel(logging.DEBUG)
apify_logger.addHandler(handler)

apify_client_logger = logging.getLogger('apify_client')
apify_client_logger.setLevel(logging.DEBUG)
apify_client_logger.addHandler(handler)


# We can't attach our log handler to the loggers normally,
# because Scrapy would remove them in the `configure_logging` call here:
# https://github.com/scrapy/scrapy/blob/a5c1ef82762c6c0910abea00c0a6249c40005e44/scrapy/utils/log.py#L95
# (even though `disable_existing_loggers` is set to False :facepalm:).
# We need to monkeypatch Scrapy's `configure_logging` method like this,
# so that our handler is attached right after Scrapy calls the `configure_logging` method,
# because otherwise we would lose some log messages.
old_configure_logging = scrapy.utils.log.configure_logging

def new_configure_logging(*args, **kwargs):
    old_configure_logging(*args, **kwargs)

    # Scrapy uses these four main loggers: https://github.com/scrapy/scrapy/blob/a5c1ef82762c6c0910abea00c0a6249c40005e44/scrapy/utils/log.py#L44-L61
    logging.getLogger('scrapy').addHandler(handler)
    logging.getLogger('twisted').addHandler(handler)
    logging.getLogger('filelock').addHandler(handler)
    logging.getLogger('hpack').addHandler(handler)

scrapy.utils.log.configure_logging = new_configure_logging


# Now we can do the rest of the setup
import asyncio
import os

import nest_asyncio

from scrapy.utils.reactor import install_reactor
from .main import main

# This is necessary so that twisted and asyncio work well together
install_reactor('twisted.internet.asyncioreactor.AsyncioSelectorReactor')
nest_asyncio.apply()

os.environ['SCRAPY_SETTINGS_MODULE'] = 'src.settings'

asyncio.run(main())


src/__main__.py

# Define the models for your scraped items here
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy


class TitlebotItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    pass


src/items.py

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from apify import Actor

from .pipelines import ActorDatasetPushPipeline
from .spiders.airbnb_spider import AirbnbSpider


async def main():
    async with Actor:
        actor_input = await Actor.get_input() or {}
        #max_depth = actor_input.get('max_depth', )
        location = actor_input.get('location', 'New York--NY--United-States')

        settings = get_project_settings()
        settings['ITEM_PIPELINES'] = { ActorDatasetPushPipeline: 1 }
        #settings['DEPTH_LIMIT'] = max_depth

        process = CrawlerProcess(settings, install_root_handler=False)

        # If you want to run multiple spiders, call `process.crawl` for each of them here
        process.crawl(AirbnbSpider, location=location)

        process.start()


src/main.py

# Define your item pipelines here
#
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html

from apify import Actor
from itemadapter import ItemAdapter


# Used to output the items into the actor's default dataset
# Enabled only when the project is run as an actor
class ActorDatasetPushPipeline:
    async def process_item(self, item, spider):
        item_dict = ItemAdapter(item).asdict()
        await Actor.push_data(item_dict)
        return item


src/pipelines.py

# Scrapy settings for this project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html


BOT_NAME = 'titlebot'

SPIDER_MODULES = ['src.spiders']
NEWSPIDER_MODULE = 'src.spiders'

REQUEST_FINGERPRINTER_IMPLEMENTATION = '2.7'
TWISTED_REACTOR = 'twisted.internet.asyncioreactor.AsyncioSelectorReactor'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False


src/settings.py

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
.venv

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

# This file tells Git which files shouldn't be added to source control

.idea
.DS_Store

apify_storage
storage

.venv/
.env/
__pypackages__
dist/
build/
*.egg-info/
*.egg

__pycache__

.mypy_cache
.dmypy.json
dmypy.json
.pytest_cache

.scrapy
*.log

# Added by Apify CLI
node_modules
.venv


.gitignore

# Add your dependencies here.
# See https://pip.pypa.io/en/latest/reference/requirements-file-format/
# for how to format them
apify ~= 1.1.1
nest-asyncio ~= 1.5.6
scrapy ~= 2.9.0


requirements.txt

[settings]
default = src.settings

[deploy]
project = src


scrapy.cfg

Effortlessly find Airbnb property details for any location with our web scraper. Input your desired location and access comprehensive information on titles, prices, amenities, reviews & more, saving time and hassle. Discover your perfect rental unit with ease!
*Note : Try to narrow down the location

Airbnb Properties details by location search in USD

Airbnb Properties details by location search in USD · Apify

Input · Airbnb Properties details by location search in USD · Apify

API · Airbnb Properties details by location search in USD · Apify

Related Actors · Airbnb Properties details by location search in USD · Apify

Effortlessly find Airbnb property details for any location with our web scraper. Input your desired location and access comprehensive information on titles, prices, amenities, reviews & more, savin...

Airbnb Properties details by location search in USD

Actor - Airbnb Properties details by location search in USD

Airbnb Scraper

Project Description

Bugs, fixes, updates, and changelog

Input Data

Sample Data

Contact

Where next?