Listing Sleuth

.git
.mise.toml
.nvim.lua
storage

# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
#   For a library or package, you might want to ignore these files since the code is
#   intended to run in multiple environments; otherwise, check them in:
.python-version

# pdm
#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
#   in version control.
#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
#  and can be added to the global gitignore or merged into this file.  For a more nuclear
#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/

.mise.toml
.nvim.lua
storage

# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore

# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class

# C extensions
*.so

# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST

# PyInstaller
#  Usually these files are written by a python script from a template
#  before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec

# Installer logs
pip-log.txt
pip-delete-this-directory.txt

# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/

# Translations
*.mo
*.pot

# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal

# Flask stuff:
instance/
.webassets-cache

# Scrapy stuff:
.scrapy

# Sphinx documentation
docs/_build/

# PyBuilder
.pybuilder/
target/

# Jupyter Notebook
.ipynb_checkpoints

# IPython
profile_default/
ipython_config.py

# pyenv
#   For a library or package, you might want to ignore these files since the code is
#   intended to run in multiple environments; otherwise, check them in:
.python-version

# pdm
#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
#pdm.lock
#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
#   in version control.
#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
.pdm.toml
.pdm-python
.pdm-build/

# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
__pypackages__/

# Celery stuff
celerybeat-schedule
celerybeat.pid

# SageMath parsed files
*.sage.py

# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/

# Spyder project settings
.spyderproject
.spyproject

# Rope project settings
.ropeproject

# mkdocs documentation
/site

# mypy
.mypy_cache/
.dmypy.json
dmypy.json

# Pyre type checker
.pyre/

# pytype static type analyzer
.pytype/

# Cython debug symbols
cython_debug/

# PyCharm
#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
#  and can be added to the global gitignore or merged into this file.  For a more nuclear
#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/

# Added by Apify CLI
node_modules

{
  "location": "San Francisco, CA",
  "propertyType": "apartment",
  "minBedrooms": 2,
  "maxBedrooms": 3,
  "minPrice": 1500,
  "maxPrice": 3000,
  "amenities": ["parking", "gym"],
  "searchType": "rent",
  "sources": ["zillow", "apartments"]
}

MIT License

Copyright (c) 2024 Listing Sleuth

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

1apify < 3.0
2langchain-openai==0.3.6
3langgraph==0.2.73
4aiohttp>=3.8.0
5langchain>=0.1.0
6pydantic>=2.0.0
7langchain-core>=0.1.0
8langchain_community==0.3.19

# First, specify the base Docker image.
# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.
# You can also use any other image from Docker Hub.
FROM apify/actor-python-playwright:3.13

# Install build dependencies first
RUN apt-get update && apt-get install -y build-essential gcc g++ python3-dev

# Second, copy just requirements.txt into the Actor image,
# since it should be the only file that affects the dependency install in the next step,
# in order to speed up the build
COPY requirements.txt ./

# Install the packages specified in requirements.txt,
# Print the installed Python version, pip version
# and all installed packages with their versions for debugging
RUN echo "Python version:" \
 && python --version \
 && echo "Pip version:" \
 && pip --version \
 && echo "Installing dependencies:" \
 && pip install --only-binary=:all: -r requirements.txt \
 && echo "All installed Python packages:" \
 && pip freeze

# Next, copy the remaining files and directories with the source code.
# Since we do this after installing the dependencies, quick build will be really fast
# for most source file changes.
COPY . ./

# Use compileall to ensure the runnability of the Actor Python code.
RUN python3 -m compileall -q .

# Specify how to launch the source code of your Actor.
# By default, the "python3 -m src" command is run
CMD ["python3", "-m", "src"]

{
	"actorSpecification": 1,
	"name": "listing-sleuth",
	"title": "Listing Sleuth - Real Estate Monitor",
	"description": "Monitors real estate listings across multiple platforms based on user-specified criteria",
	"version": "0.1",
	"buildTag": "latest",
	"restart": {
		"horizontalScaling": true
	},
	"dockerfile": "./Dockerfile",
	"input": "./input_schema.json",
	"storages": {
		"dataset": "./dataset_schema.json"
	},
	"license": "MIT",
	"monetization": {
		"type": "pay-per-event",
		"enabled": true,
		"priceSchemaPath": "./pay_per_event.json"
	}
}

{
    "actorSpecification": 1,
    "fields": {
        "type": "object",
        "properties": {
            "id": {
                "type": "string",
                "description": "Unique identifier for the property listing"
            },
            "title": {
                "type": "string",
                "description": "Property title or name"
            },
            "description": {
                "type": "string",
                "description": "Detailed description of the property"
            },
            "price": {
                "type": "number",
                "description": "Price of the property (in USD)"
            },
            "bedrooms": {
                "type": "number",
                "description": "Number of bedrooms"
            },
            "bathrooms": {
                "type": "number",
                "description": "Number of bathrooms"
            },
            "address": {
                "type": "string",
                "description": "Property address"
            },
            "property_type": {
                "type": "string",
                "description": "Type of property (apartment, house, condo, etc.)"
            },
            "source": {
                "type": "string",
                "description": "Source of the listing (zillow, realtor, apartments, etc.)"
            },
            "url": {
                "type": "string",
                "description": "Link to the original listing"
            },
            "amenities": {
                "type": "array",
                "description": "List of amenities available at the property",
                "items": {
                    "type": "string"
                }
            },
            "listed_date": {
                "type": "string",
                "description": "Date when the property was listed"
            },
            "is_new": {
                "type": "boolean",
                "description": "Whether this is a new listing since last search"
            }
        }
    },
    "views": {
        "overview": {
            "title": "Property Listings",
            "description": "Real estate property listings matching the search criteria",
            "transformation": {
                "fields": [
                    "id",
                    "title",
                    "price",
                    "bedrooms",
                    "bathrooms",
                    "address",
                    "property_type",
                    "source",
                    "url",
                    "listed_date",
                    "is_new"
                ]
            },
            "display": {
                "component": "table",
                "properties": {
                    "id": {
                        "label": "ID",
                        "format": "text"
                    },
                    "title": {
                        "label": "Title",
                        "format": "text"
                    },
                    "price": {
                        "label": "Price",
                        "format": "number"
                    },
                    "bedrooms": {
                        "label": "Bedrooms",
                        "format": "number"
                    },
                    "bathrooms": {
                        "label": "Bathrooms",
                        "format": "number"
                    },
                    "address": {
                        "label": "Address",
                        "format": "text"
                    },
                    "property_type": {
                        "label": "Property Type",
                        "format": "text"
                    },
                    "source": {
                        "label": "Source",
                        "format": "text"
                    },
                    "url": {
                        "label": "URL",
                        "format": "link"
                    },
                    "listed_date": {
                        "label": "Listed Date",
                        "format": "date"
                    },
                    "is_new": {
                        "label": "New Listing",
                        "format": "boolean"
                    }
                }
            }
        },
        "details": {
            "title": "Detailed View",
            "description": "Detailed information about property listings",
            "transformation": {
                "fields": [
                    "id",
                    "title",
                    "description",
                    "price",
                    "bedrooms",
                    "bathrooms",
                    "address",
                    "property_type",
                    "source",
                    "url",
                    "amenities",
                    "listed_date",
                    "is_new"
                ]
            },
            "display": {
                "component": "table",
                "properties": {
                    "description": {
                        "label": "Description",
                        "format": "text"
                    },
                    "amenities": {
                        "label": "Amenities",
                        "format": "array"
                    }
                }
            }
        }
    }
}

{
    "title": "Listing Sleuth - Real Estate Monitor",
    "type": "object",
    "schemaVersion": 1,
    "properties": {
        "location": {
            "title": "Location",
            "type": "string",
            "description": "City or neighborhood to search in (e.g., 'San Francisco, CA')",
            "editor": "textfield"
        },
        "propertyType": {
            "title": "Property Type",
            "type": "string",
            "description": "Type of property to look for",
            "enum": ["apartment", "house", "condo", "townhouse", "any"],
            "enumTitles": ["Apartment", "House", "Condo", "Townhouse", "Any"],
            "default": "any",
            "editor": "select"
        },
        "minBedrooms": {
            "title": "Minimum Bedrooms",
            "type": "integer",
            "description": "Minimum number of bedrooms",
            "default": 1,
            "minimum": 0,
            "editor": "number"
        },
        "maxBedrooms": {
            "title": "Maximum Bedrooms",
            "type": "integer",
            "description": "Maximum number of bedrooms (leave blank for no maximum)",
            "minimum": 0,
            "nullable": true,
            "editor": "number"
        },
        "minPrice": {
            "title": "Minimum Price",
            "type": "integer",
            "description": "Minimum price (in USD)",
            "default": 0,
            "minimum": 0,
            "editor": "number"
        },
        "maxPrice": {
            "title": "Maximum Price",
            "type": "integer",
            "description": "Maximum price (in USD)",
            "minimum": 0,
            "nullable": true,
            "editor": "number"
        },
        "amenities": {
            "title": "Amenities",
            "type": "array",
            "description": "Desired amenities for the property",
            "editor": "stringList",
            "default": []
        },
        "searchType": {
            "title": "Search Type",
            "type": "string",
            "description": "Type of search to perform",
            "enum": ["rent", "buy"],
            "enumTitles": ["Rent", "Buy"],
            "default": "rent",
            "editor": "select"
        },
        "sources": {
            "title": "Data Sources",
            "type": "array",
            "description": "Sources to search for listings",
            "editor": "stringList",
            "default": ["zillow", "realtor", "apartments"]
        },
        "llmApiToken": {
            "title": "LLM API Token",
            "type": "string",
            "description": "OpenAI API token for processing results (optional)",
            "editor": "textfield",
            "nullable": true
        }
    },
    "required": ["location"]
}

{
    "actor-start": {
        "eventTitle": "Search Initiated",
        "eventDescription": "Flat fee for starting a real estate search.",
        "eventPriceUsd": 0.1
    },
    "property-found": {
        "eventTitle": "Property Found",
        "eventDescription": "Fee for each property matching your criteria.",
        "eventPriceUsd": 0.05
    },
    "search-completed": {
        "eventTitle": "Search Completed",
        "eventDescription": "Fee for completing a full property search across all selected platforms.",
        "eventPriceUsd": 0.3
    }
}

1

1import asyncio
2
3from .main import main
4
5# Execute the Actor entry point.
6asyncio.run(main())

1"""Main entry point for the Listing Sleuth Apify Actor.
2
3This module contains the main entry point for the Actor, which searches for real estate
4listings based on user-specified criteria.
5"""
6
7import os
8import sys
9import json
10from apify import Actor
11from dotenv import load_dotenv
12
13from .models.property import SearchCriteria
14from .search_agent import SearchAgentCrew
15
16# Load environment variables from .env file if present
17load_dotenv()
18
19
20async def main() -> None:
21    """Main entry point for the Apify Actor.
22    
23    This function initializes the Actor, processes input data, runs the search agent,
24    and saves the results to the Actor's dataset.
25    """
26    # Enter the context of the Actor.
27    async with Actor:
28        # Log the Actor's version
29        Actor.log.info(f"Listing Sleuth is starting...")
30        
31        # Charge for actor start
32        await Actor.charge('actor-start')
33        
34        # Retrieve the Actor input, and use default values if not provided.
35        actor_input = await Actor.get_input() or {}
36        
37        # For local testing, try to load from INPUT.json if actor_input is empty
38        if not actor_input or 'location' not in actor_input:
39            try:
40                if os.path.exists('INPUT.json'):
41                    with open('INPUT.json', 'r') as f:
42                        actor_input = json.load(f)
43                    Actor.log.info(f"Loaded input from INPUT.json: {actor_input}")
44            except Exception as e:
45                Actor.log.error(f"Error loading from INPUT.json: {str(e)}")
46        
47        Actor.log.info(f"Using input: {actor_input}")
48        
49        # Parse location (required)
50        location = actor_input.get("location")
51        if not location:
52            Actor.log.error("No location specified in Actor input, exiting...")
53            # Just exit with an error code
54            sys.exit(1)
55        
56        # Parse other inputs with defaults
57        property_type = actor_input.get("propertyType", "any")
58        min_bedrooms = int(actor_input.get("minBedrooms", 1))
59        max_bedrooms = actor_input.get("maxBedrooms")
60        if max_bedrooms is not None:
61            max_bedrooms = int(max_bedrooms)
62        
63        min_price = float(actor_input.get("minPrice", 0))
64        max_price = actor_input.get("maxPrice")
65        if max_price is not None:
66            max_price = float(max_price)
67        
68        # Amenities as a list
69        amenities = actor_input.get("amenities", [])
70        
71        # Search type (rent/buy)
72        search_type = actor_input.get("searchType", "rent")
73        
74        # Data sources to search
75        sources = actor_input.get("sources", ["zillow", "realtor", "apartments"])
76        
77        # LLM API token (optional)
78        llm_api_token = actor_input.get("llmApiToken") or os.environ.get("OPENAI_API_KEY")
79        
80        # Create search criteria
81        search_criteria = SearchCriteria(
82            location=location,
83            property_type=property_type,
84            min_bedrooms=min_bedrooms,
85            max_bedrooms=max_bedrooms,
86            min_price=min_price,
87            max_price=max_price,
88            amenities=amenities,
89            search_type=search_type,
90            sources=sources,
91            llm_api_token=llm_api_token
92        )
93        
94        Actor.log.info(f"Search criteria: {search_criteria}")
95        
96        # Create and run the search agent
97        search_agent = SearchAgentCrew(search_criteria)
98        results = search_agent.run()
99        
100        # Charge for each property found
101        if results.total_results > 0:
102            await Actor.charge('property-found', count=results.total_results)
103        
104        # Log results
105        Actor.log.info(f"Search complete. Found {results.total_results} properties.")
106        Actor.log.info(f"New listings: {results.new_results}")
107        
108        # Charge for search completion
109        await Actor.charge('search-completed')
110        
111        # The results have already been saved to the dataset by the search agent

1"""Search agent for real estate properties."""
2
3import os
4import json
5from typing import List, Dict, Any, Optional, Tuple
6from datetime import datetime
7from crewai import Agent, Task, Crew
8from langchain.tools import BaseTool
9from langchain_openai import ChatOpenAI
10from apify import Actor
11
12from .models.property import PropertyListing, SearchCriteria, SearchResults
13from .scrapers.zillow import ZillowScraper
14from .scrapers.realtor import RealtorScraper
15from .scrapers.apartments import ApartmentsScraper
16from .utils.llm import filter_properties_with_llm, summarize_property
17from .utils.storage import (
18    load_previous_results,
19    mark_new_listings,
20    save_search_results,
21    push_results_to_dataset
22)
23
24
25class SearchTool(BaseTool):
26    """Tool for searching real estate listings."""
27    
28    name = "search_real_estate"
29    description = "Search for real estate listings based on search criteria"
30    search_criteria: SearchCriteria = None
31    
32    def __init__(self, search_criteria: SearchCriteria):
33        """Initialize the search tool.
34        
35        Args:
36            search_criteria: Search criteria
37        """
38        super().__init__()
39        self.search_criteria = search_criteria
40    
41    def _run(self, query: str) -> Dict[str, Any]:
42        """Run the search tool.
43        
44        Args:
45            query: Search query (not used, but required by BaseTool)
46            
47        Returns:
48            Search results
49        """
50        # Initialize scrapers
51        scrapers = []
52        if "zillow" in self.search_criteria.sources:
53            scrapers.append(ZillowScraper(self.search_criteria))
54        if "realtor" in self.search_criteria.sources:
55            scrapers.append(RealtorScraper(self.search_criteria))
56        if "apartments" in self.search_criteria.sources:
57            scrapers.append(ApartmentsScraper(self.search_criteria))
58        
59        # Run scrapers
60        all_listings = []
61        sources_searched = []
62        
63        for scraper in scrapers:
64            try:
65                listings = scraper.scrape()
66                all_listings.extend(listings)
67                sources_searched.append(scraper.source_name)
68            except Exception as e:
69                Actor.log.exception(f"Error scraping {scraper.source_name}: {e}")
70        
71        # Load previous results
72        previous_results = load_previous_results(self.search_criteria)
73        
74        # Mark new listings
75        marked_listings = mark_new_listings(all_listings, previous_results)
76        
77        # Create search results
78        results = SearchResults(
79            search_criteria=self.search_criteria,
80            results=marked_listings,
81            total_results=len(marked_listings),
82            new_results=sum(1 for listing in marked_listings if listing.is_new),
83            sources_searched=sources_searched
84        )
85        
86        # Save results
87        save_search_results(results)
88        push_results_to_dataset(results)
89        
90        # Return results
91        return {
92            "total_results": results.total_results,
93            "new_results": results.new_results,
94            "sources_searched": results.sources_searched,
95            "search_date": results.search_date.isoformat()
96        }
97    
98    async def _arun(self, query: str) -> Dict[str, Any]:
99        """Async version of _run.
100        
101        Args:
102            query: Search query
103            
104        Returns:
105            Search results
106        """
107        return self._run(query)
108
109
110class FilterTool(BaseTool):
111    """Tool for filtering property listings with LLM."""
112    
113    name = "filter_properties"
114    description = "Filter property listings based on search criteria using LLM"
115    search_criteria: SearchCriteria = None
116    
117    def __init__(self, search_criteria: SearchCriteria):
118        """Initialize the filter tool.
119        
120        Args:
121            search_criteria: Search criteria
122        """
123        super().__init__()
124        self.search_criteria = search_criteria
125    
126    def _run(self, query: str) -> Dict[str, Any]:
127        """Run the filter tool.
128        
129        Args:
130            query: Filter query (not used, but required by BaseTool)
131            
132        Returns:
133            Filtered search results
134        """
135        # Try to load saved results
136        try:
137            results_dict = None
138            
139            # Try to load from Apify KV store if available
140            if hasattr(Actor, 'main_kv_store'):
141                results_dict = Actor.main_kv_store.get_value("search_results")
142            # Otherwise try to load from local file
143            elif os.path.exists("storage/key_value_stores/search_results.json"):
144                with open("storage/key_value_stores/search_results.json", "r") as f:
145                    results_dict = json.load(f)
146            
147            if not results_dict:
148                return {"error": "No search results found"}
149            
150            # Convert to SearchResults
151            search_results = SearchResults(**results_dict)
152            
153            if not search_results.results:
154                return {"error": "No results to filter"}
155            
156            # Filter results with LLM if token is available
157            if self.search_criteria.llm_api_token:
158                filtered_listings = filter_properties_with_llm(
159                    search_results.results,
160                    self.search_criteria,
161                    self.search_criteria.llm_api_token
162                )
163                
164                # Update results
165                search_results.results = filtered_listings
166                search_results.total_results = len(filtered_listings)
167                
168                # Save filtered results
169                save_search_results(search_results)
170                
171                return {
172                    "total_results_after_filtering": len(filtered_listings),
173                    "filter_date": datetime.now().isoformat()
174                }
175            else:
176                return {"error": "No LLM API token provided for filtering"}
177        
178        except Exception as e:
179            Actor.log.exception(f"Error filtering properties: {e}")
180            return {"error": str(e)}
181    
182    async def _arun(self, query: str) -> Dict[str, Any]:
183        """Async version of _run.
184        
185        Args:
186            query: Filter query
187            
188        Returns:
189            Filtered search results
190        """
191        return self._run(query)
192
193
194class SummarizeTool(BaseTool):
195    """Tool for summarizing property listings."""
196    
197    name = "summarize_properties"
198    description = "Generate summaries of property listings"
199    search_criteria: SearchCriteria = None
200    
201    def __init__(self, search_criteria: SearchCriteria):
202        """Initialize the summarize tool.
203        
204        Args:
205            search_criteria: Search criteria
206        """
207        super().__init__()
208        self.search_criteria = search_criteria
209    
210    def _run(self, query: str) -> Dict[str, Any]:
211        """Run the summarize tool.
212        
213        Args:
214            query: Summarize query (not used, but required by BaseTool)
215            
216        Returns:
217            Summarized search results
218        """
219        # Try to load saved results
220        try:
221            results_dict = None
222            
223            # Try to load from Apify KV store if available
224            if hasattr(Actor, 'main_kv_store'):
225                results_dict = Actor.main_kv_store.get_value("search_results")
226            # Otherwise try to load from local file
227            elif os.path.exists("storage/key_value_stores/search_results.json"):
228                with open("storage/key_value_stores/search_results.json", "r") as f:
229                    results_dict = json.load(f)
230            
231            if not results_dict:
232                return {"error": "No search results found"}
233            
234            # Convert to SearchResults
235            search_results = SearchResults(**results_dict)
236            
237            if not search_results.results:
238                return {"error": "No results to summarize"}
239            
240            # Generate summaries if LLM API token is available
241            if self.search_criteria.llm_api_token:
242                summaries = []
243                
244                for listing in search_results.results:
245                    summary = summarize_property(listing, self.search_criteria.llm_api_token)
246                    summaries.append({
247                        "id": listing.id,
248                        "summary": summary,
249                        "is_new": listing.is_new
250                    })
251                
252                return {
253                    "summaries": summaries,
254                    "total_summaries": len(summaries),
255                    "summarize_date": datetime.now().isoformat()
256                }
257            else:
258                # Generate basic summaries without LLM
259                summaries = []
260                
261                for listing in search_results.results:
262                    basic_summary = (
263                        f"{listing.title}: {listing.bedrooms} bed, "
264                        f"{listing.bathrooms or 'unknown'} bath {listing.property_type} "
265                        f"for ${listing.price:,.2f} in {listing.address.city}, "
266                        f"{listing.address.state}."
267                    )
268                    
269                    summaries.append({
270                        "id": listing.id,
271                        "summary": basic_summary,
272                        "is_new": listing.is_new
273                    })
274                
275                return {
276                    "summaries": summaries,
277                    "total_summaries": len(summaries),
278                    "summarize_date": datetime.now().isoformat()
279                }
280        
281        except Exception as e:
282            Actor.log.exception(f"Error summarizing properties: {e}")
283            return {"error": str(e)}
284    
285    async def _arun(self, query: str) -> Dict[str, Any]:
286        """Async version of _run.
287        
288        Args:
289            query: Summarize query
290            
291        Returns:
292            Summarized search results
293        """
294        return self._run(query)
295
296
297class SearchAgentCrew:
298    """Crew of agents for property search."""
299    
300    def __init__(self, search_criteria: SearchCriteria):
301        """Initialize the search agent crew.
302        
303        Args:
304            search_criteria: Search criteria
305        """
306        self.search_criteria = search_criteria
307        self.llm = None
308        
309        # Initialize LLM if token is provided
310        if search_criteria.llm_api_token:
311            self.llm = ChatOpenAI(
312                api_key=search_criteria.llm_api_token,
313                temperature=0,
314                model="gpt-3.5-turbo"
315            )
316    
317    def run(self) -> SearchResults:
318        """Run the search agent crew.
319        
320        Returns:
321            Search results
322        """
323        # If no LLM, just run the search directly
324        if not self.llm:
325            Actor.log.info("No LLM API token provided, running basic search without agents")
326            search_tool = SearchTool(self.search_criteria)
327            search_tool._run("")
328            
329            # Load and return results
330            try:
331                # Try loading from Apify KV store if available
332                if hasattr(Actor, 'main_kv_store'):
333                    results_dict = Actor.main_kv_store.get_value("search_results")
334                # Otherwise try to load from local file
335                elif os.path.exists("storage/key_value_stores/search_results.json"):
336                    with open("storage/key_value_stores/search_results.json", "r") as f:
337                        results_dict = json.load(f)
338                else:
339                    results_dict = None
340                
341                if results_dict:
342                    return SearchResults(**results_dict)
343            except Exception as e:
344                Actor.log.error(f"Error loading search results: {e}")
345            
346            # Create empty results if loading failed
347            return SearchResults(
348                search_criteria=self.search_criteria,
349                results=[],
350                total_results=0,
351                new_results=0,
352                sources_searched=[]
353            )
354        
355        # Create tools
356        search_tool = SearchTool(self.search_criteria)
357        filter_tool = FilterTool(self.search_criteria)
358        summarize_tool = SummarizeTool(self.search_criteria)
359        
360        # Create agents
361        search_agent = Agent(
362            role="Real Estate Search Specialist",
363            goal="Find properties that match the search criteria",
364            backstory="You are an expert in finding real estate listings across multiple platforms.",
365            verbose=True,
366            allow_delegation=True,
367            tools=[search_tool],
368            llm=self.llm
369        )
370        
371        filter_agent = Agent(
372            role="Property Filter Specialist",
373            goal="Filter properties to find the best matches for the user",
374            backstory="You are an expert in analyzing property details and matching them with user preferences.",
375            verbose=True,
376            allow_delegation=True,
377            tools=[filter_tool],
378            llm=self.llm
379        )
380        
381        summarize_agent = Agent(
382            role="Property Summarizer",
383            goal="Create concise, informative summaries of properties",
384            backstory="You are skilled at creating appealing property descriptions that highlight key features.",
385            verbose=True,
386            allow_delegation=True,
387            tools=[summarize_tool],
388            llm=self.llm
389        )
390        
391        # Create tasks
392        search_task = Task(
393            description=(
394                f"Search for properties in {self.search_criteria.location} "
395                f"with {self.search_criteria.min_bedrooms}+ bedrooms, "
396                f"maximum price of ${self.search_criteria.max_price or 'any'}, "
397                f"property type: {self.search_criteria.property_type}. "
398                f"Search sources: {', '.join(self.search_criteria.sources)}."
399            ),
400            agent=search_agent,
401            expected_output="A report of the total number of properties found"
402        )
403        
404        filter_task = Task(
405            description=(
406                "Filter the search results to find properties that best match "
407                f"the user's criteria, especially regarding amenities: {', '.join(self.search_criteria.amenities)}"
408            ),
409            agent=filter_agent,
410            expected_output="A report of how many properties passed the filtering"
411        )
412        
413        summarize_task = Task(
414            description=(
415                "Create summaries for each property highlighting key features. "
416                "Mark new listings that weren't found in previous searches."
417            ),
418            agent=summarize_agent,
419            expected_output="Summaries of each property"
420        )
421        
422        # Create crew
423        crew = Crew(
424            agents=[search_agent, filter_agent, summarize_agent],
425            tasks=[search_task, filter_task, summarize_task],
426            verbose=True
427        )
428        
429        # Run the crew
430        try:
431            result = crew.kickoff()
432            
433            # Load and return results
434            try:
435                # Try loading from Apify KV store if available
436                if hasattr(Actor, 'main_kv_store'):
437                    results_dict = Actor.main_kv_store.get_value("search_results")
438                # Otherwise try to load from local file
439                elif os.path.exists("storage/key_value_stores/search_results.json"):
440                    with open("storage/key_value_stores/search_results.json", "r") as f:
441                        results_dict = json.load(f)
442                else:
443                    results_dict = None
444                
445                if results_dict:
446                    return SearchResults(**results_dict)
447            except Exception as e:
448                Actor.log.error(f"Error loading search results: {e}")
449        except Exception as e:
450            Actor.log.error(f"Error running crew: {e}")
451            
452        # If we got here, either there was an error or no results were found
453        # Create empty results
454        return SearchResults(
455            search_criteria=self.search_criteria,
456            results=[],
457            total_results=0,
458            new_results=0,
459            sources_searched=[]
460        )

1"""Agent classes for Listing Sleuth."""

1"""Scrapers for real estate platforms."""

1"""Apartments.com scraper."""
2
3import re
4import json
5import uuid
6from typing import Dict, Any, List, Optional
7from datetime import datetime
8from pydantic import HttpUrl
9
10from apify import Actor
11from apify_client import ApifyClient
12
13from .base import BaseScraper
14from ..models.property import PropertyListing, Address, SearchCriteria
15
16
17class ApartmentsScraper(BaseScraper):
18    """Apartments.com scraper."""
19    
20    @property
21    def actor_id(self) -> str:
22        """Get Apify actor ID for Apartments.com.
23        
24        Returns:
25            Actor ID
26        """
27        return "epctex/apartments-scraper"
28    
29    @property
30    def source_name(self) -> str:
31        """Get source name.
32        
33        Returns:
34            Source name
35        """
36        return "apartments"
37    
38    def prepare_input(self) -> Dict[str, Any]:
39        """Prepare input for the Apartments.com scraper.
40        
41        Returns:
42            Actor input
43        """
44        # Parse location into city and state
45        location_parts = self.search_criteria.location.split(",")
46        city = location_parts[0].strip().replace(" ", "-").lower()
47        state = ""
48        if len(location_parts) > 1:
49            state = location_parts[1].strip().lower()
50        
51        # Construct location for URL
52        if state:
53            location_url = f"{city}-{state}"
54        else:
55            location_url = city
56        
57        # Base URL
58        base_url = f"https://www.apartments.com/{location_url}"
59        
60        # Start building search parameters
61        search_params = {}
62        
63        # Bedrooms filter
64        if self.search_criteria.min_bedrooms > 0 and self.search_criteria.max_bedrooms:
65            if self.search_criteria.min_bedrooms == self.search_criteria.max_bedrooms:
66                search_params["br"] = str(self.search_criteria.min_bedrooms)
67            else:
68                search_params["br-min"] = str(self.search_criteria.min_bedrooms)
69                search_params["br-max"] = str(self.search_criteria.max_bedrooms)
70        elif self.search_criteria.min_bedrooms > 0:
71            search_params["br-min"] = str(self.search_criteria.min_bedrooms)
72        elif self.search_criteria.max_bedrooms:
73            search_params["br-max"] = str(self.search_criteria.max_bedrooms)
74        
75        # Price filter
76        if self.search_criteria.min_price > 0:
77            search_params["price-min"] = str(int(self.search_criteria.min_price))
78        if self.search_criteria.max_price:
79            search_params["price-max"] = str(int(self.search_criteria.max_price))
80        
81        # Property type - apartments.com primarily focuses on apartments, but can filter for types
82        if self.search_criteria.property_type != "any" and self.search_criteria.property_type != "apartment":
83            search_params["type"] = self.search_criteria.property_type
84        
85        return {
86            "startUrls": [{"url": base_url}],
87            "searchParams": search_params,
88            "maxItems": self.max_items,
89            "extendOutputFunction": """async ({ data, item, customData, Apify }) => {
90                return { ...item };
91            }""",
92            "proxy": {
93                "useApifyProxy": True,
94                "apifyProxyGroups": ["RESIDENTIAL"]
95            }
96        }
97    
98    def transform_item(self, item: Dict[str, Any]) -> PropertyListing:
99        """Transform an Apartments.com listing to a PropertyListing.
100        
101        Args:
102            item: Apartments.com listing
103            
104        Returns:
105            PropertyListing
106        """
107        # Parse price
108        price_str = item.get("rent", "0")
109        if isinstance(price_str, str):
110            # Extract digits from price string
111            price_match = re.search(r'(\d{1,3}(?:,\d{3})*(?:\.\d+)?)', price_str)
112            if price_match:
113                price_clean = price_match.group(1).replace(",", "")
114                price = float(price_clean)
115            else:
116                price = 0
117        else:
118            price = float(price_str) if price_str else 0
119        
120        # Parse address
121        property_address = item.get("propertyAddress", {})
122        address_line = property_address.get("addressLine", "")
123        neighborhood = property_address.get("neighborhood", "")
124        city = property_address.get("city", "")
125        state = property_address.get("state", "")
126        postal_code = property_address.get("postalCode", None)
127        
128        address = Address(
129            street=address_line,
130            city=city or neighborhood,  # Use neighborhood if city is missing
131            state=state,
132            zip_code=postal_code
133        )
134        
135        # Parse bedrooms
136        bedrooms = 0
137        beds = item.get("beds", 0)
138        if isinstance(beds, str):
139            bed_match = re.search(r'(\d+\.?\d*)', beds)
140            bedrooms = float(bed_match.group(1)) if bed_match else 0
141        else:
142            bedrooms = float(beds) if beds else 0
143        
144        # Parse bathrooms
145        bathrooms = None
146        baths = item.get("baths", None)
147        if baths:
148            if isinstance(baths, str):
149                bath_match = re.search(r'(\d+\.?\d*)', baths)
150                bathrooms = float(bath_match.group(1)) if bath_match else None
151            else:
152                bathrooms = float(baths)
153        
154        # Parse square feet
155        sqft = None
156        sqft_str = item.get("sqft", None)
157        if sqft_str:
158            if isinstance(sqft_str, str):
159                sqft_match = re.search(r'(\d+)', sqft_str.replace(',', ''))
160                sqft = int(sqft_match.group(1)) if sqft_match else None
161            else:
162                sqft = int(sqft_str)
163        
164        # Determine property type
165        property_type = "apartment"  # Default for apartments.com
166        if "condo" in item.get("title", "").lower() or "condo" in item.get("description", "").lower():
167            property_type = "condo"
168        elif "townhouse" in item.get("title", "").lower() or "townhouse" in item.get("description", "").lower():
169            property_type = "townhouse"
170        elif "house" in item.get("title", "").lower() and "townhouse" not in item.get("title", "").lower():
171            property_type = "house"
172        
173        # Get URL
174        url = item.get("url", "")
175        
176        # Get images
177        images = []
178        photos = item.get("photos", [])
179        if isinstance(photos, list):
180            for photo in photos:
181                if isinstance(photo, dict) and "url" in photo:
182                    images.append(photo["url"])
183                elif isinstance(photo, str) and photo.startswith("http"):
184                    images.append(photo)
185        
186        # Extract amenities
187        amenities = []
188        
189        # Add apartment amenities
190        apartment_amenities = item.get("apartmentAmenities", [])
191        if isinstance(apartment_amenities, list):
192            amenities.extend(apartment_amenities)
193        
194        # Add community amenities
195        community_amenities = item.get("communityAmenities", [])
196        if isinstance(community_amenities, list):
197            amenities.extend(community_amenities)
198        
199        # Also use the base extract_amenities method to catch any missed ones
200        amenities.extend(self.extract_amenities(item))
201        
202        # Remove duplicates while preserving order
203        amenities = list(dict.fromkeys(amenities))
204        
205        # Generate a unique ID
206        property_id = str(item.get("id", uuid.uuid4()))
207        
208        # Create features dictionary for additional data
209        additional_features = {}
210        for key, value in item.items():
211            if key not in [
212                "rent", "propertyAddress", "beds", "baths", "sqft", "url", "photos",
213                "apartmentAmenities", "communityAmenities", "id", "title", "description",
214            ]:
215                additional_features[key] = value
216        
217        # Parse listing date if available
218        listed_date = None
219        date_str = item.get("dateAvailable", item.get("datePosted", None))
220        if date_str and isinstance(date_str, str):
221            try:
222                # Try common date formats
223                for fmt in ["%Y-%m-%d", "%m/%d/%Y", "%b %d, %Y"]:
224                    try:
225                        listed_date = datetime.strptime(date_str, fmt)
226                        break
227                    except ValueError:
228                        continue
229            except Exception:
230                pass
231        
232        return PropertyListing(
233            id=property_id,
234            title=item.get("title", "Property Listing"),
235            description=item.get("description", None),
236            price=price,
237            address=address,
238            bedrooms=bedrooms,
239            bathrooms=bathrooms,
240            square_feet=sqft,
241            property_type=property_type,
242            url=url,
243            source="apartments",
244            amenities=amenities,
245            images=images,
246            listed_date=listed_date,
247            features=additional_features
248        )