Realestate Newsletter Agent Langgraph

1apify<3.0.0
2langchain-openai==0.3.6
3langgraph==0.2.73

1"""
2Main entry point for the Apify Actor.
3Orchestrates the autonomous real estate market research process.
4"""
5
6from __future__ import annotations
7
8import logging
9import os
10from apify import Actor
11from apify_client import ApifyClient
12from openai import AsyncOpenAI
13
14from .agents.search_agent import SearchAgent
15from .agents.extraction_agent import ExtractionAgent
16from .agents.analysis_agent import AnalysisAgent
17from .agents.newsletter_agent import NewsletterAgent
18from .models.schemas import AgentState
19
20# Configure logging
21logging.basicConfig(level=logging.INFO)
22logger = logging.getLogger(__name__)
23
24async def main() -> None:
25    """Main entry point for the Apify Actor."""
26    async with Actor:
27        logger.info("Starting real estate market analysis actor")
28        
29        # Get input
30        actor_input = await Actor.get_input() or {}
31        logger.info(f"Received input with keys: {', '.join(actor_input.keys())}")
32        
33        # Set API keys from input
34        openai_api_key = actor_input.get("openaiApiKey")
35        apify_api_key = actor_input.get("apifyApiKey")
36        
37        if not openai_api_key:
38            logger.error("OpenAI API key is required in input")
39            return
40        logger.info("OpenAI API key validated")
41            
42        if not apify_api_key:
43            logger.error("Apify API key is required in input")
44            return
45        logger.info("Apify API key validated")
46        
47        # Get location
48        location = actor_input.get("location")
49        if not location:
50            logger.error("Location is required")
51            return
52        logger.info(f"Processing location: {location}")
53            
54        # Set environment variables for API keys
55        os.environ["OPENAI_API_KEY"] = openai_api_key
56        os.environ["APIFY_TOKEN"] = apify_api_key
57        logger.info("Environment variables set")
58            
59        # Initialize OpenAI client
60        try:
61            openai_client = AsyncOpenAI(api_key=openai_api_key)
62            logger.info("OpenAI client initialized")
63        except Exception as e:
64            logger.error(f"Failed to initialize OpenAI client: {str(e)}")
65            return
66        
67        # Initialize Apify client
68        try:
69            apify_client = ApifyClient(token=apify_api_key)
70            logger.info("Apify client initialized")
71        except Exception as e:
72            logger.error(f"Failed to initialize Apify client: {str(e)}")
73            return
74            
75        # Initialize agents with shared OpenAI client
76        try:
77            newsletter_agent = NewsletterAgent(client=openai_client)
78            search_agent = SearchAgent(client=openai_client)
79            extraction_agent = ExtractionAgent(client=openai_client)
80            analysis_agent = AnalysisAgent(client=openai_client)
81            logger.info("All agents initialized successfully")
82        except Exception as e:
83            logger.error(f"Failed to initialize agents: {str(e)}")
84            return
85        
86        try:
87            # Execute workflow
88            logger.info("Starting source search...")
89            urls = await search_agent.find_sources(location)
90            if not urls:
91                logger.error("No valid URLs found for the location")
92                return
93            logger.info(f"Found {len(urls)} valid URLs")
94                
95            logger.info("Starting data extraction...")
96            market_data = await extraction_agent.extract_data(urls)
97            if not market_data:
98                logger.error("No market data could be extracted from URLs")
99                return
100            logger.info(f"Extracted market data from {len(market_data) if isinstance(market_data, list) else len(market_data.keys())} sources")
101                
102            logger.info("Starting market analysis...")
103            analysis = await analysis_agent.analyze_market(market_data)
104            if not analysis:
105                logger.error("Market analysis failed to produce results")
106                return
107            logger.info("Market analysis completed successfully")
108                
109            logger.info("Generating newsletter...")
110            newsletter = await newsletter_agent.generate_newsletter(location, market_data, analysis)
111            if not newsletter:
112                logger.error("Newsletter generation failed")
113                return
114            logger.info("Newsletter generated successfully")
115            
116            # Save output
117            logger.info("Saving results...")
118            await Actor.push_data({
119                "location": location,
120                "filtered_urls": urls,
121                "market_data": market_data,
122                "analysis": analysis,
123                "newsletter": newsletter
124            })
125            logger.info("Results saved successfully")
126            
127        except Exception as e:
128            logger.error(f"Actor failed with error: {str(e)}")
129            logger.exception("Detailed error traceback:")
130            raise
131
132if __name__ == "__main__":
133    Actor.main(main)

1"""This module defines Pydantic models for this project.
2
3These models are used mainly for the structured tool and LLM outputs.
4Resources:
5- https://docs.pydantic.dev/latest/concepts/models/
6"""
7
8from __future__ import annotations
9
10from pydantic import BaseModel
11
12
13class InstagramPost(BaseModel):
14    """Instagram Post Pydantic model.
15
16    Returned as a structured output by the `tool_scrape_instagram_profile_posts` tool.
17
18    url: The URL of the post.
19    likes: The number of likes on the post.
20    comments: The number of comments on the post.
21    timestamp: The timestamp when the post was published.
22    caption: The post caption.
23    alt: The post alt text.
24    """
25
26    url: str
27    likes: int
28    comments: int
29    timestamp: str
30    caption: str | None = None
31    alt: str | None = None
32
33
34class AgentStructuredOutput(BaseModel):
35    """Structured output for the ReAct agent.
36
37    Returned as a structured output by the ReAct agent.
38
39    total_likes: The total number of likes on the most popular posts.
40    total_comments: The total number of comments on the most popular posts.
41    most_popular_posts: A list of the most popular posts.
42    """
43
44    total_likes: int
45    total_comments: int
46    most_popular_posts: list[InstagramPost]

1"""This module defines the tools used by the agent.
2
3Feel free to modify or add new tools to suit your specific needs.
4
5To learn how to create a new tool, see:
6- https://python.langchain.com/docs/concepts/tools/
7- https://python.langchain.com/docs/how_to/#tools
8"""
9
10from __future__ import annotations
11
12from apify import Actor
13from langchain_core.tools import tool
14
15from src.models import InstagramPost
16
17
18@tool
19def tool_calculator_sum(numbers: list[int]) -> int:
20    """Tool to calculate the sum of a list of numbers.
21
22    Args:
23        numbers (list[int]): List of numbers to sum.
24
25    Returns:
26        int: Sum of the numbers.
27    """
28    return sum(numbers)
29
30
31@tool
32async def tool_scrape_instagram_profile_posts(handle: str, max_posts: int = 30) -> list[InstagramPost]:
33    """Tool to scrape Instagram profile posts.
34
35    Args:
36        handle (str): Instagram handle of the profile to scrape (without the '@' symbol).
37        max_posts (int, optional): Maximum number of posts to scrape. Defaults to 30.
38
39    Returns:
40        list[InstagramPost]: List of Instagram posts scraped from the profile.
41
42    Raises:
43        RuntimeError: If the Actor fails to start.
44    """
45    run_input = {
46        'directUrls': [f'https://www.instagram.com/{handle}/'],
47        'resultsLimit': max_posts,
48        'resultsType': 'posts',
49        'searchLimit': 1,
50    }
51    if not (run := await Actor.apify_client.actor('apify/instagram-scraper').call(run_input=run_input)):
52        msg = 'Failed to start the Actor apify/instagram-scraper'
53        raise RuntimeError(msg)
54
55    dataset_id = run['defaultDatasetId']
56    dataset_items: list[dict] = (await Actor.apify_client.dataset(dataset_id).list_items()).items
57    posts: list[InstagramPost] = []
58    for item in dataset_items:
59        url: str | None = item.get('url')
60        caption: str | None = item.get('caption')
61        alt: str | None = item.get('alt')
62        likes: int | None = item.get('likesCount')
63        comments: int | None = item.get('commentsCount')
64        timestamp: str | None = item.get('timestamp')
65
66        # only include posts with all required fields
67        if not url or not likes or not comments or not timestamp:
68            Actor.log.warning('Skipping post with missing fields: %s', item)
69            continue
70
71        posts.append(
72            InstagramPost(
73                url=url,
74                likes=likes,
75                comments=comments,
76                timestamp=timestamp,
77                caption=caption,
78                alt=alt,
79            )
80        )
81
82    return posts

1from apify import Actor
2from langchain_core.messages import ToolMessage
3
4
5def log_state(state: dict) -> None:
6    """Logs the state of the graph.
7
8    Uses the `Actor.log.debug` method to log the state of the graph.
9
10    Args:
11        state (dict): The state of the graph.
12    """
13    message = state['messages'][-1]
14    # Traverse all tool messages and print them
15    # if multiple tools are called in parallel
16    if isinstance(message, ToolMessage):
17        # Until the analyst message with tool_calls
18        for _message in state['messages'][::-1]:
19            if hasattr(_message, 'tool_calls'):
20                break
21            Actor.log.debug('-------- Tool Result --------')
22            Actor.log.debug('Tool: %s', _message.name)
23            Actor.log.debug('Result: %s', _message.content)
24
25    Actor.log.debug('-------- Message --------')
26    Actor.log.debug('Message: %s', message)
27
28    # Print all tool calls
29    if hasattr(message, 'tool_calls'):
30        for tool_call in getattr(message, 'tool_calls', []):
31            Actor.log.debug('-------- Tool Call --------')
32            Actor.log.debug('Tool: %s', tool_call['name'])
33            Actor.log.debug('Args: %s', tool_call['args'])

1"""Real Estate Newsletter Agent package."""

1import asyncio
2
3from .main import main
4
5# Execute the Actor entry point.
6asyncio.run(main())

1"""
2Analysis Agent Module - Handles market data analysis and metrics calculation
3"""
4
5import logging
6import re
7from typing import Dict, List
8from decimal import Decimal
9from apify import Actor
10from openai import AsyncOpenAI
11
12from .extraction_agent import MarketData
13
14logger = logging.getLogger(__name__)
15
16class AnalysisAgent:
17    def __init__(self, client: AsyncOpenAI):
18        """Initialize the AnalysisAgent with an OpenAI client.
19        
20        Args:
21            client (AsyncOpenAI): The OpenAI client instance to use for API calls
22        """
23        self.client = client
24        self.high_cost_markets = ["new york", "san francisco", "los angeles", "seattle", "boston", "miami"]
25
26    async def analyze_market(self, market_data: List[MarketData], location: str = "") -> Dict:
27        """Analyze the extracted market data and extract key metrics"""
28        metrics = {
29            "zillow": {},
30            "redfin": {},
31            "realtor": {},
32            "rocket": {}
33        }
34        
35        is_high_cost = any(market.lower() in location.lower() for market in self.high_cost_markets)
36        
37        min_price = 10000
38        max_price = 10000000 if is_high_cost else 2000000
39        min_valid_price = 100000
40        max_valid_price = 5000000 if is_high_cost else 1000000
41        
42        try:
43            for data in market_data:
44                source = data.source
45                metrics[source] = {
46                    "median_price": data.median_price,
47                    "price_change": data.price_change,
48                    "inventory": data.inventory,
49                    "days_on_market": data.days_on_market,
50                    "source": source
51                }
52                
53                # Validate metrics
54                if min_price <= data.median_price <= max_price:
55                    await Actor.charge('market-analyzed')
56                    logger.info(f"Successfully analyzed data from {source}")
57                
58        except Exception as e:
59            logger.error(f"Error analyzing market data: {str(e)}")
60        
61        metrics["_meta"] = {
62            "min_valid_price": min_valid_price,
63            "max_valid_price": max_valid_price
64        }
65        
66        return metrics
67
68    async def analyze_market_data(self, market_data: Dict, location: str = "") -> Dict:
69        """Analyze the extracted market data and extract key metrics"""
70        metrics = {
71            "zillow": {},
72            "redfin": {},
73            "realtor": {},
74            "rapid": {}
75        }
76        
77        await Actor.charge('market-analyzed')
78        
79        is_high_cost = any(market.lower() in location.lower() for market in self.high_cost_markets)
80        
81        min_price = 10000
82        max_price = 10000000 if is_high_cost else 2000000
83        min_valid_price = 100000
84        max_valid_price = 5000000 if is_high_cost else 1000000
85        
86        try:
87            for source, data in market_data.items():
88                text = data.get("text", "").lower()
89                
90                if not text:
91                    continue
92                    
93                metrics_found = False
94                
95                # Extract and validate metrics
96                metrics[source].update(self._extract_price_metrics(text, min_price, max_price))
97                metrics[source].update(self._extract_price_change(text))
98                metrics[source].update(self._extract_market_metrics(text))
99                
100                if metrics[source]:
101                    metrics_found = True
102                    await Actor.charge('market-analyzed')
103                
104                metrics[source]["source_date"] = data.get("metadata", {}).get("loadedTime", "")
105                
106        except Exception as e:
107            logger.error(f"Error analyzing market data: {str(e)}")
108        
109        metrics["_meta"] = {
110            "min_valid_price": min_valid_price,
111            "max_valid_price": max_valid_price,
112            "is_high_cost": is_high_cost
113        }
114        
115        source_urls = {
116            source: data.get("metadata", {}).get("canonicalUrl") or data.get("metadata", {}).get("loadedUrl", "")
117            for source, data in market_data.items()
118        }
119        
120        return {"metrics": metrics, "source_urls": source_urls}
121
122    def _extract_price_metrics(self, text: str, min_price: int, max_price: int) -> Dict:
123        """Extract and validate price metrics"""
124        metrics = {}
125        price_patterns = [
126            r"median (?:sale )?price.*?\$([0-9,.]+)[MK]?",
127            r"average.*?home value.*?\$([0-9,.]+)[MK]?",
128            r"median.*?home value.*?\$([0-9,.]+)[MK]?",
129            r"\$([0-9,.]+)[MK]?(?=.*median)",
130        ]
131        
132        for pattern in price_patterns:
133            price_match = re.search(pattern, text)
134            if price_match:
135                try:
136                    price = float(price_match.group(1).replace(",", ""))
137                    if min_price <= price <= max_price:
138                        if "m" in text[price_match.end():price_match.end()+2].lower():
139                            metrics["median_price"] = price * 1000000
140                        elif "k" in text[price_match.end():price_match.end()+2].lower():
141                            metrics["median_price"] = price * 1000
142                        else:
143                            metrics["median_price"] = price
144                        break
145                except ValueError:
146                    continue
147        
148        return metrics
149
150    def _extract_price_change(self, text: str) -> Dict:
151        """Extract and validate price change percentage"""
152        metrics = {}
153        change_patterns = [
154            r"(up|down)\s+([0-9.]+)%\s+(?:since|compared to|over|in the) last year",
155            r"([0-9.]+)%\s+(increase|decrease)\s+(?:since|compared to|over|in the) last year",
156            r"([+-]?[0-9.]+)%\s+1-yr"
157        ]
158        
159        for pattern in change_patterns:
160            change_match = re.search(pattern, text)
161            if change_match:
162                try:
163                    if len(change_match.groups()) == 2:
164                        change = float(change_match.group(2))
165                        if "down" in change_match.group(1).lower() or "decrease" in change_match.group(2).lower():
166                            change = -change
167                    else:
168                        change = float(change_match.group(1))
169                    if abs(change) <= 50:
170                        metrics["price_change"] = change
171                        break
172                except ValueError:
173                    continue
174        
175        return metrics
176
177    def _extract_market_metrics(self, text: str) -> Dict:
178        """Extract and validate market metrics (days on market, price per sqft, inventory)"""
179        metrics = {}
180        
181        # Days on market
182        dom_patterns = [
183            r"(?:sell|sold) (?:in|after) (?:around )?([0-9]+) days",
184            r"(?:average|median) (?:of )?([0-9]+) days on (?:the )?market",
185            r"([0-9]+) days on (?:the )?market",
186            r"pending in (?:around )?([0-9]+) days"
187        ]
188        
189        for pattern in dom_patterns:
190            dom_match = re.search(pattern, text)
191            if dom_match:
192                try:
193                    days = int(dom_match.group(1))
194                    if 0 <= days <= 365:
195                        metrics["days_on_market"] = days
196                        break
197                except ValueError:
198                    continue
199        
200        # Price per sqft
201        sqft_patterns = [
202            r"\$([0-9,.]+) per square (?:foot|feet|ft)",
203            r"price per (?:square )?(?:foot|feet|ft).*?\$([0-9,.]+)"
204        ]
205        
206        for pattern in sqft_patterns:
207            sqft_match = re.search(pattern, text)
208            if sqft_match:
209                try:
210                    price_sqft = float(sqft_match.group(1).replace(",", ""))
211                    if 50 <= price_sqft <= 2000:
212                        metrics["price_per_sqft"] = price_sqft
213                        break
214                except ValueError:
215                    continue
216        
217        # Inventory
218        inv_patterns = [
219            r"([0-9,]+) homes? (?:for sale|available|active)",
220            r"inventory of ([0-9,]+) homes",
221            r"([0-9,]+) properties? (?:for sale|available|active)"
222        ]
223        
224        for pattern in inv_patterns:
225            inv_match = re.search(pattern, text)
226            if inv_match:
227                try:
228                    inventory = int(inv_match.group(1).replace(",", ""))
229                    if 0 <= inventory <= 10000:
230                        metrics["inventory"] = inventory
231                        break
232                except ValueError:
233                    continue
234        
235        return metrics

1"""
2Extraction Agent Module - Handles data extraction from validated sources
3"""
4
5import logging
6from typing import Dict, List
7from dataclasses import dataclass
8from decimal import Decimal
9from apify import Actor
10from apify_client import ApifyClient
11from openai import AsyncOpenAI
12import os
13import re
14
15logger = logging.getLogger(__name__)
16
17@dataclass
18class MarketData:
19    median_price: float
20    price_change: float
21    inventory: int
22    days_on_market: int
23    source: str
24
25class ExtractionAgent:
26    def __init__(self, client: AsyncOpenAI):
27        """Initialize the ExtractionAgent with OpenAI client"""
28        self.client = client
29        self.apify_client = ApifyClient(token=os.environ["APIFY_TOKEN"])
30        
31    def _extract_price_metrics(self, text: str) -> Dict:
32        """Extract price metrics from text"""
33        metrics = {}
34        price_patterns = [
35            r"(?:median|average).*?(?:home value|sale price|sold price)[^\$]*\$([0-9,.]+)([KM]?)",
36            r"\$([0-9,.]+)([KM]?)(?=.*median)",
37            r"typical home value.*?\$([0-9,.]+)([KM]?)",
38            r"median list price.*?\$([0-9,.]+)([KM]?)"
39        ]
40        
41        for pattern in price_patterns:
42            price_match = re.search(pattern, text.lower())
43            if price_match:
44                try:
45                    price_str = price_match.group(1).replace(",", "")
46                    price = float(price_str)
47                    suffix = price_match.group(2).upper() if len(price_match.groups()) > 1 else ""
48                    
49                    # Handle price multipliers
50                    if suffix == "M":
51                        price *= 1000000
52                    elif suffix == "K":
53                        price *= 1000
54                    
55                    # Validate price range (100k to 2M for normal markets)
56                    if 100000 <= price <= 2000000:
57                        metrics["median_price"] = price
58                        logger.info(f"Extracted price: ${price:,.2f}")
59                        break
60                    else:
61                        logger.warning(f"Price {price} outside valid range")
62                except ValueError:
63                    continue
64        
65        return metrics
66
67    def _extract_price_change(self, text: str) -> Dict:
68        """Extract price change percentage from text"""
69        metrics = {}
70        change_patterns = [
71            r"(?:down|decreased)\s+([0-9.]+)%",
72            r"([0-9.]+)%\s+(?:decrease)",
73            r"-([0-9.]+)%",
74            r"(?:up|increased)\s+([0-9.]+)%",
75            r"([0-9.]+)%\s+(?:increase)",
76            r"\+([0-9.]+)%"
77        ]
78        
79        for pattern in change_patterns:
80            change_match = re.search(pattern, text.lower())
81            if change_match:
82                try:
83                    change = float(change_match.group(1))
84                    # Make negative if pattern indicates decrease
85                    if any(word in pattern.lower() for word in ["down", "decrease", "-"]):
86                        change = -change
87                    
88                    # Validate change range (-20% to +20% is reasonable)
89                    if abs(change) <= 20:
90                        metrics["price_change"] = change
91                        logger.info(f"Extracted price change: {change}%")
92                        break
93                    else:
94                        logger.warning(f"Price change {change}% outside valid range")
95                except ValueError:
96                    continue
97        
98        return metrics
99
100    def _extract_market_metrics(self, text: str) -> Dict:
101        """Extract market metrics from text"""
102        metrics = {}
103        
104        # Days on market
105        dom_patterns = [
106            r"(?:median |average )?(?:days on market|dom)[:\s]+([0-9]+)",
107            r"(?:sell|sold|pending) (?:in|after) (?:around )?([0-9]+) days",
108            r"(?:average|median) (?:of )?([0-9]+) days on (?:the )?market",
109            r"([0-9]+) days? on (?:the )?market",
110            r"average listing age of ([0-9]+) days"
111        ]
112        
113        for pattern in dom_patterns:
114            dom_match = re.search(pattern, text.lower())
115            if dom_match:
116                try:
117                    days = int(dom_match.group(1))
118                    if 0 <= days <= 180:  # Most markets are under 180 days
119                        metrics["days_on_market"] = days
120                        logger.info(f"Extracted days on market: {days}")
121                        break
122                    else:
123                        logger.warning(f"Days on market {days} outside valid range")
124                except ValueError:
125                    continue
126        
127        # Inventory
128        inv_patterns = [
129            r"([0-9,]+)\s+(?:homes?|properties?|listings?|houses?)?\s+(?:for sale|available|active)",
130            r"inventory of ([0-9,]+)",
131            r"([0-9,]+) total listings",
132            r"([0-9,]+) properties found",
133            r"([0-9,]+) homes sold"
134        ]
135        
136        for pattern in inv_patterns:
137            inv_match = re.search(pattern, text.lower())
138            if inv_match:
139                try:
140                    inventory = int(inv_match.group(1).replace(",", ""))
141                    if 0 <= inventory <= 50000:  # Increased max for larger markets
142                        metrics["inventory"] = inventory
143                        logger.info(f"Extracted inventory: {inventory}")
144                        break
145                    else:
146                        logger.warning(f"Inventory {inventory} outside valid range")
147                except ValueError:
148                    continue
149        
150        return metrics
151
152    async def extract_data(self, urls: Dict[str, str]) -> List[MarketData]:
153        """Extract market data from the filtered URLs"""
154        market_data = []
155        
156        try:
157            # Convert URLs dict to list of URLData for crawler
158            url_list = [{"url": url, "method": "GET"} for url in urls.values()]
159            
160            crawler_input = {
161                "startUrls": url_list,
162                "crawlerType": "playwright",  # Changed to playwright for better JS support
163                "maxCrawlPages": len(urls),
164                "maxCrawlDepth": 0,
165                "saveMarkdown": True,
166                "maxRequestRetries": 3,
167                "maxConcurrency": 1,
168                "proxyConfiguration": {
169                    "useApifyProxy": True,
170                    "apifyProxyGroups": ["RESIDENTIAL"]
171                },
172                "removeElementsCssSelector": "nav, footer, script, style, noscript, svg, img[src^='data:']",
173                "htmlTransformer": "readableText",
174                "waitUntil": "networkidle",  # Wait for network to be idle
175                "browserPoolOptions": {
176                    "maxOpenPagesPerBrowser": 1,
177                    "retireInstanceAfterRequestCount": 5
178                }
179            }
180            
181            logger.info("Starting Website Content Crawler...")
182            run = self.apify_client.actor("apify/website-content-crawler").call(
183                run_input=crawler_input,
184                memory_mbytes=4096
185            )
186            
187            dataset_id = run["defaultDatasetId"]
188            items = self.apify_client.dataset(dataset_id).list_items().items
189            
190            for source, url in urls.items():
191                logger.info(f"Processing {source} URL: {url}")
192                extracted = False
193                
194                for item in items:
195                    if item.get("url") == url:
196                        text_content = item.get("text", "")
197                        
198                        if not text_content:
199                            logger.warning(f"No content extracted from {source}")
200                            continue
201                        
202                        # Log first 500 chars of content for debugging
203                        logger.debug(f"{source} content preview: {text_content[:500]}")
204                            
205                        # Extract metrics using parsing logic
206                        price_metrics = self._extract_price_metrics(text_content)
207                        price_change = self._extract_price_change(text_content)
208                        market_metrics = self._extract_market_metrics(text_content)
209                        
210                        if price_metrics or price_change or market_metrics:
211                            await Actor.charge('data-extracted')
212                            logger.info(f"Successfully extracted data from {source}")
213                            
214                            market_data.append(MarketData(
215                                median_price=price_metrics.get("median_price", 0),
216                                price_change=price_change.get("price_change", 0),
217                                inventory=market_metrics.get("inventory", 0),
218                                days_on_market=market_metrics.get("days_on_market", 0),
219                                source=source
220                            ))
221                            extracted = True
222                            break
223                
224                if not extracted:
225                    logger.error(f"Failed to extract valid data from {source}")
226                        
227        except Exception as e:
228            logger.error(f"Error extracting market data: {str(e)}")
229            
230        return market_data

1"""
2Newsletter Agent Module - Handles report generation using OpenAI
3"""
4
5import logging
6from datetime import datetime
7from typing import Dict, Any
8from openai import AsyncOpenAI
9from apify import Actor
10
11logger = logging.getLogger(__name__)
12
13class NewsletterAgent:
14    def __init__(self, client: AsyncOpenAI):
15        """Initialize the NewsletterAgent with an OpenAI client.
16        
17        Args:
18            client (AsyncOpenAI): The OpenAI client instance to use for API calls
19        """
20        self.client = client
21
22    async def generate_newsletter(self, location: str, market_data: Dict, analysis: Dict) -> str:
23        """Generate a real estate market newsletter using OpenAI"""
24        try:
25            current_date = datetime.now().strftime("%B %Y")
26            
27            metrics = analysis.get("metrics", {})
28            source_urls = analysis.get("source_urls", {})
29            meta = metrics.get("_meta", {})
30            min_valid_price = meta.get("min_valid_price", 100000)
31            max_valid_price = meta.get("max_valid_price", 1000000)
32            
33            formatted_data = self._format_source_data(metrics)
34            formatted_urls = self._format_source_urls(source_urls)
35            avg_metrics = self._calculate_averages(metrics, min_valid_price, max_valid_price)
36            
37            system_content = self._get_system_prompt()
38            user_content = self._get_user_prompt(location, current_date, formatted_data, avg_metrics, formatted_urls)
39            
40            response = await self.client.chat.completions.create(
41                model="gpt-4o-mini",
42                messages=[
43                    {"role": "system", "content": system_content},
44                    {"role": "user", "content": user_content}
45                ],
46                temperature=0.7,
47                max_tokens=2000
48            )
49            
50            newsletter = response.choices[0].message.content
51            
52            # Charge for newsletter generation
53            await Actor.charge('newsletter-generated')
54            
55            return newsletter
56            
57        except Exception as e:
58            logger.error(f"Error generating newsletter: {str(e)}")
59            return f"Error generating newsletter: {str(e)}"
60
61    def _format_price(self, price):
62        """Format price with proper formatting"""
63        if price and isinstance(price, (int, float)):
64            return f"${price:,.0f}"
65        return "N/A"
66
67    def _format_percent(self, value):
68        """Format percentage with proper formatting"""
69        if value is not None:
70            return f"{value:+.1f}%" if value >= 0 else f"{value:.1f}%"
71        return "N/A"
72
73    def _format_source_data(self, metrics: Dict) -> str:
74        """Format market data from each source"""
75        formatted_data = ""
76        for source in ["zillow", "redfin", "realtor", "rapid"]:
77            source_data = metrics.get(source, {})
78            if source_data:
79                formatted_data += f"""
80{source.capitalize()}:
81- Median Price: {self._format_price(source_data.get('median_price'))}
82- Price Change: {self._format_percent(source_data.get('price_change'))}
83- Days on Market: {source_data.get('days_on_market', 'N/A')}
84- Price Per SqFt: {self._format_price(source_data.get('price_per_sqft'))}
85- Inventory: {source_data.get('inventory', 'N/A')}
86"""
87        return formatted_data
88
89    def _format_source_urls(self, source_urls: Dict) -> str:
90        """Format source URLs"""
91        return "\n".join(f"- {source.capitalize()}: {url}" for source, url in source_urls.items() if url)
92
93    def _calculate_averages(self, metrics: Dict, min_valid_price: int, max_valid_price: int) -> Dict:
94        """Calculate average metrics across sources"""
95        def calculate_average(metric_name):
96            values = []
97            for source, source_data in metrics.items():
98                if source == "_meta":
99                    continue
100                value = source_data.get(metric_name)
101                if value and isinstance(value, (int, float)):
102                    if metric_name == "median_price" and (value < min_valid_price or value > max_valid_price):
103                        continue
104                    if metric_name == "price_change" and abs(value) > 20:
105                        continue
106                    values.append(value)
107            return sum(values) / len(values) if values else None
108        
109        return {
110            "avg_price": calculate_average("median_price"),
111            "avg_price_change": calculate_average("price_change"),
112            "avg_dom": calculate_average("days_on_market")
113        }
114
115    def _get_system_prompt(self) -> str:
116        """Get system prompt for newsletter generation"""
117        return """You are an expert real estate newsletter writer. Create a professional, polished, and well-structured newsletter using the provided market data.
118
119Format the newsletter in Markdown with the following requirements:
1201. Begin with a main heading (#) that includes the location name and "Real Estate Market Update - [Month Year]"
1212. Add a "Last Updated" line right after the title
1223. Use subheadings (##) for different sections with proper spacing
1234. Include a well-formatted Markdown table comparing data sources
1245. Use emoji icons to highlight key points
1256. Format all prices with dollar signs and commas
1267. Include percentages with % symbol and +/- signs
1278. Use proper spacing between sections
1289. Use **bold** for critical information and *italic* for secondary emphasis
12910. If data is limited or inconsistent, acknowledge this and focus on reliable metrics
130
131Include these sections:
132- Executive Summary (3-4 sentences)
133- Market Overview (with validated average metrics)
134- Market Data Comparison (table)
135- Price Analysis
136- Market Activity
137- Market Forecast
138- Recommendations for Buyers and Sellers
139- Additional Resources"""
140
141    def _get_user_prompt(self, location: str, current_date: str, formatted_data: str, avg_metrics: Dict, formatted_urls: str) -> str:
142        """Get user prompt for newsletter generation"""
143        return f"""Create a real estate market newsletter for {location} for {current_date}.
144
145MARKET DATA:
146{formatted_data}
147
148AVERAGE METRICS (excluding outliers):
149- Average Price: {self._format_price(avg_metrics['avg_price'])}
150- Average Price Change: {self._format_percent(avg_metrics['avg_price_change'])}
151- Average Days on Market: {int(avg_metrics['avg_dom']) if avg_metrics['avg_dom'] else 'N/A'}
152
153SOURCE URLS:
154{formatted_urls}
155
156Please generate a comprehensive newsletter following the format in the system instructions. Make sure to:
1571. Focus on the most reliable data points
1582. Acknowledge any data inconsistencies
1593. Provide specific insights based on validated metrics
1604. Include actionable recommendations
1615. Format all numbers properly
1626. Use appropriate spacing and visual elements
1637. Bold important findings
1648. If certain metrics are missing or unreliable, explain the limitations"""