Pricing
$29.00 / 1,000 results
Find competitors near any business location using Google Maps. Enter addresses, Place IDs, or coordinates to discover nearby businesses with ratings, reviews, and rankings. Perfect for market research, site selection, and competitive analysis. Supports bulk locations and custom search terms
0.0 (0)
Pricing
$29.00 / 1,000 results
0
12
12
Last modified
2 months ago
.git.mise.toml.nvim.luastorage
# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore
# Byte-compiled / optimized / DLL files__pycache__/*.py[cod]*$py.class
# C extensions*.so
# Distribution / packaging.Pythonbuild/develop-eggs/dist/downloads/eggs/.eggs/lib/lib64/parts/sdist/var/wheels/share/python-wheels/*.egg-info/.installed.cfg*.eggMANIFEST
# PyInstaller# Usually these files are written by a python script from a template# before PyInstaller builds the exe, so as to inject date/other infos into it.*.manifest*.spec
# Installer logspip-log.txtpip-delete-this-directory.txt
# Unit test / coverage reportshtmlcov/.tox/.nox/.coverage.coverage.*.cachenosetests.xmlcoverage.xml*.cover*.py,cover.hypothesis/.pytest_cache/cover/
# Translations*.mo*.pot
# Django stuff:*.loglocal_settings.pydb.sqlite3db.sqlite3-journal
# Flask stuff:instance/.webassets-cache
# Scrapy stuff:.scrapy
# Sphinx documentationdocs/_build/
# PyBuilder.pybuilder/target/
# Jupyter Notebook.ipynb_checkpoints
# IPythonprofile_default/ipython_config.py
# pyenv# For a library or package, you might want to ignore these files since the code is# intended to run in multiple environments; otherwise, check them in:.python-version
# pdm# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.#pdm.lock# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it# in version control.# https://pdm.fming.dev/latest/usage/project/#working-with-version-control.pdm.toml.pdm-python.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm__pypackages__/
# Celery stuffcelerybeat-schedulecelerybeat.pid
# SageMath parsed files*.sage.py
# Environments.env.venvenv/venv/ENV/env.bak/venv.bak/
# Spyder project settings.spyderproject.spyproject
# Rope project settings.ropeproject
# mkdocs documentation/site
# mypy.mypy_cache/.dmypy.jsondmypy.json
# Pyre type checker.pyre/
# pytype static type analyzer.pytype/
# Cython debug symbolscython_debug/
# PyCharm# JetBrains specific template is maintained in a separate JetBrains.gitignore that can# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore# and can be added to the global gitignore or merged into this file. For a more nuclear# option (not recommended) you can uncomment the following to ignore the entire idea folder..idea/
# Visual Studio Code# Ignores the folder created by VS Code when changing workspace settings, doing debugger# configuration, etc. Can be commented out to share Workspace Settings within a team.vscode
# Zed editor# Ignores the folder created when setting Project Settings in the Zed editor. Can be commented out# to share Project Settings within a team.zed.mise.toml.nvim.luastorage
# The rest is copied from https://github.com/github/gitignore/blob/main/Python.gitignore
# Byte-compiled / optimized / DLL files__pycache__/*.py[cod]*$py.class
# C extensions*.so
# Distribution / packaging.Pythonbuild/develop-eggs/dist/downloads/eggs/.eggs/lib/lib64/parts/sdist/var/wheels/share/python-wheels/*.egg-info/.installed.cfg*.eggMANIFEST
# PyInstaller# Usually these files are written by a python script from a template# before PyInstaller builds the exe, so as to inject date/other infos into it.*.manifest*.spec
# Installer logspip-log.txtpip-delete-this-directory.txt
# Unit test / coverage reportshtmlcov/.tox/.nox/.coverage.coverage.*.cachenosetests.xmlcoverage.xml*.cover*.py,cover.hypothesis/.pytest_cache/cover/
# Translations*.mo*.pot
# Django stuff:*.loglocal_settings.pydb.sqlite3db.sqlite3-journal
# Flask stuff:instance/.webassets-cache
# Scrapy stuff:.scrapy
# Sphinx documentationdocs/_build/
# PyBuilder.pybuilder/target/
# Jupyter Notebook.ipynb_checkpoints
# IPythonprofile_default/ipython_config.py
# pyenv# For a library or package, you might want to ignore these files since the code is# intended to run in multiple environments; otherwise, check them in:.python-version
# pdm# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.#pdm.lock# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it# in version control.# https://pdm.fming.dev/latest/usage/project/#working-with-version-control.pdm.toml.pdm-python.pdm-build/
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm__pypackages__/
# Celery stuffcelerybeat-schedulecelerybeat.pid
# SageMath parsed files*.sage.py
# Environments.env.venvenv/venv/ENV/env.bak/venv.bak/
# Spyder project settings.spyderproject.spyproject
# Rope project settings.ropeproject
# mkdocs documentation/site
# mypy.mypy_cache/.dmypy.jsondmypy.json
# Pyre type checker.pyre/
# pytype static type analyzer.pytype/
# Cython debug symbolscython_debug/
# PyCharm# JetBrains specific template is maintained in a separate JetBrains.gitignore that can# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore# and can be added to the global gitignore or merged into this file. For a more nuclear# option (not recommended) you can uncomment the following to ignore the entire idea folder..idea/
# Visual Studio Code# Ignores the folder created by VS Code when changing workspace settings, doing debugger# configuration, etc. Can be commented out to share Workspace Settings within a team.vscode
# Zed editor# Ignores the folder created when setting Project Settings in the Zed editor. Can be commented out# to share Project Settings within a team.zed
# Added by Apify CLInode_modules# First, specify the base Docker image.# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.# You can also use any other image from Docker Hub.FROM apify/actor-python-playwright:3.13
# Second, copy just requirements.txt into the Actor image,# since it should be the only file that affects the dependency install in the next step,# in order to speed up the buildCOPY requirements.txt ./
# Install the packages specified in requirements.txt,# Print the installed Python version, pip version# and all installed packages with their versions for debuggingRUN echo "Python version:" \ && python --version \ && echo "Pip version:" \ && pip --version \ && echo "Installing dependencies:" \ && pip install -r requirements.txt \ && echo "All installed Python packages:" \ && pip freeze
# Install Playwright and its dependenciesRUN playwright install-deps && \ playwright install
# Next, copy the remaining files and directories with the source code.# Since we do this after installing the dependencies, quick build will be really fast# for most source file changes.COPY . ./
# Use compileall to ensure the runnability of the Actor Python code.RUN python3 -m compileall -q src/
# Specify how to launch the source code of your Actor.# By default, the "python3 -m src" command is runCMD ["python3", "-m", "src"]1apify-client2pandas3requests4playwright1import asyncio2import json3from src.main import main4from apify import Actor5
6# Create test input data7test_input = {8 "locations": [9 {10 "key": "6900 East Green Lake Way N suite i, Seattle, WA 98115",11 "value": "Blue Haven"12 }13 ],14 "searchQuery": "medical spa",15 "maxCompetitors": 3016}17
18# Write test input to the expected location19import os20os.makedirs("storage/key_value_stores/default", exist_ok=True)21with open("storage/key_value_stores/default/INPUT.json", "w") as f:22 json.dump(test_input, f, indent=2)23
24print("Test input created:")25print(json.dumps(test_input, indent=2))26print("\nRunning scraper locally...")27
28# Run the main function29asyncio.run(main())1#!/usr/bin/env python32"""3Test the Google Maps scraper locally4"""5import asyncio6import json7import sys8sys.path.insert(0, './src')9
10from main import main11
12# Create a mock Actor class for local testing13class MockActor:14 def __init__(self):15 self.input_data = {16 'inputMethod': 'structured',17 'searchQuery': 'restaurant', # Required field18 'city': 'Washington',19 'state': 'DC',20 'zipCode': '',21 'businessName': '',22 'maxCompetitors': 5, # Small number for testing23 'verbose': True24 }25 self.pushed_data = []26 27 async def __aenter__(self):28 return self29 30 async def __axit__(self, exc_type, exc_val, exc_tb):31 pass32 33 @classmethod34 async def get_input(cls):35 return cls.input_data36 37 @classmethod38 def log(cls):39 class Logger:40 @staticmethod41 def info(msg):42 print(f"[INFO] {msg}")43 44 @staticmethod45 def error(msg):46 print(f"[ERROR] {msg}")47 48 @staticmethod49 def warning(msg):50 print(f"[WARNING] {msg}")51 52 @staticmethod53 def exception(msg):54 print(f"[EXCEPTION] {msg}")55 56 return Logger()57 58 @classmethod59 async def push_data(cls, data):60 """Debug: Print the data that would be pushed"""61 print("\n=== DATA TO BE PUSHED ===")62 print(json.dumps(data, indent=2))63 print("=========================\n")64 65 # Check for potential issues66 for key, value in data.items():67 if value is None:68 print(f"⚠️ WARNING: {key} is None")69 elif isinstance(value, float) and str(value) == 'nan':70 print(f"⚠️ WARNING: {key} is NaN")71 elif isinstance(value, str) and value == '':72 print(f"ℹ️ INFO: {key} is empty string")73 74 cls.pushed_data.append(data)75
76# Replace the Actor import in main.py77import main78main.Actor = MockActor79MockActor.input_data = MockActor().input_data80MockActor.pushed_data = []81
82if __name__ == "__main__":83 print("Testing Google Maps Scraper locally...")84 print("This will scrape restaurants in Washington, DC")85 print("-" * 50)86 87 try:88 asyncio.run(main.main())89 print(f"\nTotal results pushed: {len(MockActor.pushed_data)}")90 except Exception as e:91 print(f"\nError: {e}")92 import traceback93 traceback.print_exc(){ "permissions": { "allow": [ "Bash(grep:*)", "Bash(echo:*)", "Bash(python:*)", "Bash(cp:*)", "Bash(cd:*)", "Bash(cd:*)", "Bash(cd:*)", "Bash(cd:*)", "Bash(cd:*)", "Bash(ls:*)" ], "deny": [] }}# Google Maps Competitor Analyzer
Find and analyze competitors for any business location using Google Maps data. Simply provide an address, Google Place ID, or coordinates, and this actor will discover nearby competing businesses with detailed information.
## 🎯 Use Cases
- **Market Research**: Analyze competition before opening a new location- **Site Selection**: Compare competitor density across multiple potential locations - **Lead Generation**: Find businesses in specific niches for B2B outreach- **Competitive Analysis**: Track ratings and reviews of competitors- **Location Scouting**: Identify market gaps and opportunities
## 📊 What You Get
For each competitor found:- Business name and Google rank- Average rating and review count- Business category- Full address- Google Place ID and CID- Identification if it's your business
## 🔧 Input
### Search QueryEnter the type of businesses to search for:- `medical spa`- `mexican restaurant`- `coffee shop`- `dental office`- etc.
### LocationsEnter locations in any of these formats:
**Full Address:**123 Main St, Beverly Hills, CA 90210
**Google Place ID:**ChIJ2eUgeAK6j4ARbn5u_wAGqWA
**GPS Coordinates:**34.0522,-118.2437
### Business Name (Optional)Add your business name to identify it in the results. The actor will mark it with `is_source_business: true` if found.
## 📤 Output
Results are formatted as one row per competitor, perfect for CSV export:
```json{ "input_location": "123 Main St, Beverly Hills, CA", "input_business_name": "My Med Spa", "search_query": "medical spa", "latitude": 34.0522, "longitude": -118.2437, "competitor_name": "Luxury Spa", "competitor_rank": 1, "competitor_rating": 4.8, "competitor_reviews": 245, "competitor_category": "Medical spa", "competitor_address": "456 Oak St", "is_source_business": false}⚡ Features
Bulk Processing: Analyze multiple locations in one runSmart Detection: Automatically identifies address vs Place ID vs coordinatesCSV Ready: Flattened output structure for easy Excel/CSV exportSource Identification: Marks your business if found in resultsFlexible Search: Customize search terms for any business type
🚀 Example RunInput:
Search: "coffee shops"Locations:
350 5th Ave, New York, NY | StarbucksChIJ2eUgeAK6j4ARbn5u_wAGqWA | 40.7128,-74.0060 | My Coffee Shop
Output:Up to 30 competitors per location with full details⚠️ Limitations
Maximum ~30 competitors per location (Google Maps limitation)Requires stable internet connectionBest results with specific search termsCoordinates must be in decimal format without spaces
💡 Tips
Use specific search terms for better resultsInclude business names to track your locationsTry different search terms to find all competitorsExport to CSV for easy analysis in Excel
🐛 SupportFound an issue or have a suggestion? Please create an issue on the actor page.{ "actorSpecification": 1, "name": "google-maps-competitor-analyzer", "title": "Google Maps Competitor Analyzer", "description": "Find competitors near any business location using Google Maps. Enter addresses, Place IDs, or coordinates to discover nearby businesses with ratings, reviews, and rankings. Perfect for market research, site selection, and competitive analysis. Supports bulk locations and custom search terms.", "version": "1.8", "buildTag": "latest", "meta": { "templateId": "python-playwright" }, "input": "./input_schema.json", "dockerfile": "../Dockerfile", "storages": { "dataset": "./dataset_schema.json" }}{ "actorSpecification": 1, "fields": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { "competitor_name": { "type": "string", "description": "Name of the competitor business" }, "competitor_address": { "type": "string", "description": "Full street address of the competitor" }, "competitor_city": { "type": "string", "description": "City extracted from the address" }, "competitor_state": { "type": "string", "description": "State abbreviation extracted from the address" }, "competitor_category": { "type": "string", "description": "Business category/type as shown on Google Maps" }, "competitor_rank": { "type": "integer", "description": "Ranking position in Google Maps search results", "minimum": 1 }, "competitor_rating": { "type": ["number", "null"], "description": "Average star rating (1.0 to 5.0)", "minimum": 1.0, "maximum": 5.0, "nullable": true }, "competitor_reviews": { "type": ["integer", "null"], "description": "Total number of Google reviews", "minimum": 0, "nullable": true }, "competitor_place_id": { "type": "string", "description": "Google Maps Place ID" }, "competitor_website": { "type": "string", "description": "Business website URL if available" }, "competitor_phone": { "type": "string", "description": "Business phone number if available" }, "latitude": { "type": ["number", "null"], "description": "Latitude coordinate of the search location", "minimum": -90, "maximum": 90, "nullable": true }, "longitude": { "type": ["number", "null"], "description": "Longitude coordinate of the search location", "minimum": -180, "maximum": 180, "nullable": true }, "search_query": { "type": "string", "description": "The search term used to find competitors" }, "is_source_business": { "type": "boolean", "description": "True if this result matches the input business name" }, "input_location": { "type": "string", "description": "The original location input" }, "input_business_name": { "type": "string", "description": "The business name provided with the location" } }, "required": [ "competitor_name", "competitor_address", "competitor_city", "competitor_state", "competitor_category", "competitor_rank", "latitude", "longitude", "search_query", "is_source_business", "input_location", "input_business_name" ], "additionalProperties": false }, "views": {}}{ "title": "🗺️ Google Maps Competitor Analyzer", "description": "🔍 Discover competitors near any business location using Google Maps! Perfect for market research, site selection, and competitive analysis. Get detailed insights including ratings, reviews, rankings, and contact info for businesses in your target market. 🎯", "type": "object", "schemaVersion": 1, "properties": { "inputMethod": { "title": "📝 Input Method", "type": "string", "description": "Choose how you want to enter your locations", "default": "structured", "enum": ["structured", "bulk"], "enumTitles": ["🏗️ Structured Input (City/State/Zip)", "📋 Bulk Paste"], "editor": "select" }, "searchQuery": { "title": "🔎 Search Category", "type": "string", "description": "🏪 What type of businesses are you looking for? Be specific for best results! Examples: \"restaurant\", \"coffee shop\", \"dentist\", \"gym\", \"hair salon\", \"auto repair\", \"pharmacy\", \"hotel\"", "editor": "textfield" }, "city": { "title": "🏙️ City", "type": "string", "description": "Enter the city name (e.g., Seattle, Los Angeles, New York)", "editor": "textfield" }, "state": { "title": "📍 State", "type": "string", "description": "Enter the 2-letter state code (e.g., WA, CA, NY)", "editor": "textfield", "pattern": "^[A-Z]{2}$", "maxLength": 2 }, "zipCode": { "title": "📮 ZIP Code", "type": "string", "description": "Enter the 5-digit ZIP code (optional)", "editor": "textfield", "pattern": "^\\d{5}$", "maxLength": 5 }, "businessName": { "title": "🏢 Your Business Name", "type": "string", "description": "Optional: Enter your business name to identify it in the results", "editor": "textfield" }, "bulkLocations": { "title": "📋 Bulk Location Input", "type": "string", "description": "Paste multiple locations, one per line. Supported formats:\n\n✅ **Full addresses**: 123 Main St, Seattle, WA 98101\n✅ **City, State**: Seattle, WA\n✅ **ZIP codes**: 98101\n✅ **Place IDs**: ChIJ2eUgeAK6j4ARbn5u_wAGqWA\n✅ **Coordinates**: 47.6062,-122.3321\n✅ **With business names**: 123 Main St, Seattle, WA | My Business\n\n💡 Tip: Use pipe (|) to separate address from business name", "editor": "textarea" }, "locations": { "title": "📍 Advanced Location Input", "type": "array", "description": "For power users: Enter locations with full control over format", "editor": "keyValue", "prefill": [ { "key": "123 Main St, Beverly Hills, CA 90210", "value": "My Business" } ] }, "maxCompetitors": { "title": "📊 Maximum Competitors per Location", "type": "integer", "description": "🎯 How many nearby competitors do you want to analyze? Google Maps typically shows 20-30 businesses in a search area. More competitors = more comprehensive analysis!", "default": 30, "minimum": 1, "maximum": 30, "editor": "number" }, "verbose": { "title": "🔧 Debug Mode", "type": "boolean", "description": "🐛 Enable detailed logging to troubleshoot issues. This shows exactly how the scraper parses each business listing - useful if you're getting unexpected results!", "default": false, "editor": "checkbox" }, "proxyConfiguration": { "title": "🌐 Proxy Settings", "type": "object", "description": "🛡️ Use residential proxies to avoid rate limits and blocks. Highly recommended when analyzing multiple locations or running frequently!", "editor": "proxy", "prefill": { "useApifyProxy": true } } }, "required": ["searchQuery"]}1
1import asyncio2
3from .main import main4
5# Execute the Actor entry point.6asyncio.run(main())1import asyncio2import re3import json4from urllib.parse import quote_plus5from apify import Actor6from playwright.async_api import async_playwright, TimeoutError7
8# ======================================================================9# HELPER FUNCTIONS v1.9 - Generic category extraction10# ======================================================================11def is_valid_business_name(name):12 if not name or len(name.strip()) < 1: return False13 if name.strip().startswith('"') and name.strip().endswith('"'): return False14 name_lower = name.strip().lower()15 if len(name_lower) > 80: return False16 # Reject single digits or numbers - these are list markers17 if name.strip().isdigit(): return False18 # Reject common UI elements19 if name_lower in ['x', '+', '-', '·', '|', '/', '\\', '^', '*']: return False20 if name_lower.startswith(('·', '$', '#', '@', 'http', 'sponsored')): return False21 # Skip if it looks like an address (starts with number and has street suffix)22 if re.match(r'^\d+\s+\w+\s+(st|street|ave|rd|dr|blvd)', name_lower, re.IGNORECASE): return False23 # Skip if it's just a number followed by a dot (like "1." or "9.")24 if re.match(r'^\d+\.?$', name.strip()): return False25 # Skip "Open/Closes" entries26 if re.match(r'^(open|closes?)\s*[⋅·]\s*(closes?|opens?)\s*\d+', name_lower): return False27 skip_patterns = ['google maps', 'search results', 'filter by', 'sort by', 'reviews', 'directions', 'open now', 'closed']28 if any(pattern in name_lower for pattern in skip_patterns): return False29 return True30
31def clean_business_name(name):32 if not name: return ""33 cleaned = name.strip().rstrip('&|\\').strip()34 cleanup_patterns = [35 r'\s*·.*$', r'^\s*\d+\.\s*', r'\s*\(\d+\)\s*$', r'\s*\d+\s*reviews?$',36 r'\s*\$+.*$', r'\s*Open\s*(24\s*hours?|now).*$', r'\s*Closed.*$'37 ]38 for pattern in cleanup_patterns:39 cleaned = re.sub(pattern, '', cleaned, flags=re.IGNORECASE)40 return cleaned.strip()41
42def extract_rating(lines):43 for line in lines:44 match = re.search(r'\b(\d\.\d)\b', line)45 if match:46 try: 47 rating = float(match.group(1))48 if 1.0 <= rating <= 5.0:49 return rating50 except: continue51 return None52
53def extract_review_count(lines):54 for line in lines:55 match = re.search(r'\((\d{1,5})\)', line)56 if match:57 try: return int(match.group(1))58 except: continue59 return None60
61def extract_category(lines):62 """Extract the Google-provided business category"""63 # Google typically shows category in one of these formats:64 # 1. "Restaurant · $$ · Address"65 # 2. "Restaurant" (on its own line)66 # 3. "Italian restaurant · Address"67 # 4. "4.5 (123) · Restaurant · Address"68 69 category_found = None70 71 for i, line in enumerate(lines):72 line = line.strip()73 74 # Skip empty lines, numbers, ratings, etc.75 if not line or line.isdigit() or re.match(r'^\d\.\d$', line) or re.match(r'^\(\d+\)$', line):76 continue77 78 # Skip common UI elements79 if line.lower() in ['ad', 'ads', 'sponsored', 'open', 'closed', 'open now', 'temporarily closed']:80 continue81 82 # Skip if it's clearly an address (starts with number + street)83 if re.match(r'^\d+\s+\w+\s+(st|street|ave|avenue|rd|road|dr|drive)', line, re.IGNORECASE):84 continue85 86 # Check if line contains separator (·) - often used to separate category from address87 if '·' in line:88 parts = [p.strip() for p in line.split('·')]89 for part in parts:90 # Skip price indicators ($, $$, $$$)91 if re.match(r'^\$+$', part):92 continue93 # Skip if it looks like an address94 if re.search(r'\d+.*?(st|street|ave|avenue|rd|road|dr|drive)', part, re.IGNORECASE):95 continue96 # Skip if it's a rating97 if re.match(r'^\d\.\d$', part):98 continue99 # Skip if it's empty or too short100 if not part or len(part) < 3:101 continue102 103 # This could be our category104 # Categories are usually generic terms without too many capital letters105 capital_count = sum(1 for c in part if c.isupper())106 if capital_count <= len(part.split()) + 1: # Allow one capital per word plus one107 # Common business type indicators108 if any(indicator in part.lower() for indicator in 109 ['restaurant', 'cafe', 'bar', 'shop', 'store', 'salon', 'spa', 110 'clinic', 'center', 'service', 'company', 'agency', 'studio',111 'gym', 'fitness', 'market', 'pharmacy', 'hotel', 'motel',112 'bakery', 'deli', 'pizza', 'sushi', 'thai', 'chinese', 'mexican',113 'dental', 'medical', 'law', 'repair', 'auto', 'car']):114 category_found = part115 break116 117 # If no category found with separator, check standalone lines118 if not category_found and i < 5: # Categories usually appear early119 # Skip if line is too long (probably not a category)120 if len(line) < 50:121 # Check if it contains common category words122 if any(indicator in line.lower() for indicator in 123 ['restaurant', 'cafe', 'bar', 'shop', 'store', 'salon', 'spa', 124 'clinic', 'center', 'service', 'company', 'agency', 'studio',125 'gym', 'fitness', 'market', 'pharmacy', 'hotel', 'motel']):126 # Make sure it's not a business name (check capitalization pattern)127 if not re.match(r'^[A-Z][a-z]+(\s+[A-Z][a-z]+){2,}', line): # Not like "John Smith Auto Repair"128 category_found = line129 break130 131 # Return the found category or a generic fallback132 return category_found if category_found else "Business"133
134def extract_address(lines):135 """Extract address from text lines"""136 # Try multiple patterns to find address137 address_parts = []138 found_address_start = False139 140 for i, line in enumerate(lines):141 # Skip very short lines (likely not addresses)142 if len(line) < 5:143 continue144 145 # Skip common non-address elements146 if line.lower() in ['ad', 'ads', 'sponsored', 'open', 'closed', 'open now', 'temporarily closed', 'opens soon', 'onsite services']:147 continue148 149 # Skip lines that are just hours/status info150 if re.match(r'^(opens?|closes?)\s+', line, re.IGNORECASE):151 continue152 153 # Skip if it's just a number (list marker)154 if line.strip().isdigit():155 continue156 157 # Check if this line contains rating or review count (not part of address)158 if re.match(r'^[\d.]+$', line) or re.match(r'^\(\d+\)$', line):159 continue160 161 # Check if line contains a category separator (·)162 # If so, extract the part after it as potential address163 if '·' in line and not found_address_start:164 parts = line.split('·')165 # Look for address patterns in each part166 for j, part in enumerate(parts):167 part = part.strip()168 if re.search(r'\d+.*?(?:st|street|ave|avenue|rd|road|dr|drive|blvd|boulevard|way|lane|ln|pkwy|parkway|ct|court|pl|place|suite|ste|unit|apt|#)', part, re.IGNORECASE):169 found_address_start = True170 # Include this part and all remaining parts (they might be part of address)171 address_parts.extend([p.strip() for p in parts[j:] if p.strip()])172 break173 if found_address_start:174 continue175 176 # Look for common address patterns177 has_street_pattern = re.search(r'\d+.*?(?:st|street|ave|avenue|rd|road|dr|drive|blvd|boulevard|way|lane|ln|pkwy|parkway|ct|court|pl|place|suite|ste|unit|apt|#)', line, re.IGNORECASE)178 has_state_zip = re.search(r'\b[A-Z]{2}\s+\d{5}\b', line)179 has_po_box = re.search(r'(p\.?o\.?\s*box|pmb)\s*\d+', line, re.IGNORECASE)180 181 if has_street_pattern or has_state_zip or has_po_box:182 # Found start of address183 found_address_start = True184 # Clean up the line185 cleaned_line = line.split('·')[-1].strip()186 # Remove common prefixes187 cleaned_line = re.sub(r'^(Open|Closed|Hours|Directions).*?·\s*', '', cleaned_line, flags=re.IGNORECASE)188 # Remove "Onsite services" suffix189 cleaned_line = re.sub(r',?\s*Onsite services\s*$', '', cleaned_line, flags=re.IGNORECASE)190 address_parts.append(cleaned_line)191 elif found_address_start and i < len(lines) - 1:192 # Stop if we hit status/hours info193 if re.search(r'(closes?|opens?|hours|open now|closed now|temporarily)', line, re.IGNORECASE):194 break195 # Skip if it looks like category info196 if any(word in line.lower() for word in ['restaurant', 'cafe', 'shop', 'salon', 'clinic', 'service']):197 if len(line) < 30: # Short category lines198 continue199 # Skip if it contains other non-address info200 if re.search(r'(reviews?|rating|website|call|directions)', line, re.IGNORECASE):201 continue202 # Skip "Onsite services"203 if 'onsite services' in line.lower():204 continue205 # Check if it might be city/state/zip206 if re.search(r'\b[A-Z]{2}\b', line) or re.search(r'\b\d{5}\b', line) or (len(line) < 50 and not line[0].isdigit()):207 address_parts.append(line.strip())208 else:209 # Probably not part of address anymore210 break211 212 # Join address parts213 if address_parts:214 # Join with comma if parts don't already have commas215 full_address = address_parts[0]216 for part in address_parts[1:]:217 if not full_address.endswith(',') and not part.startswith(','):218 full_address += ', ' + part219 else:220 full_address += ' ' + part221 return full_address.strip()222 223 return ""224
225def extract_place_ids(html_content: str) -> dict:226 ids = {'place_id': None, 'cid': None, 'website': None}227 place_id_match = re.search(r'(ChIJ[A-Za-z0-9_-]+)', html_content)228 if place_id_match:229 ids['place_id'] = place_id_match.group(1)230 cid_match = re.search(r'(0x[a-f0-9:]+)', html_content)231 if cid_match:232 ids['cid'] = cid_match.group(1)233 return ids234
235def extract_price_range(lines):236 """Extract price range indicators like $, $$, $$$"""237 for line in lines:238 # Look for price indicators239 if re.match(r'^\$+$', line.strip()):240 return line.strip()241 # Also check if it's part of a line with category242 if '·' in line and re.search(r'\$+', line):243 parts = line.split('·')244 for part in parts:245 if re.match(r'^\s*\$+\s*$', part):246 return part.strip()247 return ""248
249def extract_website_from_page(lines):250 """Extract website URL from detail panel"""251 for i, line in enumerate(lines):252 # Skip very long lines (likely not URLs)253 if len(line) > 200:254 continue255 256 # Look for lines that are URLs257 if re.match(r'^https?://', line):258 # Skip Google domains259 if any(domain in line for domain in ['google.com', 'maps.google', 'gstatic.com', 'googleapis.com']):260 continue261 return line.strip()262 263 # Look for domain patterns without protocol264 domain_match = re.match(r'^(?:www\.)?([a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)+)$', line)265 if domain_match:266 return f"https://{line.strip()}"267 268 return ""269
270def extract_booking_url(lines):271 """Extract booking/appointment URL"""272 booking_keywords = ['book', 'appointment', 'schedule', 'reserve', 'booking']273 274 for i, line in enumerate(lines):275 line_lower = line.lower()276 277 # Check if line contains booking keywords278 if any(keyword in line_lower for keyword in booking_keywords):279 # Check next few lines for URLs280 for j in range(i, min(i + 3, len(lines))):281 if re.match(r'^https?://', lines[j]):282 return lines[j].strip()283 284 return ""285
286def extract_services_offered(lines):287 """Extract services from business listing"""288 services = []289 services_section = False290 291 for line in lines:292 line_lower = line.lower()293 294 # Look for services section295 if 'services' in line_lower or 'offerings' in line_lower:296 services_section = True297 continue298 299 if services_section:300 # Stop at next section301 if any(keyword in line_lower for keyword in ['hours', 'reviews', 'photos', 'about']):302 break303 304 # Skip empty lines and UI elements305 if line.strip() and len(line) > 3 and not line.isdigit():306 services.append(line.strip())307 308 # Limit to reasonable number309 if len(services) > 10:310 break311 312 return services313
314def extract_attributes(lines):315 """Extract business attributes/amenities"""316 attributes = []317 attribute_keywords = ['wheelchair', 'parking', 'delivery', 'takeout', 'dine-in', 318 'wifi', 'accepts', 'outdoor', 'seating', 'drive-through',319 'curbside', 'restroom', 'masks required', 'appointment']320 321 for line in lines[:100]: # Check first 100 lines322 line_lower = line.lower()323 324 for keyword in attribute_keywords:325 if keyword in line_lower and len(line) < 100:326 attributes.append(line.strip())327 break328 329 return list(set(attributes)) # Remove duplicates330
331def extract_highlights(lines):332 """Extract business highlights"""333 highlights = []334 highlight_keywords = ['popular for', 'known for', 'best', 'famous', 'specialty']335 336 for line in lines:337 line_lower = line.lower()338 339 if any(keyword in line_lower for keyword in highlight_keywords):340 highlights.append(line.strip())341 342 if len(highlights) >= 3:343 break344 345 return highlights346
347def extract_photo_count(lines):348 """Extract number of photos"""349 for line in lines:350 # Look for "X photos" pattern351 photo_match = re.search(r'(\d+)\s*photos?', line, re.IGNORECASE)352 if photo_match:353 try:354 return int(photo_match.group(1))355 except:356 pass357 return 0358
359def extract_hours(lines):360 """Extract business hours"""361 hours = {}362 days = ['monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']363 364 for i, line in enumerate(lines):365 line_lower = line.lower()366 367 # Check if line contains day name368 for day in days:369 if day in line_lower:370 # Look for time pattern in same or next line371 time_pattern = r'\d{1,2}(?::\d{2})?\s*[ap]\.?m\.?\s*[–-]\s*\d{1,2}(?::\d{2})?\s*[ap]\.?m\.?'372 373 time_match = re.search(time_pattern, line, re.IGNORECASE)374 if time_match:375 hours[day.capitalize()] = time_match.group()376 elif i + 1 < len(lines):377 # Check next line378 time_match = re.search(time_pattern, lines[i + 1], re.IGNORECASE)379 if time_match:380 hours[day.capitalize()] = time_match.group()381 382 # Also check for "Closed"383 if 'closed' in line_lower:384 hours[day.capitalize()] = 'Closed'385 386 break387 388 return hours389
390def parse_address_components(address):391 """Extract city and state from address string"""392 if not address:393 return {'city': '', 'state': ''}394 395 components = {'city': '', 'state': ''}396 original_address = address397 398 # Remove "Onsite services" which is not part of address399 address = re.sub(r',?\s*Onsite services\s*,?', '', address, flags=re.IGNORECASE)400 401 # Common cities for this data - expanded to include Seattle area402 known_wa_cities = ['Seattle', 'Bellevue', 'Kirkland', 'Redmond', 'Renton', 'Newcastle', 403 'Issaquah', 'Sammamish', 'Bothell', 'Kenmore', 'Woodinville', 'Shoreline',404 'Lake Forest Park', 'Mercer Island', 'Tukwila', 'Burien', 'SeaTac']405 known_va_cities = ['Ashburn', 'Sterling', 'Leesburg', 'Herndon', 'Reston', 'Chantilly', 'Fairfax']406 known_ca_cities = ['Los Angeles', 'Santa Monica', 'Beverly Hills', 'Pasadena', 'Whittier', 407 'Encino', 'Burbank', 'Glendale', 'Hollywood', 'Sherman Oaks']408 409 all_known_cities = known_wa_cities + known_va_cities + known_ca_cities410 411 412 # Try to extract state (2 letter abbreviation)413 # IMPORTANT: Exclude NE, NW, SE, SW when they appear after street names414 state_pattern = r'\b(AL|AK|AZ|AR|CA|CO|CT|DE|FL|GA|HI|ID|IL|IN|IA|KS|KY|LA|ME|MD|MA|MI|MN|MS|MO|MT|NV|NH|NJ|NM|NY|NC|ND|OH|OK|OR|PA|RI|SC|SD|TN|TX|UT|VT|VA|WA|WV|WI|WY|DC)\b'415 416 # Look for state abbreviation, but check context to avoid matching NE/NW/SE/SW as state417 state_matches = []418 for match in re.finditer(state_pattern, address, re.IGNORECASE):419 # Check if this is actually a directional (NE) after a street name420 if match.group(1).upper() == 'NE':421 # Look at what comes before - if it's a street pattern, skip this match422 before_text = address[:match.start()].strip()423 if re.search(r'(Ave|Avenue|St|Street|Rd|Road|Dr|Drive|Way|Pl|Place|Blvd|Boulevard)\s*$', before_text, re.IGNORECASE):424 continue425 state_matches.append(match)426 427 if state_matches:428 # Use the last state match (most likely to be the actual state)429 state_match = state_matches[-1]430 components['state'] = state_match.group(1).upper()431 # Remove state and clean up432 address = address[:state_match.start()] + address[state_match.end():]433 address = address.strip().rstrip(',').strip()434 435 # Try to find city from known cities first436 city_found = False437 for known_city in all_known_cities:438 if known_city.lower() in original_address.lower():439 components['city'] = known_city440 city_found = True441 break442 443 # If no known city found and we have certain indicators, make educated guesses444 if not city_found:445 # For Seattle area addresses (contain NE, NW, SE, SW in street)446 if re.search(r'\b\d+\s+\d+\w*\s+(?:Ave|Avenue|St|Street|Way|Pl|Place)\s+(?:NE|NW|SE|SW)\b', original_address, re.IGNORECASE):447 # If we see these patterns, it's likely Seattle area448 if components['state'] == 'WA' or 'WA' in original_address:449 # Default to Seattle for unspecified WA addresses with directionals450 components['city'] = 'Seattle'451 elif re.search(r'(Kirkland|Bellevue|Redmond)', original_address, re.IGNORECASE):452 # Check surrounding text for city hints453 for city in ['Kirkland', 'Bellevue', 'Redmond']:454 if city.lower() in original_address.lower():455 components['city'] = city456 break457 458 # If still no city, try to extract from address structure459 if not components['city']:460 # Remove street address parts to find city461 # Split by comma and analyze parts462 parts = original_address.split(',')463 for part in reversed(parts): # Start from the end464 part = part.strip()465 466 # Skip if it's empty or too short467 if not part or len(part) < 3:468 continue469 470 # Skip "Onsite services" and similar471 if part.lower() in ['onsite services', 'online', 'virtual']:472 continue473 474 # Skip if it's a street address (starts with number)475 if re.match(r'^\d+\s+', part):476 continue477 478 # Skip if it contains suite/unit/building info479 if re.search(r'\b(suite|ste|unit|apt|building|bldg|floor|fl|#)\s*[\w\d-]*\b', part, re.IGNORECASE):480 continue481 482 # Skip if it ends with street type483 if re.search(r'\b(st|street|ave|avenue|rd|road|dr|drive|blvd|boulevard|way|lane|ln|pkwy|parkway|ct|court|pl|place|plaza|plz|cir|circle|ter|terrace|commons|cmns)\s*(?:#?\d+)?$', part, re.IGNORECASE):484 continue485 486 # Skip if it's just a state + zip pattern487 if re.match(r'^[A-Z]{2}\s+\d{5}(?:-\d{4})?$', part):488 continue489 490 # This might be the city491 if part and not re.match(r'^\d', part):492 components['city'] = part493 break494 495 # Final pass - if we identified the state but not city, use common defaults496 if components['state'] and not components['city']:497 if components['state'] == 'WA':498 # For WA addresses without city, check if address suggests certain areas499 if 'Green Lake' in original_address or 'Greenlake' in original_address:500 components['city'] = 'Seattle'501 elif any(indicator in original_address for indicator in ['120th', '122nd', '116th', '130th', '148th', '164th']):502 # These numbered streets are common in Bellevue/Kirkland area503 if 'NE' in original_address:504 components['city'] = 'Bellevue' # Default for NE numbered streets505 elif components['state'] == 'VA' and any(city in original_address for city in known_va_cities):506 for city in known_va_cities:507 if city.lower() in original_address.lower():508 components['city'] = city509 break510 511 return components512
513def parse_location_type(location_str):514 """Determine if the location is an address, Place ID, or coordinates"""515 location_str = location_str.strip()516 517 # Check if it's a Place ID (starts with ChIJ)518 if location_str.startswith('ChIJ'):519 return 'place_id', location_str520 521 # Check if it's coordinates (format: lat,lng) - now more forgiving522 # Remove spaces around comma for better matching523 normalized = location_str.replace(' ', '')524 coord_pattern = r'^-?\d+\.?\d*,-?\d+\.?\d*$'525 if re.match(coord_pattern, normalized):526 parts = normalized.split(',')527 try:528 lat = float(parts[0])529 lng = float(parts[1])530 return 'coordinates', (lat, lng)531 except:532 pass533 534 # Otherwise, treat as address535 return 'address', location_str536
537# ======================================================================538# ACTOR SCRAPING LOGIC - UPDATED WITH TWO-PASS APPROACH539# ======================================================================540async def geocode_with_address(page, address: str) -> dict:541 Actor.log.info(f"Geocoding address: {address}...")542 query = quote_plus(address)543 maps_url = f"https://www.google.com/maps/search/{query}"544 await page.goto(maps_url, wait_until='load', timeout=25000)545 await asyncio.sleep(5)546 coord_match = re.search(r'@(-?\d+\.\d+),(-?\d+\.\d+)', page.url)547 if not coord_match:548 return {'found': False, 'error': 'Could not find coordinates from address'}549 lat, lng = float(coord_match.group(1)), float(coord_match.group(2))550 html_content = await page.content()551 ids = extract_place_ids(html_content)552 return {'found': True, 'latitude': lat, 'longitude': lng, **ids}553
554async def geocode_with_place_id(page, place_id: str) -> dict:555 Actor.log.info(f"Geocoding Place ID: {place_id}...")556 maps_url = f"https://www.google.com/maps/search/?api=1&query=some_text&query_place_id={place_id}"557 await page.goto(maps_url, wait_until='load', timeout=25000)558 await asyncio.sleep(5)559 coord_match = re.search(r'@(-?\d+\.\d+),(-?\d+\.\d+)', page.url)560 if not coord_match: return {'found': False, 'error': 'Could not find coordinates from Place ID'}561 lat, lng = float(coord_match.group(1)), float(coord_match.group(2))562 html_content = await page.content()563 ids = extract_place_ids(html_content)564 return {'found': True, 'latitude': lat, 'longitude': lng, 'place_id': place_id, **ids}565
566async def scrape_competitors_two_pass(playwright_instance, source_company_name: str, lat: float, lon: float, 567 search_query: str, max_competitors: int, search_location: str = None, 568 verbose: bool = False):569 """Two-pass scraping approach: Pass 1 at 25% zoom, Pass 2 at 50% zoom for details"""570 571 Actor.log.info(f"Starting two-pass scraping at {lat},{lon} for '{search_query}'...")572 query = quote_plus(search_query)573 zoom_level = 11574 url = f"https://www.google.com/maps/search/{query}/@{lat},{lon},{zoom_level}z"575 576 # ========== PASS 1: DISCOVERY AT 25% ZOOM ==========577 Actor.log.info("PASS 1: Discovering competitors at 25% zoom...")578 579 browser1 = await playwright_instance.chromium.launch(headless=False)580 context1 = await browser1.new_context(581 viewport={'width': 2560, 'height': 1440},582 user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'583 )584 page1 = await context1.new_page()585 586 competitors = []587 588 try:589 await page1.goto(url, wait_until='domcontentloaded', timeout=45000)590 591 # Set zoom to 25% for maximum results592 await page1.evaluate('document.body.style.zoom = "0.25"')593 await asyncio.sleep(3)594 595 # Trigger content loading596 Actor.log.info("Triggering content loading...")597 await page1.mouse.move(960, 540)598 await page1.mouse.wheel(0, 500)599 await asyncio.sleep(2)600 await page1.mouse.wheel(0, -500)601 await asyncio.sleep(3)602 603 # Find all business elements604 selectors = ['[role="article"]', '.Nv2PK.THOPZb.CpccDe', '.lI9IFe', '.hfpxzc']605 elements = []606 for selector in selectors:607 elements = await page1.query_selector_all(selector)608 if elements and len(elements) > 5:609 Actor.log.info(f"Found {len(elements)} results with selector: {selector}")610 break611 612 if not elements:613 Actor.log.error("No elements found in Pass 1")614 return []615 616 await asyncio.sleep(5) # Let everything load617 618 # Extract basic info from all competitors619 Actor.log.info(f"Extracting basic info from {len(elements)} elements...")620 cleaned_source_name = re.sub(r'[^a-z0-9]', '', source_company_name.lower()) if source_company_name else ""621 622 for i, element in enumerate(elements[:max_competitors + 5]): # Extra for duplicates623 if len(competitors) >= max_competitors:624 break625 626 try:627 text = await element.inner_text()628 if not text or len(text.strip()) < 5:629 continue630 631 lines = [line.strip() for line in text.split('\n') if line.strip()]632 633 # Extract business name634 business_name = None635 for line in lines[:10]:636 if line.strip().isdigit() or re.match(r'^\d+\.$', line):637 continue638 if line.lower() in ['ad', 'ads', 'sponsored', 'open', 'closed', 'open now']:639 continue640 if re.match(r'^\d\.\d$', line) or re.match(r'^\(\d+\)$', line):641 continue642 643 if is_valid_business_name(line):644 business_name = clean_business_name(line)645 if business_name:646 break647 648 if not business_name:649 continue650 651 # Check for duplicates652 if any(c['business_name'] == business_name for c in competitors):653 continue654 655 # Check if it's the source business656 is_source = False657 if cleaned_source_name:658 cleaned_competitor_name = re.sub(r'[^a-z0-9]', '', business_name.lower())659 is_source = cleaned_source_name in cleaned_competitor_name or cleaned_competitor_name in cleaned_source_name660 661 # Extract basic info662 rating = extract_rating(lines)663 review_count = extract_review_count(lines)664 address = extract_address(lines)665 category = extract_category(lines)666 price_range = extract_price_range(lines)667 668 competitor = {669 'business_name': business_name,670 'google_rank': len(competitors) + 1,671 'is_source_business': is_source,672 'average_rating': rating,673 'number_of_reviews': review_count,674 'main_category': category,675 'address': address,676 'city': '',677 'state': '',678 'website': '',679 'phone': '',680 'place_id': '',681 'price_range': price_range,682 'services_offered': [],683 'attributes': [],684 'highlights': [],685 'photo_count': 0,686 'hours': {},687 'booking_url': ''688 }689 690 competitors.append(competitor)691 Actor.log.info(f"#{len(competitors)}: {business_name} - {category}")692 693 except Exception as e:694 continue695 696 Actor.log.info(f"PASS 1 COMPLETE: Found {len(competitors)} competitors")697 698 except Exception as e:699 Actor.log.error(f"Error in Pass 1: {e}")700 701 finally:702 Actor.log.info("Closing Pass 1 browser...")703 await browser1.close()704 705 # ========== PASS 2: GET DETAILS AT 50% ZOOM ==========706 if not competitors:707 return []708 709 Actor.log.info("PASS 2: Getting detailed information at 50% zoom...")710 711 browser2 = await playwright_instance.chromium.launch(headless=False)712 context2 = await browser2.new_context(713 viewport={'width': 1920, 'height': 1080},714 user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'715 )716 page2 = await context2.new_page()717 718 try:719 await page2.goto(url, wait_until='domcontentloaded', timeout=45000)720 721 # Set zoom to 50% from the start722 await page2.evaluate('document.body.style.zoom = "0.5"')723 await asyncio.sleep(5)724 725 # Trigger some interaction to ensure content loads726 await page2.mouse.move(500, 400)727 await page2.mouse.wheel(0, 300)728 await asyncio.sleep(2)729 await page2.mouse.wheel(0, -300)730 await asyncio.sleep(3)731 732 Actor.log.info("Starting to click on businesses for details...")733 734 # Process each competitor735 successful_extractions = 0736 last_clicked_name = None # Track what we last clicked737 738 for idx, competitor in enumerate(competitors[:max_competitors]):739 try:740 Actor.log.info(f"Getting details for #{idx + 1}: {competitor['business_name']}")741 742 # IMPORTANT: Ensure we're starting fresh by closing any open panels743 try:744 await page2.keyboard.press('Escape')745 await asyncio.sleep(0.5)746 except:747 pass748 749 # Find and click on businesses that are visible750 elements = await page2.query_selector_all('[role="article"]')751 if not elements:752 elements = await page2.query_selector_all('.Nv2PK.THOPZb.CpccDe')753 754 clicked = False755 for element in elements:756 try:757 # First check if element is visible758 is_visible = await element.is_visible()759 if not is_visible:760 continue761 762 text = await element.inner_text()763 764 # Skip if this looks like the last business we clicked765 if last_clicked_name and last_clicked_name.lower() in text.lower():766 continue767 768 # Flexible matching769 competitor_name_parts = [part for part in competitor['business_name'].lower().split() if len(part) > 2]770 element_text_lower = text.lower()771 772 matches = sum(1 for part in competitor_name_parts if part in element_text_lower)773 774 if matches >= max(1, len(competitor_name_parts) * 0.6):775 # Store what we're clicking776 last_clicked_name = competitor['business_name']777 778 await element.click()779 clicked = True780 781 # Wait for panel to fully load782 await asyncio.sleep(1.5) # Slightly longer wait for panel to load783 784 # Get text ONLY from the detail panel, not the entire page785 # Try to find the detail panel specifically786 detail_panel = None787 try:788 # Common selectors for Google Maps detail panels789 panel_selectors = [790 '[role="main"]',791 '.m6QErb.DxyBCb',792 '.xaLxOe',793 'div[jslog*="panel"]'794 ]795 796 for selector in panel_selectors:797 panels = await page2.query_selector_all(selector)798 if panels:799 # Get the last panel (usually the detail panel)800 detail_panel = panels[-1]801 break802 except:803 pass804 805 # Get text from detail panel or fallback to full page806 if detail_panel:807 panel_text = await detail_panel.inner_text()808 lines = panel_text.split('\n')809 else:810 # Fallback to full page but be more careful811 full_page_text = await page2.inner_text('body')812 lines = full_page_text.split('\n')813 814 # Extract website with improved filtering815 website = extract_website_from_page(lines)816 if website:817 # Double-check this website isn't the same as previous818 if idx > 0 and competitors[idx-1].get('website') == website:819 Actor.log.info(f" Detected duplicate website {website}, skipping")820 else:821 competitor['website'] = website822 successful_extractions += 1823 Actor.log.info(f" Website: {website}")824 825 # Extract booking URL826 booking_url = extract_booking_url(lines)827 if booking_url and booking_url != competitor.get('website'):828 competitor['booking_url'] = booking_url829 Actor.log.info(f" Booking URL: {booking_url}")830 831 # Look for phone in detail panel832 for line in lines[:50]: # Check first 50 lines only833 phone_match = re.search(r'\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}', line)834 if phone_match:835 competitor['phone'] = phone_match.group()836 Actor.log.info(f" Phone: {competitor['phone']}")837 break838 839 # Extract additional details840 services = extract_services_offered(lines)841 if services:842 competitor['services_offered'] = services843 Actor.log.info(f" Services found: {len(services)}")844 845 attributes = extract_attributes(lines)846 if attributes:847 competitor['attributes'] = attributes848 Actor.log.info(f" Attributes: {', '.join(attributes[:3])}...")849 850 highlights = extract_highlights(lines)851 if highlights:852 competitor['highlights'] = highlights853 854 photo_count = extract_photo_count(lines)855 if photo_count:856 competitor['photo_count'] = photo_count857 Actor.log.info(f" Photos: {photo_count}")858 859 hours = extract_hours(lines)860 if hours:861 competitor['hours'] = hours862 Actor.log.info(f" Hours: Found for {len(hours)} days")863 864 # Extract place_id from page content865 page_html = await page2.content()866 place_id_match = re.search(r'(ChIJ[A-Za-z0-9_-]+)', page_html)867 if place_id_match:868 competitor['place_id'] = place_id_match.group(1)869 870 # IMPORTANT: Close panel and wait871 await page2.keyboard.press('Escape')872 await asyncio.sleep(1) # Wait for panel to close873 break874 875 except Exception as e:876 continue877 878 if not clicked:879 Actor.log.info(f" Could not find {competitor['business_name']} in view")880 # Try scrolling to find more businesses881 try:882 await page2.evaluate('document.querySelector("[role=\'feed\']").scrollTop += 500')883 await asyncio.sleep(1.5) # Wait for scroll to complete884 except:885 pass886 887 except Exception as e:888 Actor.log.error(f" Error: {str(e)}")889 # Always try to close any open panel on error890 try:891 await page2.keyboard.press('Escape')892 await asyncio.sleep(0.5)893 except:894 pass895 continue896 897 Actor.log.info(f"PASS 2 COMPLETE: Extracted details for {successful_extractions} businesses")898 899 # Parse city/state for all competitors900 for competitor in competitors:901 if competitor['address']:902 components = parse_address_components(competitor['address'])903 competitor['city'] = components['city']904 competitor['state'] = components['state']905 906 # Use search location as fallback907 if search_location and (not competitor['city'] or not competitor['state']):908 search_components = parse_address_components(search_location)909 if not competitor['city'] and search_components['city']:910 competitor['city'] = search_components['city']911 if not competitor['state'] and search_components['state']:912 competitor['state'] = search_components['state']913 914 except Exception as e:915 Actor.log.error(f"Error in Pass 2: {e}")916 917 finally:918 Actor.log.info("Closing Pass 2 browser...")919 await browser2.close()920 921 return competitors# ======================================================================922# MAIN ACTOR FUNCTION923# ======================================================================924async def main() -> None:925 async with Actor:926 actor_input = await Actor.get_input() or {}927 928 # Parse input from new schema929 input_method = actor_input.get('inputMethod', 'structured')930 search_query = actor_input.get('searchQuery', '')931 932 # Make search query required933 if not search_query:934 Actor.log.error("No search query provided. Please specify what type of business to search for.")935 await Actor.push_data({936 'error': 'No search query provided',937 'message': 'Please provide a search query (e.g., "restaurant", "auto repair", "dentist")'938 })939 return940 941 max_competitors = actor_input.get('maxCompetitors', 30)942 proxy_configuration = actor_input.get('proxyConfiguration')943 verbose = actor_input.get('verbose', False)944 945 # Process locations based on input method946 locations = []947 business_names = []948 949 if input_method == 'structured':950 # Handle city/state/zip input951 city = actor_input.get('city', '').strip()952 state = actor_input.get('state', '').strip()953 zip_code = actor_input.get('zipCode', '').strip()954 business_name = actor_input.get('businessName', '').strip()955 956 # Build location string957 location_parts = []958 if city:959 location_parts.append(city)960 if state:961 location_parts.append(state)962 if zip_code:963 location_parts.append(zip_code)964 965 if location_parts:966 location_str = ', '.join(location_parts)967 locations.append(location_str)968 business_names.append(business_name)969 Actor.log.info(f"Structured input: {location_str} | Business: {business_name or 'None'}")970 971 elif input_method == 'bulk':972 # Handle bulk paste input973 bulk_text = actor_input.get('bulkLocations', '')974 if bulk_text:975 lines = [line.strip() for line in bulk_text.strip().split('\n') if line.strip()]976 for line in lines:977 # Check if line contains pipe separator for business name978 if '|' in line:979 location, business = line.split('|', 1)980 locations.append(location.strip())981 business_names.append(business.strip())982 else:983 locations.append(line)984 business_names.append('')985 Actor.log.info(f"Bulk input: {len(locations)} locations parsed")986 987 # Also check for legacy locations array988 locations_data = actor_input.get('locations', [])989 if locations_data and not locations:990 for item in locations_data:991 if isinstance(item, dict):992 locations.append(item.get('key', ''))993 business_names.append(item.get('value', ''))994 else:995 locations.append(str(item))996 business_names.append('')997 Actor.log.info(f"Legacy input: {len(locations)} locations from array")998 999 # Log input summary1000 Actor.log.info(f"Total locations to process: {len(locations)}")1001 Actor.log.info(f"Search query: '{search_query}'")1002 Actor.log.info(f"Max competitors: {max_competitors}")1003 if verbose:1004 Actor.log.info(f"Verbose mode: ENABLED")1005 1006 if not locations:1007 Actor.log.error("No locations provided. Please add at least one location.")1008 await Actor.push_data({1009 'error': 'No locations provided',1010 'message': 'Please provide at least one location (address, Place ID, or coordinates)'1011 })1012 return1013 1014 Actor.log.info(f"Processing {len(locations)} locations...")1015 Actor.log.info(f"Search query: '{search_query}'")1016 Actor.log.info(f"Max competitors: {max_competitors}")1017 1018 # Track total results pushed1019 total_results = 01020 1021 # Launch browser for geocoding only1022 async with async_playwright() as p:1023 # Handle proxy if configured1024 browser_options = {"headless": True}1025 1026 if proxy_configuration and proxy_configuration.get('useApifyProxy'):1027 try:1028 # For Apify proxy, you might need to check the correct method1029 # This is a placeholder - check Apify docs for your SDK version1030 Actor.log.info("Proxy requested but implementation needs to be verified for your Apify SDK version")1031 except Exception as e:1032 Actor.log.warning(f"Could not set up proxy: {e}")1033 1034 Actor.log.info("Launching geocoding browser...")1035 browser = await p.chromium.launch(**browser_options)1036 context = await browser.new_context(1037 viewport={'width': 1920, 'height': 1080},1038 user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'1039 )1040 page = await context.new_page()1041
1042 # Process each location1043 for idx, location_str in enumerate(locations):1044 # Get business name from the paired value1045 business_name = business_names[idx] if idx < len(business_names) else None1046 1047 # Skip empty locations1048 if not location_str or not location_str.strip():1049 continue1050 1051 # Parse location type1052 location_type, location_data = parse_location_type(location_str)1053 1054 Actor.log.info(f"\nProcessing location {idx + 1}/{len(locations)}: {location_str}")1055 if business_name:1056 Actor.log.info(f"Business name: {business_name}")1057 1058 geo = {'found': False}1059 1060 try:1061 if location_type == 'coordinates':1062 lat, lng = location_data1063 Actor.log.info(f"Using provided coordinates: {lat}, {lng}")1064 geo = {'found': True, 'latitude': lat, 'longitude': lng}1065 elif location_type == 'place_id':1066 geo = await geocode_with_place_id(page, location_data)1067 else: # address1068 geo = await geocode_with_address(page, location_data)1069 except Exception as e:1070 Actor.log.exception(f"Geocoding failed: {e}")1071 geo = {'found': False, 'error': str(e)}1072 1073 # Scrape competitors if geocoding succeeded1074 competitors_list = []1075 if geo.get('found'):1076 try:1077 # Use the two-pass approach1078 competitors_list = await scrape_competitors_two_pass(1079 p, # Pass playwright instance1080 business_name, 1081 geo['latitude'], 1082 geo['longitude'],1083 search_query,1084 max_competitors,1085 location_str, # Pass the search location for inference1086 verbose # Pass verbose flag1087 )1088 Actor.log.info(f"Found {len(competitors_list)} competitors for {business_name or location_str}")1089 except Exception as e:1090 Actor.log.exception(f"Competitor scraping failed: {e}")1091 else:1092 Actor.log.warning(f"Skipping competitor search - geocoding failed: {geo.get('error', 'Unknown error')}")1093 1094 # Push results - FLATTENED for CSV1095 if competitors_list:1096 # One row per competitor1097 for competitor in competitors_list:1098 result = {1099 # Competitor details in EXACT requested order1100 'competitor_name': str(competitor['business_name']),1101 'competitor_address': str(competitor['address'] or ''),1102 'competitor_city': str(competitor['city'] or ''),1103 'competitor_state': str(competitor['state'] or ''),1104 'competitor_category': str(competitor['main_category']),1105 'competitor_rank': int(competitor['google_rank']),1106 'competitor_rating': float(competitor['average_rating']) if competitor['average_rating'] is not None else 0.0,1107 'competitor_reviews': int(competitor['number_of_reviews']) if competitor['number_of_reviews'] is not None else 0,1108 'competitor_place_id': str(competitor['place_id'] or ''),1109 'latitude': float(geo.get('latitude')) if geo.get('latitude') is not None else 0.0,1110 'longitude': float(geo.get('longitude')) if geo.get('longitude') is not None else 0.0,1111 'search_query': str(search_query),1112 'competitor_website': str(competitor.get('website', '')),1113 'competitor_phone': str(competitor.get('phone', '')),1114 # Source business info at the end1115 'is_source_business': bool(competitor['is_source_business']),1116 'input_location': str(location_str),1117 'input_business_name': str(business_name or '')1118 }1119 # Debug: Log the result before pushing1120 if verbose:1121 Actor.log.info(f"Pushing result: {json.dumps(result, indent=2)}")1122 await Actor.push_data(result)1123 total_results += 11124 else:1125 # No competitors found - still push location info1126 result = {1127 # Empty competitor details in EXACT requested order1128 'competitor_name': '',1129 'competitor_address': '',1130 'competitor_city': '',1131 'competitor_state': '',1132 'competitor_category': '',1133 'competitor_rank': 0,1134 'competitor_rating': 0.0,1135 'competitor_reviews': 0,1136 'competitor_place_id': '',1137 'latitude': float(geo.get('latitude')) if geo.get('latitude') is not None else 0.0,1138 'longitude': float(geo.get('longitude')) if geo.get('longitude') is not None else 0.0,1139 'search_query': str(search_query),1140 'competitor_website': '',1141 'competitor_phone': '',1142 # Source business info at the end1143 'is_source_business': False,1144 'input_location': str(location_str),1145 'input_business_name': str(business_name or '')1146 }1147 await Actor.push_data(result)1148 total_results += 11149 1150 await browser.close()1151 Actor.log.info(f"\nScraping completed! Processed {len(locations)} locations and found {total_results} total results.")1152
1153# Entry point1154if __name__ == "__main__":1155 asyncio.run(main())