Sold Realestate Au

1import asyncio
2
3from .main import main
4
5# Execute the Actor entry point.
6asyncio.run(main())

1API_KEY = '7uYsiEQtr721LUrV7jghPBbgrzDC1uRDgTVYQHS5m9feJFfyv4RxZT70Hq2o'
2SITEMAP_ID = 1275385
3SEARCH_TYPE = 'sold'

1from apify import Actor
2from httpx import AsyncClient
3from time import sleep
4from .utils import (
5    extract_data_lists,
6    generate_url
7)
8
9from .webscraper_ai import (
10    start_scraping,
11    get_scraping_status_by_job,
12    get_scraping_results
13)
14
15from .all_config import SEARCH_TYPE
16
17
18async def main() -> None:
19    async with Actor:
20        # Retrieve the input object for the Actor. The structure of input is defined in input_schema.json.
21        actor_input = await Actor.get_input() or {}
22        address = actor_input.get('address')
23        gen_url = generate_url(address, search_type=SEARCH_TYPE)
24
25        # {'success': True, 'data': {'id': 28731019, 'custom_id': None}}
26        start_scraping_st = start_scraping(url_=gen_url)
27        if start_scraping_st.get('success'):
28            Actor.log.info(f'Started scraping process on webscraper_io')
29        else:
30            Actor.log.error(
31                f'Failed to start scraping process on webscraper_io')
32            return
33
34        job_id = start_scraping_st.get('data', {}).get('id', None)
35        if job_id:
36            Actor.log.info(
37                f'Started successfully with job ID: {job_id} | webscraper_io')
38
39            # Check the status of the scraping job until it is finished.
40            while True:
41                job_status = get_scraping_status_by_job(job_id)
42                info_status = job_status.get('data', {}).get('status', None)
43                sleep(10)
44                Actor.log.info(f'Job status: {info_status}')
45                if info_status == 'finished':
46                    Actor.log.info(f'Scraping job has finished')
47                    break
48                if info_status == 'stopped':
49                    Actor.log.error(f'Scraping job has been stopped')
50                    break
51
52            check_after_status = get_scraping_status_by_job(job_id)
53            check_after_status = check_after_status.get(
54                'data', {}).get('status', None)
55
56            # if 'finished' == str(check_after_status) or 'stopped' == str(check_after_status):
57            #     Actor.log.error(f'Scraping job did not finish successfully')
58            #     return
59
60            # Retrieve the scraping results from the job.
61            scraping_results = get_scraping_results(job_id)
62            if scraping_results:
63                Actor.log.info(f'Found Scraping results:')
64
65                # Extract the data from the scraping results.
66                Actor.log.info(f'Extracting data from the scraping results')
67                data_lists = extract_data_lists(scraping_results, SEARCH_TYPE)
68                Actor.log.info(f'Total items: {len(data_lists)}')
69                if data_lists:
70                    for item in data_lists:
71                        Actor.log.info(
72                            f'---------------------------------------------')
73                        Actor.log.info(f'Extracted data: {item}')
74                        Actor.log.info(
75                            f'---------------------------------------------')
76                        await Actor.push_data(item)
77
78            else:
79                Actor.log.error(f'Failed to retrieve scraping results')
80                return
81
82        else:
83            Actor.log.error(f'Failed to retrieve job ID')
84            return

1from parsel.selector import Selector
2
3
4def extract_data_lists(driver_source, search_type="buy"):
5    items = []
6
7    try:
8
9        response = Selector(text=str(driver_source))
10        properties = response.xpath(
11            '//*[contains(@class, "PropertyCardLayout")]')
12        for prop in properties:
13            branding_title = ''
14            agent_name = ''
15            agent_img = ''
16            property_img = ''
17            address_heading = ''
18            added = ''
19            price = ''
20            inspection = ''
21            property_type = ''
22            property_details = ''
23            sold_status = ''
24            sold_on = ''
25
26            item = {}
27            '----------------------------------------------------------------'
28            prop_content_details = prop.xpath(
29                './/*[contains(@class, "residential-card__content-wrapper")]')
30            prop_content_upper = prop.xpath(
31                './/*[contains(@class, "residential-card__banner-strip")]')
32
33            '----------------------------------------------------------------'
34            address_heading = prop_content_details.xpath(
35                './/*[contains(@class, "residential-card__address-heading")]//text()').extract_first()
36            residential_card_primary = prop_content_details.xpath(
37                './/*[contains(@class, "residential-card__primary")]')
38
39            price = prop_content_details.xpath(
40                './/*[contains(@class, "property-price")]//text()').extract_first()
41            property_type = residential_card_primary.xpath(
42                './p//text()').extract_first()
43            property_details = residential_card_primary.xpath(
44                './/li//@aria-label').extract()
45            property_details = ", ".join(property_details)
46
47            added = prop_content_upper.xpath(
48                './/*[contains(@class, "styles__Content")]//text()').extract()
49            added = [ad for ad in added if "Added" in str(ad)]
50            added = "".join(added)
51
52            inspection = prop_content_details.xpath(
53                './/*[contains(@class, "inspection__long-label")]//text()').extract_first()
54            branding_title = prop.xpath(
55                './/*[contains(@class, "branding__image")]//@alt').extract_first()
56            agent_name = prop.xpath(
57                './/*[contains(@class, "agent__name")]//text()').extract_first()
58            agent_img = prop.xpath(
59                './/*[contains(@class, "agent__name")]//following-sibling::img//@src').extract_first()
60
61            property_img = prop.xpath(
62                './/*[@class="property-image"]//@data-url').extract_first()
63
64            sold_status = prop_content_upper.xpath(
65                './/span[contains(@class, "styles__Content")]//text()').extract_first()
66            sold_on = prop_content_details.xpath(
67                './/span[contains(text(), "Sold on")]/text()').extract_first()
68
69            '----------------------------------------------------------------'
70            item['address_heading'] = address_heading
71            item['price'] = price
72            if search_type == "buy":
73                item['added'] = added
74                item['inspection'] = inspection
75            if search_type == "sold":
76                item['sold_status'] = sold_status
77                item['sold_on'] = sold_on
78            item['property_img'] = property_img
79            item['property_type'] = property_type
80            item['property_details'] = property_details
81            item['branding_title'] = branding_title
82            item['agent_name'] = agent_name
83            item['agent_img'] = agent_img
84            item['search_type'] = search_type
85
86            items.append(item)
87
88    except Exception as e:
89        print(f"Error: {e}")
90        return items
91
92    return items
93
94
95def generate_url(input_address, search_type="buy", page=1):
96    correct_address = input_address.replace(" ", "+")
97    url = f"https://www.realestate.com.au/{search_type}/in-{correct_address}/list-{page}"
98    return url
99
100
101def properties_pages_count(driver_source):
102    try:
103        response_ = Selector(text=str(driver_source))
104        return int(int(response_.xpath('//*[contains(@class, "StyledResultsCount")]//text()').extract_first().split()[-2])/25)
105    except Exception as e:
106        print(f"Error: {e}")
107        return 0

1import json
2import httpx
3from .all_config import (
4    API_KEY, SITEMAP_ID
5)
6
7BASE_URL = "https://api.webscraper.io/api/v1/scraping-job"
8
9
10def start_scraping(sitemap_id=SITEMAP_ID, url_=None):
11    """Start a scraping job for a given sitemap ID."""
12    url = f"{BASE_URL}?api_token={API_KEY}"  # Ensure API token is included in the URL
13
14    headers = {"Content-Type": "application/json",
15               "Accept": "application/json"}
16    payload = {
17        "sitemap_id": sitemap_id,
18        "driver": "fulljs",
19        "page_load_delay": 2000,
20        "request_interval": 1000,
21        "proxy": 1,
22        "start_urls": [
23            url_
24        ],
25    }
26
27    with httpx.Client() as client:
28        # Use json=payload instead of data=
29        response = client.post(url, headers=headers, json=payload)
30
31    if response.status_code == 200:
32        return response.json()
33    else:
34        print(
35            f"Error starting scraping: {response.status_code} - {response.text}")
36        return {}
37
38# start_scrape: {'success': True, 'data': {'id': 28731019, 'custom_id': None}}
39
40
41def get_scraping_status_by_job(job_id):
42    """Fetches the status of a scraping job."""
43    url = f"{BASE_URL}/{job_id}?api_token={API_KEY}"  # Job status endpoint
44
45    with httpx.Client() as client:
46        response = client.get(url)
47
48    if response.status_code == 200:
49        return response.json()
50    else:
51        print(
52            f"Error fetching status: {response.status_code} - {response.text}")
53        return {}
54
55
56# Scraping Job Status:
57# {'success': True, 'data': {'id': 28731019, 'custom_id': None,  'sitemap_id': 1274721, 'status': 'started',
58#  'test_run': 0, 'sitemap_name': 'realestate_au',
59#  'time_created': 1740759317, 'jobs_scheduled': 2,
60#  'jobs_executed': 29, 'jobs_failed': 0,
61#  'stored_record_count': 29, 'request_interval': 1000,
62#  'page_load_delay': 2000, 'driver': 'fulljs',
63#  'jobs_empty': 0, 'jobs_no_value': 0, 'scraping_duration': 207,
64#  'scheduled': 0}}
65
66
67def get_scraping_results(job_id):
68    """Fetches the scraped data of a completed job."""
69    url = f"{BASE_URL}/{job_id}/json?api_token={API_KEY}"  # Results endpoint
70
71    with httpx.Client() as client:
72        response = client.get(url)
73
74    if response.status_code == 200:
75        try:
76            # Capture the raw response text
77            data = response.text
78            print("Raw Response Text:")
79            # print(data)  # Print the raw response text
80
81            # Process each line in the response as an individual JSON object
82            results = []
83            for line in data.splitlines():
84                try:
85                    json_data = json.loads(line)  # Parse each line as JSON
86                    results.append(json_data)
87                except json.JSONDecodeError:
88                    print(f"Error decoding line: {line}")
89                    continue
90
91            print("Parsed JSON Data:")
92            # Print the parsed JSON data in a readable format
93            # print(json.dumps(results, indent=4))
94
95            # Save the parsed data to a file
96            with open("scraped_data.json", "w") as json_file:
97                json.dump(results, json_file, indent=4)
98
99            return results
100        except Exception as e:
101            print(f"Error processing response: {e}")
102            return {}
103
104    else:
105        print(f"Error fetching results: {response.status_code}")
106        return {}
107
108
109"""
110Method: POST
111URL: https://api.webscraper.io/api/v1/scraping-job?api_token=<YOUR API TOKEN>
112JSON:
113{
114	"sitemap_id": 123,
115	"driver": "fast", // "fast" or "fulljs"
116	"page_load_delay": 2000,
117	"request_interval": 2000,
118	"proxy": 0, // 0: No proxy, 1: Use proxy, 123: Custom proxy id, 'residential-*': Use residential proxy, replace * with country code, for example, 'residential-us'
119	"start_urls": [		// optional, if set, will overwrite sitemap start URLs
120		"https://www.webscraper.io/test-sites/e-commerce/allinone/computers",
121		"https://www.webscraper.io/test-sites/e-commerce/allinone/phones"
122	],
123	"custom_id": "custom-scraping-job-12" // optional, will be included in webhook notification
124}
125
126"""

1# Feel free to add your Python dependencies below. For formatting guidelines, see:
2# https://pip.pypa.io/en/latest/reference/requirements-file-format/
3
4apify < 3.0
5parsel
6httpx