For Sale By Owner Scraper

1import { Actor } from 'apify';
2import { launchPuppeteer, sleep, CheerioCrawler, log } from 'crawlee';
3import _ from 'lodash';
4await Actor.init();
5const results = {}
6// Create an instance of the CheerioCrawler class - a crawler
7// that automatically loads the URLs and parses their HTML using the cheerio library.
8const cheerioCrawler = new CheerioCrawler({
9    // The crawler downloads and processes the web pages in parallel, with a concurrency
10    // automatically managed based on the available system memory and CPU (see AutoscaledPool class).
11    // Here we define some hard limits for the concurrency.
12    minConcurrency: 10,
13    maxConcurrency: 50,
14
15    // On error, retry each page at most once.
16    maxRequestRetries: 3,
17
18    // Increase the timeout for processing of each page.
19    requestHandlerTimeoutSecs: 30,
20
21    // Limit to 10 requests per one crawl
22    maxRequestsPerCrawl: 1000,
23
24    // This function will be called for each URL to crawl.
25    // It accepts a single parameter, which is an object with options as:
26    // https://crawlee.dev/api/cheerio-crawler/interface/CheerioCrawlerOptions#requestHandler
27    // We use for demonstration only 2 of them:
28    // - request: an instance of the Request class with information such as the URL that is being crawled and HTTP method
29    // - $: the cheerio object containing parsed HTML
30    async requestHandler({ request, $ }) {
31        log.debug(`Processing ${request.url}...`);
32
33        // Store the results to the dataset. In local configuration,
34        // the data will be stored as JSON files in ./storage/datasets/default
35        await Actor.pushData({
36            url: request.url,
37            ...results[request.url],
38            seller: $('.modal-body div').eq(2).children().eq(1).text().trim(),
39            phone: $('.modal-body div').eq(2).children().eq(3).text().trim(),
40            price: $('.price').text().trim(),
41        });
42    },
43
44    // This function is called if the page processing failed more than maxRequestRetries + 1 times.
45    failedRequestHandler({ request }) {
46        log.debug(`Request ${request.url} failed twice.`);
47    },
48});
49
50const input = await Actor.getInput();
51const maxPages = input.maxPages || 1;
52console.log(`maxPages ${maxPages}`);
53if (!input.searchQueries?.length) {
54    Actor.abort('Input must contain at least one search query');
55}
56else {
57    // Launch the web browser.
58    const browser = await launchPuppeteer({
59        launchOptions: {
60            headless: false,
61            args: ['--no-sandbox'],
62        },
63    });
64    let res = [];
65
66    for (const searchQuery of input.searchQueries) {
67        console.log(searchQuery)
68
69        // Create and navigate new page
70        console.log('Open target page');
71        const page = await browser.newPage();
72        await page.goto('https://fsbo.com/listings/search/results/');
73
74        await page.type('.search-query', searchQuery);
75        await page.evaluate(() => {
76            document.querySelector("#RefineSearchDistance").value = '100';
77            document.querySelector("form .btn-fsbo-primary").click()
78        })
79        await sleep(3000)
80
81        // Submit the form and wait for full load of next page    
82
83        let hasNextPage = true;
84
85        while (hasNextPage) {
86            console.log('next page')
87            const currentPageResults = await page.evaluate(() => {
88                const result = [];
89                for (const listing of document.querySelectorAll('.listing-item')) {
90                    const askingPrice = listing.querySelector('.listing-right h4').textContent;
91                    const address1 = listing.querySelector('.listing-right').innerText.replace(askingPrice, '').split('\n')[1];
92                    const address2 = listing.querySelector('.listing-right').innerText.replace(askingPrice, '').split('\n')[2];
93
94                    const zipcode = address2.slice(-5);
95                    const city = address2.replace(zipcode, '').trim();
96                    result.push({
97                        url: listing.querySelector('a').href,
98                        askingPrice,
99                        address1,
100                        address2,
101                        city,
102                        zipcode,
103                        title: listing.querySelector('h4').textContent,
104                    });
105                }
106
107                return result;
108            });
109
110            hasNextPage = await page.evaluate(() => {
111                return !!document.querySelector('.nextPage');
112            });
113            console.log(`Current page: ${currentPageResults.length.length} | ${currentPageResults[0].url}`)
114            res = [...res, ...currentPageResults];
115            if (hasNextPage) {
116
117                console.log('clicking nextpage')
118                await page.click('.nextPage')
119                await sleep(5000)
120            }
121        }
122
123
124        res.forEach(r => {
125            results[r.url] = { ...r }
126        });
127        console.log(`All: ${res.length}`);
128        const allPages = _.uniq(res.map(r => r.url));
129        console.log(allPages)
130        console.log(`Unique Pages: ${allPages.length}`)
131        await cheerioCrawler.run(allPages);
132
133        // Store data in default dataset
134        // await Actor.pushData(currentPageResults);
135
136        // Close browser
137        await browser.close();
138    }
139    await Actor.exit();
140}

1import { Dataset, createPuppeteerRouter } from 'crawlee';
2
3export const router = createPuppeteerRouter();
4
5router.addDefaultHandler(async ({ enqueueLinks, log }) => {
6    log.info(`enqueueing new URLs`);
7    await enqueueLinks({
8        globs: ['https://www.zillow.com/ybor-city-tampa-fl/?searchQueryState=%7B%22isMapVisible%22%3Atrue%2C%22mapBounds%22%3A%7B%22north%22%3A27.98882307307199%2C%22south%22%3A27.957592306053694%2C%22east%22%3A-82.42739039666748%2C%22west%22%3A-82.46936160333252%7D%2C%22filterState%22%3A%7B%22sort%22%3A%7B%22value%22%3A%22globalrelevanceex%22%7D%2C%22ah%22%3A%7B%22value%22%3Atrue%7D%7D%2C%22isListVisible%22%3Atrue%2C%22mapZoom%22%3A15%2C%22regionSelection%22%3A%5B%7B%22regionId%22%3A126709%2C%22regionType%22%3A8%7D%5D%2C%22pagination%22%3A%7B%7D%7D'],
9        label: 'detail',
10    });
11});
12
13router.addHandler('detail', async ({ request, page, log }) => {
14    await page.type('input', 'tampa, fl');
15    await page.click('.btn-fsbo-primary');
16    await page.waitForNavigation();
17    const title = await page.title();
18    log.info(`${title}`, { url: request.loadedUrl });
19    const num = await page.evaluate(() => {
20        return document.querySelector(".detail-box-phone a").text;
21    })
22    const listings = document.querySelectorAll(".listing-item div h4").textContent;
23    for (let listing of listings) {
24        await Dataset.pushData({
25            url: listing,
26            // title,
27            // num,
28        });
29    }
30
31});