src/routes.js

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

.eslintrc

# This file tells Git which files shouldn't be added to source control

.idea
node_modules


.gitignore

# First, specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-playwright-firefox:16

# Second, copy just package.json and package-lock.json since it should be
# the only file that affects "npm install" in the next step, to speed up the build
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
 && npm install --only=prod --no-optional \
 && echo "Installed NPM packages:" \
 && (npm list --only=prod --no-optional --all || true) \
 && echo "Node.js version:" \
 && node --version \
 && echo "NPM version:" \
 && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./

# Optionally, specify how to launch the source code of your actor.
# By default, Apify's base Docker images define the CMD instruction
# that runs the Node.js source code using the command specified
# in the "scripts.start" section of the package.json file.
# In short, the instruction looks something like this:
#
# CMD npm start


Dockerfile

Max items

The best option is usually Automatic proxy. But you can also use your own proxies or no proxy

Proxy configuration

Start URLs

Simple Realtor Scraper

INPUT_SCHEMA.json

{
     "env": { "npm_config_loglevel": "silent" }
}


apify.json

/* eslint-disable camelcase */
const Apify = require('apify');
const playwright = require('playwright');

const { utils: { log } } = Apify;

let detailsEnqueued = 0;

Apify.events.on('migrating', async () => {
    await Apify.setValue('detailsEnqueued', detailsEnqueued);
});

Apify.main(async () => {
    const { startUrls, maxItems = 10, proxyConfig } = await Apify.getInput();

    const requestList = await Apify.openRequestList('start-urls', startUrls);
    const requestQueue = await Apify.openRequestQueue();
    const proxyConfiguration = await Apify.createProxyConfiguration(proxyConfig);

    detailsEnqueued = await Apify.getValue('detailsEnqueued');
    if (!detailsEnqueued) {
        detailsEnqueued = 0;
    }

    const crawler = new Apify.PlaywrightCrawler({
        requestList,
        requestQueue,
        proxyConfiguration,
        launchContext: {
            launcher: playwright.firefox,
        },
        useSessionPool: true,
        maxConcurrency: 1,
        handlePageFunction: async (context) => {
            const { page } = context;
            const { url, userData: { label } } = context.request;
            log.info('Page opened.', { label, url });

            const text = await page.evaluate(() => {
                return document.querySelector('script[id="__NEXT_DATA__"]').textContent;
            });

            const { props: { pageProps: { totalProperties, properties } } } = JSON.parse(text);
            const resultsPerPage = 42;
            const pages = Math.ceil(totalProperties / resultsPerPage);
            const currentPage = url.match(/pg-(\d+)/) ? parseInt(url.match(/pg-(\d+)/)[1], 10) : 1;
            let nextUrl = `${url}/pg-2`;
            if (url.includes('/pg-')) {
                nextUrl = currentPage < pages ? url.replace(/pg-(\d+)/, `pg-${currentPage + 1}`) : null;
            }
            // eslint-disable-next-line max-len
            log.info(`totalResultCount: ${totalProperties}; resultsPerPage: ${resultsPerPage}; totalPages: ${pages}; nextUrl: ${nextUrl}`);

            for (let index = 0; index < properties.length; index++) {
                const { list_price, primary_photo, permalink, status, description: { beds, baths, sqft, lot_sqft, type },
                    location: { address: { line, postal_code, state, city,
                        coordinate }, county }, branding, list_date } = properties[index];
                await Apify.pushData({
                    url: `https://www.realtor.com/realestateandhomes-detail/${permalink}`,
                    imageUrl: primary_photo ? primary_photo.href : null,
                    price: list_price,
                    listingDate: list_date,
                    houseSqft: sqft,
                    lotSize: lot_sqft ? Math.round((lot_sqft / 43560) * 100) / 100 : null,
                    streetAddress: line,
                    zipcode: postal_code,
                    city,
                    state,
                    latitude: coordinate ? coordinate.lat : null,
                    longitude: coordinate ? coordinate.lon : null,
                    bedrooms: beds,
                    bathrooms: baths,
                    county: county ? county.name : null,
                    homeStatus: status,
                    homeType: type,
                    brokerName: branding[0].name,
                });

                if (maxItems && ++detailsEnqueued >= maxItems) {
                    return;
                }
            }

            if (nextUrl) {
                await requestQueue.addRequest({ url: nextUrl });
            }
        },
        preNavigationHooks: [async ({ page }, gotoOptions) => {
            gotoOptions.waitUntil = 'domcontentloaded';
            gotoOptions.timeout = 60000;

            await page.route('**/*', (route) => {
                return route.request().resourceType() === 'image'
                    ? route.abort()
                    : route.continue();
            });
        }],
    });

    log.info('Starting the crawl.');
    await crawler.run();
    log.info('Crawl finished.');
});


main.js

simple-realter-scraper

package.json

Simple Realtor scraper will enable you to extracting properties (ex: image url, price, address, latitude, longitude, bathrooms, bedrooms, home status, home type, broker name) from www.realtor.com very fast and consume less resources

simple-realtor-scraper

Simple Realtor scraper will enable you to extracting properties from www.realtor.com very fast and consume less resources

Where next?