# Specify the base Docker image. You can read more about
# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent


.actor/Dockerfile

Crawlee and Cheerio project in javascript.

zillow-master-cheerio

Project Cheerio Crawler Javascript

.actor/actor.json

write Region for example 'Los Angeles County'

Region

Write State in which the above Region Lies, Only State Abbreviations, for example 'CA'

State

Zillow Input

.actor/input_schema.json

import axios from "axios";
export const CreateContact = async (data) => {
    const customeFields = [
        {
            id: "abnDCNFvV64zQiLJnAgG",
            name: "Photo URL",
            fieldKey: "contact.photo_url",
            placeholder: "",
            dataType: "TEXT",
            position: 250,
            key: "IMAGE1",
        },
        {
            id: "YZkPvdZJKuYrj6Svvmfa",
            name: "Agency Phone",
            fieldKey: "contact.agency_phone",
            placeholder: "",
            dataType: "PHONE",
            position: 300,
            key: "AGENCY NUMBER",
        },
        {
            id: "p0kHOiQA6tZIGKn6gMr5",
            name: "MLS ID",
            fieldKey: "contact.mls_id",
            placeholder: "",
            dataType: "TEXT",
            position: 350,
            key: "MLS ID",
        },
        {
            key: "PRICE",
            _id: "9OC07TKtu2CMXxOvZXXv",
            allowCustomOption: false,
            dataType: "MONETORY",
            dateAdded: "2023-05-08T17:50:05.599Z",
            documentType: "field",
            fieldKey: "contact.listing_price",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "VbDEA2dU7PA9u3ns6YjH",
            placeholder: "",
            position: 50,
            showInForms: true,
            standard: false,
            fieldsCount: 0,
            id: "9OC07TKtu2CMXxOvZXXv",
            name: "Listing Price",
        },
        {
            key: "ZESTIMATE",
            _id: "PkfhKCqkYqHkyq6E2beb",
            allowCustomOption: false,
            dataType: "MONETORY",
            dateAdded: "2023-05-08T17:51:02.012Z",
            documentType: "field",
            fieldKey: "contact.zestimate",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "VbDEA2dU7PA9u3ns6YjH",
            placeholder: "",
            position: 100,
            showInForms: true,
            standard: false,
            fieldsCount: 0,
            id: "PkfhKCqkYqHkyq6E2beb",
            name: "Zestimate",
        },
        {
            key: "PROPERTY TYPE",
            _id: "iUlZWl8BcZQOpftOprT7",
            allowCustomOption: false,
            dataType: "TEXT",
            dateAdded: "2023-05-08T17:51:45.544Z",
            documentType: "field",
            fieldKey: "contact.property_type",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "VbDEA2dU7PA9u3ns6YjH",
            placeholder: "",
            position: 150,
            showInForms: true,
            standard: false,
            fieldsCount: 0,
            id: "iUlZWl8BcZQOpftOprT7",
            name: "Property Type",
        },
        {
            key: "URLS",
            _id: "lKYgi4KmNEWFHl6rURte",
            allowCustomOption: false,
            dataType: "TEXT",
            dateAdded: "2023-05-08T17:53:15.700Z",
            documentType: "field",
            fieldKey: "contact.listing_url",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "VbDEA2dU7PA9u3ns6YjH",
            placeholder: "",
            position: 200,
            showInForms: true,
            standard: false,
            fieldsCount: 0,
            id: "lKYgi4KmNEWFHl6rURte",
            name: "Listing URL",
        },
        {
            key: "AGENCY",
            _id: "bq2UYHymnk9iv41zT1zd",
            dataType: "STANDARD_FIELD",
            dateAdded: "2023-03-16T15:03:18.353Z",
            documentType: "field",
            fieldKey: "contact.company_name",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "fjy3WwXH4i6sySPMkCfO",
            position: 0,
            standard: true,
            fieldsCount: 0,
            id: "bq2UYHymnk9iv41zT1zd",
            name: "Business Name",
        },
        {
            key: "STREET ADDRESS",
            _id: "aQ6494pEqJs4oWtD2Fpd",
            dataType: "STANDARD_FIELD",
            dateAdded: "2023-03-16T15:03:18.355Z",
            documentType: "field",
            fieldKey: "contact.address1",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "fjy3WwXH4i6sySPMkCfO",
            position: 50,
            standard: true,
            fieldsCount: 0,
            id: "aQ6494pEqJs4oWtD2Fpd",
            name: "Street Address",
        },
        {
            key: "CITY",
            _id: "V3TNBdcFoasEMXpYHZuJ",
            dataType: "STANDARD_FIELD",
            dateAdded: "2023-03-16T15:03:18.355Z",
            documentType: "field",
            fieldKey: "contact.city",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "fjy3WwXH4i6sySPMkCfO",
            position: 100,
            standard: true,
            fieldsCount: 0,
            id: "V3TNBdcFoasEMXpYHZuJ",
            name: "City",
        },
        {
            key: "COUNTRY",
            _id: "GJEUQTEY4Zn3TJyDGneg",
            dataType: "STANDARD_FIELD",
            dateAdded: "2023-03-16T15:03:18.355Z",
            documentType: "field",
            fieldKey: "contact.country",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "fjy3WwXH4i6sySPMkCfO",
            position: 150,
            standard: true,
            fieldsCount: 0,
            id: "GJEUQTEY4Zn3TJyDGneg",
            name: "Country",
        },
        {
            key: "STATE",
            _id: "XgcWvU6Q8hTqXaC4LC9J",
            dataType: "STANDARD_FIELD",
            dateAdded: "2023-03-16T15:03:18.355Z",
            documentType: "field",
            fieldKey: "contact.state",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "fjy3WwXH4i6sySPMkCfO",
            position: 200,
            standard: true,
            fieldsCount: 0,
            id: "XgcWvU6Q8hTqXaC4LC9J",
            name: "State",
        },
        {
            key: "ZIP CODE",
            _id: "Qxi9FEaBI5AHLQF2IQOn",
            dataType: "STANDARD_FIELD",
            dateAdded: "2023-03-16T15:03:18.355Z",
            documentType: "field",
            fieldKey: "contact.postal_code",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "fjy3WwXH4i6sySPMkCfO",
            position: 250,
            standard: true,
            fieldsCount: 0,
            id: "Qxi9FEaBI5AHLQF2IQOn",
            name: "Postal Code",
        },
        // {key:"Zillo",
        //     _id: "RjEaVVEFOvRdOBbtWSBe",
        //     dataType: "STANDARD_FIELD",
        //     dateAdded: "2023-03-16T15:03:18.355Z",
        //     documentType: "field",
        //     fieldKey: "contact.website",
        //     locationId: "uBJYmtdHkVFZ1S1oeG3Y",
        //     model: "contact",
        //     parentId: "fjy3WwXH4i6sySPMkCfO",
        //     position: 300,
        //     standard: true,
        //     fieldsCount: 0,
        //     id: "RjEaVVEFOvRdOBbtWSBe",
        //     name: "Website",
        // },
        // {
        //     _id: "1CNSMgx3kEzemFLSgzAW",
        //     dataType: "STANDARD_FIELD",
        //     dateAdded: "2023-03-16T15:03:18.355Z",
        //     documentType: "field",
        //     fieldKey: "contact.timezone",
        //     locationId: "uBJYmtdHkVFZ1S1oeG3Y",
        //     model: "contact",
        //     parentId: "fjy3WwXH4i6sySPMkCfO",
        //     position: 350,
        //     standard: true,
        //     fieldsCount: 0,
        //     id: "1CNSMgx3kEzemFLSgzAW",
        //     name: "Time Zone",
        // },
        {
            key: "COUNTY",
            _id: "xLs2EA0Qz8SxTcpJAQbh",
            allowCustomOption: false,
            dataType: "TEXT",
            dateAdded: "2023-05-08T17:48:38.169Z",
            documentType: "field",
            fieldKey: "contact.county",
            locationId: "uBJYmtdHkVFZ1S1oeG3Y",
            model: "contact",
            parentId: "fjy3WwXH4i6sySPMkCfO",
            placeholder: "",
            position: 400,
            showInForms: true,
            standard: false,
            fieldsCount: 0,
            id: "xLs2EA0Qz8SxTcpJAQbh",
            name: "County",
        },
    ];
    let updated_Data = {
        email:
            data?.["AGENT EMAIL"] && data?.["AGENT EMAIL"] != "--"
                ? data?.["AGENT EMAIL"]
                : data?.["AGENCY EMAIL"] && data?.["AGENCY EMAIL"] != "--"
                ? data?.["AGENCY EMAIL"]
                : undefined,
        phone:
            data?.["AGENT NUMBER"] && data?.["AGENT NUMBER"] != "--"
                ? data?.["AGENT NUMBER"]
                : data?.["SECONDARY AGENCY NUMBER"] &&
                  data?.["SECONDARY AGENCY NUMBER"] != "--"
                ? data?.["SECONDARY AGENCY NUMBER"]
                : undefined,
        name: data?.["AGENT"],
        companyName: data?.["AGENCY"],
        address1:
            data?.["STREET ADDRESS"] && data?.["STREET ADDRESS"] != "--"
                ? data?.["STREET ADDRESS"]
                : undefined,
        city:
            data?.["CITY"] && data?.["CITY"] != "--"
                ? data?.["CITY"]
                : undefined,
        state:
            data?.["STATE"] && data?.["STATE"] != "--"
                ? data?.["STATE"]
                : undefined,

        postalCode:
            data?.["ZIP CODE"] && data?.["ZIP CODE"] != "--"
                ? data?.["ZIP CODE"]
                : undefined,
        tags: ["Zillow"],
        source: "Apify",

        customField: (() => {
            let obj = {};
            customeFields.forEach((cus) => {
                obj[cus.id] = data?.[cus.key];
            });
            return obj;
        })(),
    };
    const config = {
        method: "post",
        maxBodyLength: Infinity,
        url: "https://rest.gohighlevel.com/v1/contacts/",
        headers: {
            Authorization:
                "Bearer eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJsb2NhdGlvbl9pZCI6InVCSlltdGRIa1ZGWjFTMW9lRzNZIiwiY29tcGFueV9pZCI6IlRKZUk5ajh1enVRRFcxakRQQUFTIiwidmVyc2lvbiI6MSwiaWF0IjoxNjgzODA4MjQ1NTQ4LCJzdWIiOiIzVXZhNm5rWTE0RHM0VjhUa3RGWCJ9.xkrJxCrTwgGrniGwZNGShcZYSObw8wrOv1Ok_jOq6IM",
            "content-type": "application/json",
        },
        data: JSON.stringify(updated_Data),
    };

    return await axios(config)
        .then(function (response) {
            // console.log(JSON.stringify(response.data));
        })
        .catch(function (error) {
            console.log(error.response.data);
        });
};


src/goHighLevel.js

export const removeEmpty = (obj) => {
    return Object.fromEntries(
        Object.entries(obj)
            .filter(([_, v]) => v != null)
            .map(([k, v]) => [k, v === Object(v) ? removeEmpty(v) : v])
    );
};
export const title = [
    "MLS ID",
    "ZILLOW ID",
    "VIEW COUNT",
    "SAVE COUNT",
    "PRICE",
    "DESCRIPTION",
    "DESCRIPTION COUNT",
    "SUB TYPE",
    "TIME ON ZILLOW",
    "YEAR BUILT",
    "LAST SOLD PRICE",
    "LAST SOLD DATE",
    "PREVIOUS SOLD DATE",
    "PREVIOUS SOLD PRICE",
    "ZESTIMATE",
    "STREET ADDRESS",
    "ADDRESS",
    "BEDS",
    "BATHS",
    "PROPERTY STATUS",
    "PROPERTY TYPE",
    "LIVING AREA",
    "LOT AREA ACRES",
    "LOT AREA SQFT",
    "CITY_STATES",
    "AVERAGE",
    "CITY",
    "COUNTY",
    "ZIP CODE",
    "STATE",
    "AGENT NUMBER",
    "SECONDARY AGENT NUMBER",
    "AGENT",
    "AGENT EMAIL",
    "AGENCY",
    "AGENCY NUMBER",
    "AGENCY EMAIL",
    "POSTING DATE",
    "DAYS ON ZILLOW",
    "URLS",
    "IMAGE1",
    "IMAGE2",
    "IMAGE3",
    "IMAGE4",
    "IMAGE5",
    "IMAGE6",
    "IMAGE7",
    "IMAGE8",
    "IMAGE9",
    "IMAGE10",
    "SEARCH URL",
];
// token=apify_api_Hf41dWEuugrEK0aQ1QN3eEWq8faxtt04ogjG

src/helper.js

/**
 * Welcome to Zillow Scraper Made by Moazzam Malek (moazzamalek@gmail.com)
 * The tool is created at Pimatrix System, LLC
 * All Rights are reserved by Pimatrix System, LLC
 * For any query: Contact@pimatrix.in
 * As per custom request, This Tool was given to :
 * Steve (steve.eskander@gmail.com) at 9 MAY 2023
 */

// For more information, see https://docs.apify.com/sdk/js
import { Actor } from "apify";
// For more information, see https://crawlee.dev
import { CheerioCrawler, Dataset, KeyValueStore } from "crawlee";
import { router } from "./routes.js";
import { removeEmpty } from "./helper.js";

// Initialize the Apify SDK
await Actor.init();

let startUrls = [];

const params = { region: "Los Angeles County", state: "CA" };
if (Actor.isAtHome()) {
    try {
        const data = await Actor.getInput();
        if (data.region) params.region = data.region;
        if (data.state) params.state = data.state;
    } catch (error) {}
}
const INPUT = [{ RegionName: params.region, StateName: params.state }];
const {
    maxBuiltYear,
    minBuiltYear,
    maxPrice,
    minPrice,
    daysOnZillow,
    filterFromDOZ,
    filterToDOZ,
    otherDaysOnZillow,
    otherFilterFromDOZ,
    otherFilterToDOZ,
    view,
    amenities,
    basementFinished,
    basementUnfinished,
    minLotSqft,
    maxLotSqft,
    minLivingSqft,
    maxLivingSqft,
    minBeds,
    maxBeds,
    minBath,
    maxBath,
    hasGarage,
    maxHoa,
    homeType,
    forSaleorSold,
    FSBAorFSBO,
    notSoldFromYear,
    includeImages,
    excludeRegion,
    excludeState,
    forSaleType,
    tours,
    singleStoryOnly,
    seniorLiving,
    minParkingSpot,
} = {
    maxBuiltYear: "",
    minBuiltYear: "",
    maxPrice: "",
    minPrice: "",
    daysOnZillow: "1",
    filterFromDOZ: "",
    filterToDOZ: "",
    otherDaysOnZillow: "",
    otherFilterFromDOZ: "",
    otherFilterToDOZ: "",
    view: "",
    amenities: "",
    basementFinished: "",
    basementUnfinished: "",
    minLotSqft: "",
    maxLotSqft: "",
    minLivingSqft: "",
    maxLivingSqft: "",
    minBeds: "",
    maxBeds: "",
    minBath: "",
    maxBath: "",
    hasGarage: "",
    maxHoa: "",
    homeType: [
        "TOWNHOMES",
        "MANUFACTURED",
        "APARTMENTS",
        "CONDO",
        "MULTI",
        "SINGLE",
        // 'LOTS'
    ].toString(),
    forSaleorSold: "forSale",
    FSBAorFSBO: "",
    notSoldFromYear: "",
    includeImages: "",
    excludeRegion: "",
    excludeState: "",
    forSaleType: [
        "By Agent",
        // "By Owner"
    ].toString(),
    tours: "",
    singleStoryOnly: "",
    seniorLiving: "",
    minParkingSpot: "",
};

const filterState = (() => {
    const sort = { value: "globalrelevanceex" };
    const ah = { value: true };

    const price =
        maxPrice || minPrice
            ? {
                  min: parseFloat(minPrice) > 0 ? minPrice : null,
                  max: parseFloat(maxPrice) > 0 ? maxPrice : null,
              }
            : null;
    const sf = { value: homeType.includes("SINGLE") ? true : false };
    const hoa = maxHoa
        ? {
              max: parseFloat(maxHoa) > 0 ? maxHoa : null,
          }
        : null;
    const mf = { value: homeType.includes("MULTI") ? true : false };
    const con = { value: homeType.includes("CONDO") ? true : false };
    const apco = { value: homeType.includes("APARTMENTS") ? true : false };
    const land = { value: homeType.includes("LOTS") ? true : false };
    const apa = { value: homeType.includes("APARTMENTS") ? true : false };
    const manu = {
        value: homeType.includes("MANUFACTURED") ? true : false,
    };

    const tow = { value: homeType.includes("TOWNHOMES") ? true : false };
    const built =
        maxBuiltYear || minBuiltYear
            ? {
                  min: parseFloat(minBuiltYear) > 0 ? minBuiltYear : null,
                  max: parseFloat(maxBuiltYear) > 0 ? maxBuiltYear : null,
              }
            : null;

    const basf = basementFinished == "Yes" ? { value: true } : null;
    const basu = basementUnfinished == "Yes" ? { value: true } : null;
    const lot =
        parseFloat(minLotSqft) > 0 || parseFloat(maxLotSqft) > 0
            ? {
                  min: parseFloat(minLotSqft) > 0 ? minLotSqft : null,
                  max: parseFloat(maxLotSqft) > 0 ? maxLotSqft : null,
              }
            : null;
    const sqft =
        parseFloat(maxLivingSqft) > 0 || parseFloat(maxLivingSqft) > 0
            ? {
                  min: parseFloat(minLivingSqft) > 0 ? minLivingSqft : null,
                  max: parseFloat(maxLivingSqft) > 0 ? maxLivingSqft : null,
              }
            : null;

    const beds =
        parseFloat(maxBeds) > 0 || parseFloat(minBeds) > 0
            ? {
                  min: parseFloat(minBeds) > 0 ? minBeds : null,
                  max: parseFloat(maxBeds) > 0 ? maxBeds : null,
              }
            : null;

    const baths =
        parseFloat(maxBath) > 0 || parseFloat(minBath) > 0
            ? {
                  min: parseFloat(minBath) > 0 ? minBath : null,
                  max: parseFloat(maxBath) > 0 ? maxBath : null,
              }
            : null;

    const gar = hasGarage == "Yes" ? { value: true } : null;

    const fsba = {
        value: forSaleType?.includes("By Agent") ? true : false,
    };
    const fsbo = {
        value: forSaleType?.includes("By Owner") ? true : false,
    };
    const nc = {
        value: forSaleType?.includes("New Construction") ? true : false,
    };
    const cmsn = {
        value: forSaleType?.includes("Coming Soon") ? true : false,
    };
    const auc = { value: forSaleType?.includes("Auction") ? true : false };
    const fore = {
        value: forSaleType?.includes("Foreclosed") ? true : false,
    };
    const pf = {
        value: forSaleType?.includes(" Foreclosures") ? true : false,
    };
    const pmf = {
        value: forSaleType?.includes("Pre-Foreclosures") ? true : false,
    };
    const abo = {
        value: forSaleType?.includes("Accepting Backup Offers") ? true : false,
    };
    const pnd = {
        value: forSaleType?.includes("Pending & Under Contract") ? true : false,
    };
    const ac = { value: amenities?.includes("A/C") ? true : false };
    const pool = { value: amenities?.includes("Pool") ? true : false };
    const wat = { value: amenities?.includes("Water") ? true : false };
    const watv = { value: view?.includes("Water") ? true : false };
    const parkv = { value: view?.includes("Park") ? true : false };
    const mouv = { value: view?.includes("Mountain") ? true : false };
    const cityv = { value: view?.includes("City") ? true : false };

    const open = {
        value: tours?.includes("Open House") ? true : false,
    };
    const _3d = {
        value: tours?.includes("3D Tour") ? true : false,
    };
    const parks =
        parseFloat(minParkingSpot) > 0
            ? {
                  min: parseFloat(minParkingSpot) > 0 ? minParkingSpot : null,
              }
            : null;
    const sto = {
        value: singleStoryOnly ? true : false,
    };
    const _55plus = {
        value: seniorLiving ? true : false,
    };
    const filterState = {
        wat,
        ac,
        open,
        "3d": _3d,
        "55plus": _55plus,
        sto,
        parks,
        pnd,
        abo,
        pmf,
        pf,
        fore,
        auc,
        cmsn,
        nc,
        fsba,
        fsbo,
        sort,
        ah,
        hoa,
        price,
        mf,
        sf,
        con,
        apco,
        land,
        apa,
        manu,
        built,
        tow,
        doz: { value: daysOnZillow },
        cityv,
        mouv,
        parkv,
        watv,
        pool,
        basf,
        basu,
        lot,
        sqft,
        beds,
        baths,
        gar,
    };
    return filterState;
})();
if (forSaleorSold.includes("forSale") && forSaleType?.includes("By Agent"))
    startUrls = [
        ...INPUT.map((record) => {
            return {
                userData: {
                    RegionName: record["RegionName"],
                    CAT2: false,
                },
                url: `https://www.zillow.com/homes/${encodeURI(
                    `${record["RegionName"]}, ${record["StateName"]}`
                )}/?searchQueryState={"pagination":{"currentPage":1},"isMapVisible":false,"usersSearchTerm":"${encodeURI(
                    `${record["RegionName"]}, ${record["StateName"]}`
                )}","filterState":${JSON.stringify(
                    removeEmpty(filterState)
                )},"isListVisible":true,"mapZoom":7}`,
            };
        }),
        ...startUrls,
    ];
const proxyConfiguration = await Actor.createProxyConfiguration({
    // proxyUrls: [
    //     "http://moazzamalek:n3L8tiZLIcDBL7yy@proxy.packetstream.io:31112",
    // ],
    // useApifyProxy: false,
});
export const thisDatasetId = "zillow-" + new Date().getTime();
export const requestQueue = await Actor.openRequestQueue(
    process.env.APIFY_DEFAULT_REQUEST_QUEUE_ID
        ? process.env.APIFY_DEFAULT_REQUEST_QUEUE_ID
        : "zillow-" + new Date().getTime()
);
let datasetId = Actor.getEnv()?.actorTaskId
    ? Actor.getEnv()?.actorTaskId
    : Actor.getEnv()?.actorId
    ? Actor.getEnv()?.actorId
    : "".concat(
          JSON.stringify(params.region + params.state)
              .replaceAll(" ", "-")
              .replaceAll(".", "-")
              .replaceAll('"', "")
              .replaceAll(",", "-")
      );
console.log({ datasetId });
export const dataset = await Actor.openDataset(datasetId, { forceCloud: true });
export const datasetData = await (await dataset.getData()).items;
const crawler = new CheerioCrawler({
    proxyConfiguration,
    ignoreSslErrors: true,
    useSessionPool: false,
    requestHandler: router,
    maxConcurrency: 5,
    maxRequestRetries: 10,
    navigationTimeoutSecs: 120,
    requestHandlerTimeoutSecs: 120,
    requestQueue: requestQueue,
});

await crawler.run(startUrls);
const data = (await (await Dataset.open(thisDatasetId)).getData()).items;

let haveOldData = false;
if (datasetData?.length) haveOldData = true;
let output = [];
data?.forEach((dt) => {
    if (
        !datasetData?.find((ddt) => ddt["ZILLOW ID"] == dt["ZILLOW ID"])?.[
            "ZILLOW ID"
        ]
    ) {
        output.push(dt);
    }
});
output = [...new Map(output.map((item) => [item["ZILLOW ID"], item])).values()];
await KeyValueStore.setValue("output", output);
await dataset.pushData(output);
await Dataset.pushData(output);
await Dataset.exportToCSV("csv");
// Exit successfully
await Actor.exit();


src/main.js

import {
    Dataset,
    KeyValueStore,
    createCheerioRouter,
    LoggerText,
    log,
} from "crawlee";
import { thisDatasetId, datasetData } from "./main.js";
import { CreateContact } from "./goHighLevel.js";
log.child({ prefix: "TestCrawlee" }).setOptions({
    logger: new LoggerText({ skipTime: false }),
});
log.setLevel(log.LEVELS.OFF);

export const router = createCheerioRouter();
let totalListings = 0;

router.addDefaultHandler(async ({ request, $, enqueueLinks, log, crawler }) => {
    if (request.loadedUrl?.includes("captcha")) {
        throw "Got Captcha will try again with another IP";
    }
    try {
        // console.console.log("Hello Start Here!")
        // console.log(proxyInfo.url);
        console.log({ title: $("title").text() });

        const body = $(
            'script[data-zrr-shared-data-key="mobileSearchPageStore"]'
        ).html();
        const requrl = decodeURIComponent(request.url).split("=")[0];
        const reqfilter = JSON.parse(
            decodeURIComponent(request.url).split("=")[1]
        );
        let json;
        try {
            const rawJson = body.match(//)[1];
            json = JSON.parse(rawJson);
        } catch (error) {
            json = JSON.parse($("#__NEXT_DATA__").html()).props.pageProps;
        }

        const totalListing =
            json["categoryTotals"][request?.userData?.CAT2 ? "cat2" : "cat1"][
                "totalResultCount"
            ];
        totalListings = totalListings + totalListing;
        await (await KeyValueStore).setValue("totalListing", totalListings);
        const totalPage = Math.min(Math.ceil(totalListing / 40), 20);
        if (Math.ceil(totalListing / 40) > 20)
            console.warn(
                "PLEASE ADD MORE FILTERS WE ARE MISSING OUT ON RESULTS"
            );
        console.log(
            `Total Lisitng Found : ${totalListing} \n Total Page : ${totalPage}`
        );
        if (totalPage >= 1) {
            const body = $(
                'script[data-zrr-shared-data-key="mobileSearchPageStore"]'
            ).html();
            let json;
            try {
                const rawJson = body.match(//)[1];
                json = JSON.parse(rawJson);
            } catch (error) {
                json = JSON.parse($("#__NEXT_DATA__").html()).props.pageProps;
            }
            const listing =
                json[request?.userData?.CAT2 ? "cat2" : "cat1"][
                    "searchResults"
                ]["listResults"];
            // console.log(listing);
            // console.log(JSON.stringify(listing[0]));

            if (listing.length != 0) {
                for (let i = 0; i < listing.length; i++) {
                    let list = listing[i];
                    let notToSkip = true;
                    if (
                        datasetData?.find(
                            (dt) =>
                                dt["ZILLOW ID"] ==
                                (list.zpid ? parseInt(list.zpid) : "")
                        )?.["ZILLOW ID"]
                    )
                        notToSkip = false;

                    let url = list["detailUrl"];
                    console.log(`Requesting Detail Page: ${url}`);
                    if (notToSkip)
                        await crawler.requestQueue.addRequest({
                            url,
                            userData: {
                                search: request.url,
                                label: "DETAIL",
                                data: list,
                            },
                        });
                }
            }
        }
        for (let i = 2; i <= totalPage; i++) {
            reqfilter.pagination = { currentPage: i };
            let requestUrl =
                requrl +
                "=" +
                JSON.stringify(reqfilter) +
                "&requestId=" +
                (i + 1);
            // console.log(`Requesting Pagination : ${requestUrl}`);
            await crawler.requestQueue.addRequest({
                url: requestUrl,
                userData: {
                    label: "PAGINATION",
                    CAT2: request?.userData?.CAT2,
                },
            });
        }
        // session.markGood();
    } catch (error) {
        await (
            await Dataset.open("error")
        ).pushData({
            url: request.url,
            loadedurl: request.loadedUrl,
            error: error?.message ? error?.message : error,
        });
        throw error;
    }
});

router.addHandler("PAGINATION", async ({ request, $, crawler, log }) => {
    if (request.loadedUrl?.includes("captcha")) {
        throw "Got Captcha will try again with another IP";
    }
    try {
        // console.console.log("Hello Start Here!")
        console.log({ title: $("title").text() });
        // console.log(proxyInfo.url);
        const body = $(
            'script[data-zrr-shared-data-key="mobileSearchPageStore"]'
        ).html();
        let json;
        try {
            const rawJson = body.match(//)[1];
            json = JSON.parse(rawJson);
        } catch (error) {
            json = JSON.parse($("#__NEXT_DATA__").html()).props.pageProps;
        }
        const listing =
            json[request?.userData?.CAT2 ? "cat2" : "cat1"]["searchResults"][
                "listResults"
            ];
        // console.log(listing);
        // console.log(JSON.stringify(listing[0]));

        if (listing.length != 0) {
            for (let i = 0; i < listing.length; i++) {
                let list = listing[i];
                let notToSkip = true;
                if (
                    datasetData?.find(
                        (dt) =>
                            dt["ZILLOW ID"] ==
                            (list.zpid ? parseInt(list.zpid) : "")
                    )?.["ZILLOW ID"]
                )
                    notToSkip = false;

                let url = list["detailUrl"];
                console.log(`Requesting Detail Page: ${url}`);
                if (notToSkip)
                    await crawler.requestQueue.addRequest({
                        url,
                        userData: {
                            search: request.url,
                            label: "DETAIL",
                            data: list,
                        },
                    });
            }
        }
        // session.markGood();
    } catch (error) {
        await (
            await Dataset.open("error")
        ).pushData({
            url: request.url,
            loadedurl: request.loadedUrl,
            error: error?.message ? error?.message : error,
        });
        throw error;
    }
});
router.addHandler("DETAIL", async ({ request, $, log }) => {
    if (request.loadedUrl?.includes("captcha")) {
        throw "Got Captcha will try again with another IP";
    }
    try {
        // console.log({ title: $("title").text() });

        // console.log(proxyInfo.url);
        if ($("title").text()) {
            // session.markBad();
        }

        let jData = JSON.parse($("#hdpApolloPreloadedData").html());

        try {
            jData = JSON.parse(jData["apiCache"]);
            const objK = Object.keys(jData);
            jData = jData[objK[1]];
            jData = jData["property"];
        } catch (error) {
            jData = JSON.parse(
                JSON.parse($("#__NEXT_DATA__").html()).props.pageProps
                    .gdpClientCache
            )[
                Object.keys(
                    JSON.parse(
                        JSON.parse($("#__NEXT_DATA__").html()).props.pageProps
                            .gdpClientCache
                    )
                )[0]
            ].property;
        }
        // console.log(JSON.stringify(jData));
        // await fs.writeFile(
        //     './data.json',
        //     JSON.stringify(jData),
        //     function (err, data) {
        //         if (err) {
        //             return console.log(err);
        //         }
        //         // console.log(data);
        //     }
        // );
        const data = request.userData.data;

        let arr = {};
        arr["URLS"] = "https://www.zillow.com" + jData["hdpUrl"];
        arr["SEARCH URL"] = request.userData.search;
        arr["ZILLOW ID"] = jData["zpid"];
        arr["MLS ID"] = jData["mlsid"];
        arr["VIEW COUNT"] = jData["pageViewCount"];
        arr["SAVE COUNT"] = jData["favoriteCount"];
        arr["PRICE"] = data["price"];
        if (arr["PRICE"]) {
            arr["PRICE"] = arr["PRICE"]
                ?.replaceAll(",", "")
                ?.replaceAll("$", "");
            arr["PRICE"] = arr["PRICE"]?.replaceAll("K", "000");
        }

        arr["DESCRIPTION"] = jData["description"];
        arr["TIME ON ZILLOW"] = jData["timeOnZillow"];
        arr["YEAR BUILT"] = jData["yearBuilt"];
        // /LAST SOLD DATE
        if (jData["lastSoldPrice"] > 0) {
            let i = 0;
            const priceHistory = jData["priceHistory"]?.filter((history) => {
                return history["event"] == "Sold";
            });
            // consloe.log(JSON.stringify(priceHistory));
            if (priceHistory[0]?.price && priceHistory[0]?.date) {
                arr["LAST SOLD PRICE"] = priceHistory[0].price;
                arr["LAST SOLD DATE"] = priceHistory[0].date;
            }
            if (priceHistory[1]?.price && priceHistory[1]?.date) {
                arr["PREVIOUS SOLD PRICE"] = priceHistory[1].price;
                arr["PREVIOUS SOLD DATE"] = priceHistory[1].date;
            }
        }

        arr["ZESTIMATE"] = jData["zestimate"];
        arr["ADDRESS"] = data["address"];
        arr["STREET ADDRESS"] = data["addressStreet"];
        arr["COUNTY"] = jData["county"];
        // arr['County'] = jData['county']?.replaceAll('County', '');
        arr["CITY"] = data["addressCity"];
        arr["STATE"] = data["addressState"];
        arr["ZIP CODE"] = data["addressZipcode"];
        arr["BEDS"] = data["beds"];
        arr["BATHS"] = data["baths"];
        arr["PROPERTY STATUS"] = jData["homeStatus"];
        // arr['Status'] = data['hdpData']['homeInfo']['homeStatus']?.replaceAll(
        //     '_',
        //     ' '
        // );
        arr["PROPERTY TYPE"] = jData["homeType"];
        // arr['Property Type'] = data['hdpData']['homeInfo']['homeType']?.replaceAll(
        //     '_',
        //     ' '
        // );
        const sub_type = jData["listing_sub_type"];
        // console.log(sub_type);
        try {
            const SUB_TYPE = Object.keys(sub_type)?.map((key) => {
                return { name: key, value: sub_type[key] };
            });
            // console.log(SUB_TYPE);
            arr["SUB TYPE"] = SUB_TYPE.filter((type) => type.value)
                .map((t) => t.name.replace("is_", ""))
                .toString();
        } catch (error) {
            console.log(error);
        }
        arr["LIVING AREA"] = jData["livingArea"];
        // arr['SqFt'] = data['area'];
        arr["LOT AREA ACRES"] = jData["lotAreaValue"];

        arr["LOT AREA SQFT"] = jData["lotSize"];
        arr["POSTING DATE"] = jData["datePostedString"];
        arr["DAYS ON ZILLOW"] = jData["daysOnZillow"];
        arr["DAYS ON ZILLOW"] = arr["DAYS ON ZILLOW"]?.toString();
        // let features = jData['resoFacts']['atAGlanceFacts'];
        // const valid = ['Heating', 'Cooling', 'Parking', 'Lot', 'Price/sqft'];
        // for (let feature of features) {
        //     if (valid.includes(feature['factLabel'])) {
        //         arr[feature['factLabel']] = feature['factValue'];
        //         if (feature['factValue']) {
        //             arr[feature['factLabel']] = feature['factValue']
        //                 ?.replaceAll(',', '')
        //                 ?.replaceAll('$', '');
        //         }
        //     }
        // }
        try {
            arr["AGENCY"] = jData["attributionInfo"]["brokerName"];
            arr["AGENT"] = jData["attributionInfo"]["agentName"];
            arr["AGENT NUMBER"] = jData["attributionInfo"]["agentPhoneNumber"];
            if (!arr["AGENT NUMBER"])
                arr["AGENT NUMBER"] = jData?.contactFormRenderData?.data
                    ?.agent_module?.phone?.number
                    ? jData?.contactFormRenderData?.data?.agent_module?.phone
                          ?.areacode +
                      jData?.contactFormRenderData?.data?.agent_module?.phone
                          ?.prefix +
                      jData?.contactFormRenderData?.data?.agent_module?.phone
                          ?.number
                    : "";
            if (arr["AGENT NUMBER"].includes("@")) {
                arr["AGENT EMAIL"] =
                    jData["attributionInfo"]["agentPhoneNumber"];
                arr["AGENT NUMBER"] = "--";
            }
            arr["AGENT EMAIL"] = jData["attributionInfo"]["agentEmail"]
                ? jData["attributionInfo"]["agentEmail"]
                : "--";
            arr["SECONDARY AGENT NUMBER"] =
                jData["attributionInfo"]["coAgentNumber"];
            const BrokerContact =
                jData["attributionInfo"]["brokerPhoneNumber"].split(";");
            BrokerContact.forEach((b) => {
                if (b.includes("@")) {
                    arr["AGENCY EMAIL"] = b;
                } else {
                    arr["AGENCY NUMBER"] = b;
                }
            });
            // let fullName = jData['attributionInfo']['agentName'];
            // if (fullName) {
            //     fullName = fullName.trim();
            // }
            // fullName = fullName.split('-')[0];
            // arr['Full Name'] = fullName;
            // if (arr['Full Name'].split(' ').length == 2) {
            //     let firstName = fullName.split(' ')[0];
            //     let lastName = fullName.split(' ')[1];
            //     // if(firstName.trim() && lastName.trim() == ''){
            //     //     lastName = fullName?.replaceAll(firstName,'').trim()
            //     // }
            //     arr['First Name'] = firstName;
            //     arr['Last Name'] = lastName;
            // }
            // if (arr['Full Name'].split(' ').length == 3) {
            //     arr['First Name'] = arr['Full Name'].split(' ')[0];
            //     arr['Last Name'] = arr['Full Name'].split(' ')[2];
            //     if (arr['Last Name'].trim() == '') {
            //         arr['Last Name'] = arr['Full Name'].split(' ')[1];
            //     }
            // }
        } catch (err) {
            console.log("not found ");
        }
        // arr['MLS Number'] = jData['mlsid'];

        jData["responsivePhotos"]?.every((img, i) => {
            if (i < 10) {
                // arr["IMAGE" + (i + 1).toString()] = `=image("${img["url"]}")`;
                arr["IMAGE" + (i + 1).toString()] = `${img["url"]}`;
                return true;
            } else {
                return false;
            }
        });
        console.log({ URL: arr["URLS"], ADDRESS: arr["ADDRESS"] });
        // process.exit()
        // await Apify.openDataset(requestQueueId).pushData(arr);
        await (await Dataset.open(thisDatasetId)).pushData(arr);
        await CreateContact(arr);
        // session.markGood();
    } catch (error) {
        await (
            await Dataset.open("error")
        ).pushData({
            url: request.url,
            loadedurl: request.loadedUrl,
            error: error?.message ? error?.message : error,
        });
        throw error;
    }
});

src/routes.js

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage

# Added by Apify CLI
.venv


.gitignore

## Welcome to Fastest Zilow Scraper

Scrape real estate data from Zillow to discover new properties, compare prices, predict market trends, and more from the Zillow property listing page.
Scrape millions of listings and download your data as HTML, JSON, CSV, Excel, XML, and RSS feed.

### Features
Collect data periodically
Gets only last 48 hours data
Covers 15+ distinct data points for each property
Download real estate listings from the Zillow search results page. Just provide the Zillow search results URLs to extract property listings, and the scraper will get you the updated real estate data from Zillow in a spreadsheet.

Schedule crawlers hourly, daily, or weekly to get updated property listings on your Dropbox.

### My list will include
URL of Listing
Image URL of Listing
Price of Listing
Area of Property
No. of Bedrooms & Bathrooms and other info of Property
Address (Street, City, Zip Code) of Property
Type (For Sale, For-closures) of Property
Time Duration (Post On Zillow Time) of Property
Listing Agent First & Last Name and Contact Number
Listing Agency Email Address not at any additional cost
Owner Contact Number (If Given)



## Authors

- [@moazzammalek](https://linkedin.com/in/moazzammalek)



README.md

package-lock.json

package.json

You have to Enter Region and state as input and it will get properties which were created or updated last 48 hours

Zillow Fastest Scraper

Welcome to Fastest Zilow Scraper

Features

My list will include

Authors

Where next?