# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.idea
node_modules
storage


.gitignore

# Specify the base Docker image. You can read more about
# the available images at https://sdk.apify.com/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY . ./


# Run the image.
CMD npm start --silent


Dockerfile

Gumtree list links that contains products you want to be scraped.

Gumtree List Link:

Scraper will stop running when successfully scrapes X amount of products or when reaches maximum number of page Gumtree allows.

Maximum Product Count:

Select proxies to be used by your crawler.

Proxy configuration

You can either scrape generic info (30 products with 1 web request) or more detailled info (30 products with 31 web requests).

Scraper Mode:

Input schema for Gumtree Scraper

INPUT_SCHEMA.json

// This is the main Node.js source code file of your actor.
// An actor is a program that takes an input and produces an output.

// For more information, see https://sdk.apify.com
import { Actor } from 'apify';
// For more information, see https://crawlee.dev
import { CheerioCrawler } from 'crawlee';
import { RequestQueue } from 'crawlee';
import { Dataset } from 'crawlee';

// Initialize the Apify SDK
await Actor.init();

const input = await Actor.getInput();
console.log("Start URL: "+input.gumtree_link);
console.log("Scraper Mode:"+input.scraper_mode);
const requestQueue = await RequestQueue.open();

for(let input_link_index=0; input_link_index < input.gumtree_link.length; input_link_index++)
{
    await requestQueue.addRequest({ url: input.gumtree_link[input_link_index] });
}



const proxyConfiguration = await Actor.createProxyConfiguration(input.proxy_config);


let record_count = 0;
// Create the crawler and add the queue with our URL
// and a request handler to process the page.
const crawler = new CheerioCrawler({
    proxyConfiguration,
    requestQueue,
    minConcurrency: 2,
    maxConcurrency: 5,
    maxRequestRetries: 3,
    maxRequestsPerCrawl: 999999,

    // The `$` argument is the Cheerio object
    // which contains parsed HTML of the website.
    async requestHandler({ $, request }) {

        // Record Page
        if(request.url.includes("/p/"))
        {

            console.log(` - Scraping More Details of: ${request.url}`);
            let pagecontent = $.html();

            // Record Details JSON
            let record_details_json = pagecontent.substr(pagecontent.indexOf('window.clientData =')+19);
            record_details_json = record_details_json.substr(0, record_details_json.indexOf(';</script>')).trim();

            if(record_details_json.includes("adDetails"))
            {

                let record_data = JSON.parse(record_details_json);

                // Scrape Record Details
                let record_link = request.url;
                let record_name = record_data.adDetails.title;
                let record_description =  record_data.page.description;
                let record_price = record_data.adDetails.price;
                let record_location = record_data.adDetails.address;
                let record_thumbnail = record_data.images[0];

                let record_image_array = record_data.images;
                let record_video_array = record_data.video.urls;
                let record_description_long = $('[itemprop="description"]').text();
                let category_array = record_data.breadcrumbs.links;

                // Data in array format joined with '|' character for smooth display in Excel/CSV
                let record_images = record_image_array.join('|');
                let record_videos = record_video_array.join('|');
                let record_breadcrumb = "";


                for(let pindex = 0; pindex < category_array.length; pindex++)
                {
                    record_breadcrumb += category_array[pindex].name + "|";
                }


                let record_specs = {};


                // Get Attributes (Car specs, Product specs, Property info, Pet info etc.)  &  add them to output with given key & value pair
                
                try{
                if(record_data.adDetails.attributes.attributeGroups != null && record_data.adDetails.attributes.attributeGroups != undefined)
                {
                    for(let aindex = 0; aindex < record_data.adDetails.attributes.attributeGroups.length; aindex++ )
                    {
                        for(let atindex = 0; atindex < record_data.adDetails.attributes.attributeGroups[aindex].attributes.length; atindex++)
                        {
                            
                            record_specs[record_data.adDetails.attributes.attributeGroups[aindex].attributes[atindex].key] = record_data.adDetails.attributes.attributeGroups[aindex].attributes[atindex].value + " " + record_data.adDetails.attributes.attributeGroups[aindex].attributes[atindex].unit;
                            record_specs[record_data.adDetails.attributes.attributeGroups[aindex].attributes[atindex].key] = record_specs[record_data.adDetails.attributes.attributeGroups[aindex].attributes[atindex].key].trim();
                        
                        }
                    }
                }
                }catch(e)
                {}

                try{
                if(record_data.adDetails.attributes.commonAttributes != null && record_data.adDetails.attributes.commonAttributes != undefined)
                {
                    for(let aindex = 0; aindex < record_data.adDetails.attributes.commonAttributes.length; aindex++ )
                    {
                    
                        record_specs[record_data.adDetails.attributes.commonAttributes[aindex].key] = record_data.adDetails.attributes.commonAttributes[aindex].value + " " + record_data.adDetails.attributes.commonAttributes[aindex].unit;
                        record_specs[record_data.adDetails.attributes.commonAttributes[aindex].key] = record_specs[record_data.adDetails.attributes.commonAttributes[aindex].key].trim();
                    
                    }

                }
                }catch(e)
                {}


                try{
                if(record_data.adDetails.attributes[0].key != null && record_data.adDetails.attributes[0].key != undefined)
                {
                    for(let aindex = 0; aindex < record_data.adDetails.attributes.length; aindex++ )
                    {
                    
                        record_specs[record_data.adDetails.attributes[aindex].key] = record_data.adDetails.attributes[aindex].value + " " + record_data.adDetails.attributes[aindex].unit;
                        record_specs[record_data.adDetails.attributes[aindex].key] = record_specs[record_data.adDetails.attributes[aindex].key].trim();
                
                    }    
                }
                }catch(e)
                {}


                let scraped_data = {
                            list_url: input.gumtree_link,
                            record_url: record_link,
                            record_name: record_name,
                            record_description: record_description,
                            record_price: record_price,
                            record_location: record_location,
                            record_thumbnail: record_thumbnail,
                            record_images : record_image_array,
                            record_videos : record_video_array,
                            record_description_long : record_description_long,
                            record_breadcrumb : record_breadcrumb,
                            record_specs: record_specs
                            

                };
                await Dataset.pushData(scraped_data);
                console.log(`Success: Record details scraped!`);
                record_count++;
                
                if(record_count >= input.max_record)
                {
                    // Exit successfully
                    await Actor.exit();
                }

            }else
            {
                console.log(`Fail: Missing/faulty product JSON in source code!`);
            }
            

            
               


        }else if(request.url.includes("gumtree.com/"))
        {
            // List URL
             let current_page_no = $('.pagination-currentpage').text();
            current_page_no = current_page_no.replace("Currently on\nPage\n","");
            current_page_no = current_page_no.replace("Currently on Page","").trim();
            console.log("Current Page: "+current_page_no+"\n");
            let current_page_no_number = parseInt(current_page_no);
        
            const record_containers = $('.listing-link');

            // If List Page Doesn't Have Results, Retry
            if(record_containers.length < 1 && isNan(current_page_no_number))
            {
                console.log("- Retrying List Page: "+ request.url );
                await requestQueue.addRequest({ url:  request.url });
            }else if(record_containers.length < 1 && current_page_no_number)
            {
                console.log("- Retrying List Page: "+ request.url );
                await requestQueue.addRequest({ url:  request.url });
            }
            else
            {
                for(let pindex = 0; pindex < record_containers.length; pindex++)
                {
                    let record_link = "https://www.gumtree.com"+record_containers.eq(pindex).attr('href');
                    let record_name = record_containers.eq(pindex).find('.listing-title').eq(0).text().trim();
                    let record_description = record_containers.eq(pindex).find('.listing-description').eq(0).text().trim();
                    let record_price = record_containers.eq(pindex).find('.listing-price').eq(0).text().trim();
                    let record_location = record_containers.eq(pindex).find('.listing-location').eq(0).text().trim();
                    let record_thumbnail = record_containers.eq(pindex).find('.listing-thumbnail').eq(0).find('img').eq(0).attr("src");
                    let posted_ago = record_containers.eq(pindex).find('[data-q="listing-adAge"]').eq(0).text().replace("Ad posted","").trim();
                    
                    if(record_thumbnail == "")
                        record_thumbnail = record_containers.eq(pindex).find('.listing-thumbnail').eq(0).find('img').eq(0).attr("data-src");
                    

                    // Fast Mode
                    if(record_link != "https://www.gumtree.comundefined")
                    {


                        if(input.scraper_mode == "0")
                        {
                            let record_specs = {};

                            let attributes_in_list =  record_containers.eq(pindex).find('.listing-attributes > li');
                            for(let aindex = 0; aindex < attributes_in_list.length; aindex++ )
                            {
                                var slug_key = attributes_in_list.eq(aindex).find('span').eq(0).text().toLowerCase().trim().replace(/[^\w\s-]/g, '').replace(/[\s_-]+/g, '-').replace(/^-+|-+$/g, '');
                                var att_value = attributes_in_list.eq(aindex).find('span').eq(1).text();

                                record_specs[slug_key] = att_value;
                            }

                            let scraped_data = {
                                        list_url: input.gumtree_link,
                                        record_url: record_link,
                                        record_name: record_name,
                                        record_description: record_description,
                                        record_price: record_price,
                                        record_location: record_location,
                                        record_thumbnail: record_thumbnail,
                                        posted_ago: posted_ago,
                                        record_specs: record_specs
                                        
                            };
                            await Dataset.pushData(scraped_data);
                            console.log(`Scraping basic data of: ${record_link}`);
                            record_count++;
                            
                            if(record_count >= input.max_record)
                            {
                                // Exit successfully
                                await Actor.exit();
                            }


                        }else // More Details Mode (Slower)
                        {
                            await requestQueue.addRequest({ url: record_link });
                        }


                       
                    }

              


                }


                // Get Next Page Link
                const next_page_element = $('.pagination-next > a');
                if(next_page_element != null)
                {
                    let next_page_link = next_page_element.attr("href");

                    if(next_page_link!= null && next_page_link!= undefined)
                    {
                        if(next_page_link.includes("gumtree.com"))
                        {}
                        else
                            next_page_link = "https://www.gumtree.com"+next_page_link;
                        
                        await requestQueue.addRequest({ url: next_page_link });

                        console.log(" -- Navigating to Next Page: "+next_page_link); 
                    }
                    
                }
            }
            
        }
        
            
        
        

        
    }
})

// Start the crawler and wait for it to finish
await crawler.run();



// Exit successfully
await Actor.exit();


main.js

gumtree-scraper

package.json

Export data from search results of Gumtree.com, the free classified ads, and community website. Get images, location, prices, and descriptions of the available products and adverts. Save the data as HTML, JSON, CSV, Excel, or XML.

Gumtree Scraper

Gumtree Scraper · Apify

Input · Gumtree Scraper · Apify

API · Gumtree Scraper · Apify

Related Actors · Gumtree Scraper · Apify

Export data from search results of Gumtree.com, the free classified ads, and community website. Get images, location, prices, and descriptions of the available products and adverts. Save the data a...

Where next?