# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY --chown=myuser . ./


# Run the image. If you know you won't need headful browsers,
# you can remove the XVFB start script for a micro perf gain.
CMD ./start_xvfb_and_run_cmd.sh && npm start --silent


.actor/Dockerfile

Powerful Turo.com scraper to extract data from Turo car listings. Download Turo listings data as a HTML, JSON, CSV, Excel, or XML doc.

turo-scraper

Turo Car Listing Scraper

.actor/actor.json

Timeout in which the function passed as handlePageFunction needs to finish, in seconds.

Handle Page Timeout Secs

Limit

Sets the maximum concurrency (parallelism) for the crawl. Shortcut to the corresponding AutoscaledPool option.

Max Concurrency

Indicates how many times the request is retried if BasicCrawlerOptions.handleRequestFunction fails.

Max Request Retries

Maximum number of pages that the crawler will open. The crawl will stop when this limit is reached.

Max Requests Per Crawl 

Sets the minimum concurrency (parallelism) for the crawl. Shortcut to the corresponding AutoscaledPool option.

Min Concurrency

Locations

Scrape full data

Start URLs

PlaywrightCrawler Template

.actor/input_schema.json

{
    // Use IntelliSense to learn about possible attributes.
    // Hover to view descriptions of existing attributes.
    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
    "version": "0.2.0",
    "configurations": [
        {
            "type": "node",
            "request": "launch",
            "name": "Launch Program",
            "skipFiles": [
                "<node_internals>/**"
            ],
            "program": "${workspaceFolder}\\src\\main.js",
            "outFiles": [
                "${workspaceFolder}/**/*.js"
            ]
        }
    ]
}

.vscode/launch.json

/* eslint linebreak-style: ["error", "windows"] */
/**
 * This template is a production ready boilerplate for developing with `PuppeteerCrawler`.
 * Use this to bootstrap your projects using the most up-to-date code.
 * If you're looking for examples or want to learn more, see README.
 */

// For more information, see https://docs.apify.com/sdk/js
import { Actor } from 'apify';
// For more information, see https://crawlee.dev
import { PuppeteerCrawler } from 'crawlee';
import { router } from './routes.js';

// Initialize the Apify SDK
await Actor.init();

const input = await Actor.getInput();
input.scrapeFullData = input.scrapeFullData || false;
input.limit = input.limit || 3;
input.minConcurrency = input.minConcurrency || 1;
input.maxConcurrency = input.maxConcurrency || 3;
input.maxRequestRetries = input.maxRequestRetries || 2;
input.maxRequestsPerCrawl = input.maxRequestsPerCrawl || 1000;
input.handlePageTimeoutSecs = input.handlePageTimeoutSecs || 60;

const startUrls = [];

if (input.queries) {
    for (const query of input.queries) {
        startUrls.push({
            url: 'https://turo.com/us/en/search?country=US',
            userData: {
                label: 'search',
                query,
            },
        });
    }
}

if (input.startUrls) {
    for (const startUrl of input.startUrls) {
        startUrls.push({
            url: startUrl.url,
            userData: {
                label: 'searchByUrl',
            },
        });
    }
}

/* const startUrls = [{
    url: 'https://turo.com/us/en/search?country=US',
    //url: 'https://turo.com/us/en/car-rental/united-states/Waltham-MA/Honda/Accord/1749080',
    userData: { label: 'search' },
}];
 */
const proxyConfiguration = await Actor.createProxyConfiguration();

const crawler = new PuppeteerCrawler({
    proxyConfiguration,
    requestHandler: router,
    headless: true,
    requestHandlerTimeoutSecs: input.handlePageTimeoutSecs,
    minConcurrency: input.minConcurrency,
    maxConcurrency: input.maxConcurrency,
    maxRequestRetries: input.maxRequestRetries,
    maxRequestsPerCrawl: input.maxRequestsPerCrawl,
    preNavigationHooks: [
        async (crawlingContext) => {
            const { page, request } = crawlingContext;
            crawlingContext.input = input;
            if (request.userData.label === 'vehicle') {
                if (request.url.includes('/car-rental/')) {
                // await page.setRequestInterception(true);
                    const RESOURCE_EXCLUSIONS = ['image', 'media', 'font', 'stylesheet'];
                    await page.setRequestInterception(true);

                    page.on('request', (request) => {
                        if (RESOURCE_EXCLUSIONS.includes(request.resourceType())) {
                            return request.abort();
                        }
                        return request.continue();
                    });

                    crawlingContext.dataPromise = page.waitForResponse((response) => response.url().startsWith('https://turo.com/api/vehicle/detail?'), { timeout: 30 * 1000 });
                }
            }
            if (request.userData.label === 'searchByUrl') {
                if (request.url.includes('search') || request.url != 'https://turo.com/us/en/search?country=US') {
                    crawlingContext.searchDataPromise = page.waitForResponse((response) => response.url() === 'https://turo.com/api/v2/search', { timeout: 30 * 1000 });
                }
            }
        },
    ],
    postNavigationHooks: [
        async (crawlingContext) => {
            const { request } = crawlingContext;
            if (request.userData.label === 'vehicle') {
                crawlingContext.data = await crawlingContext.dataPromise;
            }
            if (request.userData.label === 'searchByUrl') {
                crawlingContext.searchData = await crawlingContext.searchDataPromise;
            }
        },
    ],

});

await crawler.run(startUrls);

// Exit successfully
await Actor.exit();


src/main.js

/* eslint linebreak-style: ["error", "windows"] */
import { Actor } from 'apify';
import { Dataset, createPuppeteerRouter } from 'crawlee';

export const router = createPuppeteerRouter();

router.addDefaultHandler(async ({ enqueueLinks, log }) => {
    // log.info(`enqueueing new URLs`);
    // await enqueueLinks({
    //     globs: ['https://apify.com/*'],
    //     label: 'detail',
    // });
});

router.addHandler('search', async ({ request, page, log, crawler }) => {
    if (request.userData.query) {
        const location = request.userData.query;
        log.info(`Search listings in ${location}`);
        await page.waitForSelector('#location');
        await page.type('#location', location, { delay: 250 });
        await page.keyboard.press('Enter');

        let searchResponseReceived = false;

        page.on('response', async (response) => {
            if (!searchResponseReceived) {
                if (response.url() === 'https://turo.com/api/v2/search') {
                    const res = await response.json();
                    if (res.vehicles.length) {
                        searchResponseReceived = true;
                        log.info(`Enqueued ${res.vehicles.length} new listing for detailed scraping`);
                        for (const vehicle of res.vehicles) {
                            const searchString = `${vehicle.location.city.replaceAll(' ', '-')
                            }-${vehicle.location.state.replaceAll(' ', '-')}/${vehicle.make.replaceAll(' ', '-')}/${vehicle.model.replaceAll(' ', '-')}/${vehicle.id}`;
                            const link = `https://turo.com/us/en/car-rental/united-states/${searchString}`;
                            await crawler.requestQueue.addRequest({
                                url: link,
                                userData: {
                                    label: 'vehicle',
                                },
                            });
                        }
                    }
                }
            }
        });

        await page.waitForResponse('https://turo.com/api/v2/search');
        await page.waitForSelector('a[data-testid="vehicle-card-link-box"]');
    }
});

router.addHandler('searchByUrl', async ({ searchData, crawler, log, request }) => {
    log.info(`Search listings for URL ${request.loadedUrl}`);
    const data = await searchData.json();
    if (data.vehicles.length) {
        for (const vehicle of data.vehicles) {
            try{
            let state = '';
            if(vehicle.location.state){
                state = `-${vehicle.location.state.replaceAll(' ', '-')}`
            }
            const searchString = `${vehicle.location.city.replaceAll(' ', '-')
            }${state}/${vehicle.make.replaceAll(' ', '-')}/${vehicle.model.replaceAll(' ', '-')}/${vehicle.id}`;
            const link = `https://turo.com/us/en/car-rental/united-states/${searchString}`;
            await crawler.requestQueue.addRequest({
                url: link,
                userData: {
                    label: 'vehicle',
                },
            });
        }catch(e){
            log.info(e)
        }
        }
    }
});

router.addHandler('vehicle', async ({ request, log, data, input }) => {
    log.info(`Handling listing details`, { url: request.loadedUrl });
    await vehicleInfoParse(await data.json(), input);
    const datasetData = await Dataset.getData();
    if (datasetData.total >= input.limit) {
        Actor.exit();
    }
});

async function vehicleInfoParse(data, input) {
    let vehicleData = {};
    if (input.scrapeFullData) {
        vehicleData = data;
    } else {
        vehicleData.id = data.vehicle.id;
        vehicleData.type = data.vehicle.type;
        vehicleData.vin = data.vehicle.vin;
        vehicleData.make = data.vehicle.make;
        vehicleData.model = data.vehicle.model;
        vehicleData.year = data.vehicle.year;
        vehicleData.automaticTransmission = true;
        vehicleData.licensPlate = data.vehicle.registration.licensePlate;
        vehicleData.licensPlateState = data.vehicle.registration.state;
        vehicleData.listingCreatedTime = data.vehicle.listingCreatedTime;
        vehicleData.url = data.vehicle.url;
        vehicleData.marketCountry = data.vehicle.marketCountry;
        vehicleData.description = data.description;
        vehicleData.basicCarDetails = data.basicCarDetails;
        vehicleData.images = data.images.map((image) => { return image.originalImageUrl; });
        vehicleData.location = {};
        vehicleData.location.country = data.location.country;
        vehicleData.location.state = data.location.state;
        vehicleData.location.city = data.location.city;
        vehicleData.location.address = data.location.address;
        vehicleData.location.latitude = data.location.latitude;
        vehicleData.location.longitude = data.location.longitude;
        vehicleData.minimumAgeInYearsToRent = data.minimumAgeInYearsToRent;
        vehicleData.numberOfFavorites = data.numberOfFavorites;
        vehicleData.numberOfRentals = data.numberOfRentals;
        vehicleData.numberOfReviews = data.numberOfReviews;
        vehicleData.turoGo = data.turoGo;
        vehicleData.dailyPrice = data.rate.averageDailyPrice;
        vehicleData.owner = {};
        vehicleData.owner.firstName = data.owner.firstName;
        vehicleData.owner.lastName = data.owner.lastName;
        vehicleData.owner.id = data.owner.id;
        vehicleData.owner.url = data.owner.url;
        vehicleData.owner.image = data.owner.image.originalImageUrl;
    }
    await Dataset.pushData(vehicleData);
}


src/routes.js

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = crlf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage
!storage/key_value_stores/default/INPUT.json
# Added by Apify CLI
.venv


.gitignore

package.json

Powerful Turo.com scraper to extract data from Turo car listings. Download Turo listings data as an HTML, JSON, CSV, Excel, or XML doc.

Where next?