Google Hotels Scraper
Pricing
Pay per usage
Google Hotels Scraper is an actor for extracting prices from Google search results.
Pricing
Pay per usage
Rating
3.6
(5)
Developer
Martin Forejt
Maintained by CommunityActor stats
20
Bookmarked
463
Total users
3
Monthly active users
6 hours ago
Last modified
Categories
Share
# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed filesnode_modules
# git folder.gitroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf{ "root": true, "env": { "browser": true, "es2020": true, "node": true }, "extends": [ "@apify/eslint-config-ts" ], "parserOptions": { "project": "./tsconfig.json", "ecmaVersion": 2020 }, "ignorePatterns": [ "node_modules", "dist", "**/*.d.ts" ]}# This file tells Git which files shouldn't be added to source control
.DS_Store.ideadistnode_modulesapify_storagestorage
# Added by Apify CLI.venv{ "name": "google-hotels-scraper", "version": "0.0.1", "type": "module", "description": "This is an example of an Apify actor.", "engines": { "node": ">=18.0.0" }, "dependencies": { "apify": "^3.7.2", "crawlee": "^3.17.0", "playwright": "*" }, "devDependencies": { "@apify/eslint-config-ts": "^0.3.0", "@apify/tsconfig": "^0.1.0", "@typescript-eslint/eslint-plugin": "^6.7.2", "@typescript-eslint/parser": "^6.7.2", "eslint": "^8.50.0", "tsx": "^4.6.2", "typescript": "^5.3.3" }, "scripts": { "start": "npm run start:dev", "start:prod": "node dist/main.js", "start:dev": "tsx src/main.ts", "build": "tsc", "lint": "eslint ./src --ext .ts", "lint:fix": "eslint ./src --ext .ts --fix", "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1", "postinstall": "npx crawlee install-playwright-browsers" }, "author": "It's not you it's me", "license": "ISC"}{ "extends": "@apify/tsconfig", "compilerOptions": { "module": "NodeNext", "moduleResolution": "NodeNext", "target": "ES2022", "outDir": "dist", "noUnusedLocals": false, "skipLibCheck": true, "lib": ["DOM"] }, "include": [ "./src/**/*" ]}# Specify the base Docker image. You can read more about# the available images at https://crawlee.dev/docs/guides/docker-images# You can also use any other image from Docker Hub.FROM apify/actor-node-playwright-chrome:20 AS builder
# Copy just package.json and package-lock.json# to speed up the build using Docker layer cache.COPY package*.json ./
# Install all dependencies. Don't audit to speed up the installation.RUN npm install --include=dev --audit=false
# Next, copy the source files using the user set# in the base image.COPY . ./
# Install all dependencies and build the project.# Don't audit to speed up the installation.RUN npm run build
# Create final imageFROM apify/actor-node-playwright-chrome:20
# Copy just package.json and package-lock.json# to speed up the build using Docker layer cache.COPY package*.json ./
# Install NPM packages, skip optional and development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version \ && rm -r ~/.npm
# Copy built JS files from builder imageCOPY /home/myuser/dist ./dist
# Next, copy the remaining files and directories with the source code.# Since we do this after NPM install, quick build will be really fast# for most source file changes.COPY . ./
# Run the image. If you know you won't need headful browsers,# you can remove the XVFB start script for a micro perf gain.CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent{ "actorSpecification": 1, "name": "google-hotels-scraper", "title": "Google Hotels Scraper", "description": "Google Hotels Scraper is an actor for extracting prices from Google search results.", "version": "0.1", "meta": { "templateId": "ts-crawlee-playwright-chrome" }, "input": "./input_schema.json", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }, "dockerfile": "./Dockerfile"}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "url": { "type": "string", "title": "URL", "description": "Direct link to the hotel detail page on Google Travel.", "example": "https://www.google.com/travel/hotels/entity/abc123" }, "title": { "type": "string", "title": "Hotel name", "description": "Name of the hotel as displayed on Google Travel.", "example": "Hilton Prague" }, "website": { "type": "string", "title": "Website", "description": "Official website URL of the hotel, if available.", "example": "https://www.hilton.com" }, "address": { "type": "string", "title": "Address", "description": "Street address of the hotel, if available.", "example": "Pobrezni 1, 186 00 Karlin, Czechia" }, "phone": { "type": "string", "title": "Phone", "description": "Contact phone number of the hotel, if available.", "example": "+420 224 841 111" }, "photos": { "type": "array", "title": "Photos", "description": "List of photo image URLs of the hotel.", "items": { "type": "string" }, "example": ["https://lh3.googleusercontent.com/photo1", "https://lh3.googleusercontent.com/photo2"] }, "thumbnail": { "type": "string", "title": "Thumbnail", "description": "URL of the first/primary hotel photo used as a thumbnail.", "example": "https://lh3.googleusercontent.com/photo1" }, "rating": { "type": "number", "title": "Rating", "description": "Average guest rating of the hotel (0 when no reviews are available).", "example": 4.5 }, "reviews": { "type": "number", "title": "Number of reviews", "description": "Total number of guest reviews (0 when no reviews are available).", "example": 1234 }, "priceRange": { "type": "string", "title": "Price range", "description": "Price or price range across providers in the selected currency. A single value when only one price is found, or a 'min - max' range.", "example": "120 - 180" }, "prices": { "type": "array", "title": "Prices", "description": "Per-provider prices with a link to book.", "items": { "type": "object", "properties": { "provider": { "type": "string", "title": "Provider", "description": "Name of the booking provider.", "example": "Booking.com" }, "price": { "type": "number", "title": "Price", "description": "Nightly price offered by the provider in the selected currency.", "example": 149 }, "link": { "type": "string", "title": "Link", "description": "Link to the provider's booking page.", "example": "https://www.google.com/travel/clk?..." } }, "required": ["provider", "price", "link"] } } }, "required": ["url", "title", "photos", "rating", "reviews", "prices"] }, "views": { "hotels": { "title": "Overview", "description": "Overview of all scraped hotels", "transformation": { "fields": [ "thumbnail", "title", "address", "rating", "reviews", "priceRange", "url" ] }, "display": { "component": "table", "properties": { "thumbnail": { "label": "Thumbnail", "format": "image" }, "title": { "label": "Hotel name", "format": "text" }, "address": { "label": "Address", "format": "text" }, "rating": { "label": "Rating", "format": "text" }, "reviews": { "label": "Number of reviews", "format": "text" }, "priceRange": { "label": "Price range", "format": "text" }, "url": { "label": "URL", "format": "link" } } } } }}{ "title": "PlaywrightCrawler Template", "type": "object", "schemaVersion": 1, "properties": { "searchQuery": { "title": "Search query", "type": "string", "description": "Query to search on google travel hotels.", "example": "Prague", "prefill": "Prague", "editor": "textfield" }, "checkInDate": { "title": "Check-in date", "type": "string", "description": "Check-in date for hotels search. Format: YYYY-MM-DD.", "example": "2024-09-01", "prefill": "2024-12-01", "editor": "textfield" }, "checkOutDate": { "title": "Check-out date", "type": "string", "description": "Check-out date for hotels search. Format: YYYY-MM-DD.", "example": "2024-09-02", "prefill": "2024-12-05", "editor": "textfield" }, "numberOfAdults": { "title": "Number of adults", "type": "integer", "description": "Number of adults for hotels search.", "prefill": 2 }, "numberOfChildren": { "title": "Number of children", "type": "integer", "description": "Number of children for hotels search.", "prefill": 0 }, "currencyCode": { "title": "Currency code", "type": "string", "description": "Currency code for hotels search.", "prefill": "EUR", "editor": "textfield" }, "maxResults": { "title": "Max crawled hotels", "type": "integer", "description": "Use to limit number of hotels you want to get from crawler. If you fill nothing all hotels will be scrape", "minimum": 0, "default": 5, "prefill": 5, "example": 5 }, "proxyConfig": { "title": "Proxy configuration", "type": "object", "description": "This is required if you want to use Apify Proxy.", "prefill": { "useApifyProxy": true }, "default": { "useApifyProxy": true }, "editor": "proxy", "sectionCaption" : "Proxy configuration", "sectionDescription" : "Choose which proxies to use." }, "maxRequestsPerCrawl": { "title": "Max Requests per Crawl", "type": "integer", "description": "Maximum number of requests that can be made by this crawler." } }, "required": [ "searchQuery", "checkInDate", "checkOutDate", "numberOfAdults", "numberOfChildren", "currencyCode", "proxyConfig" ]}{ "actorOutputSchemaVersion": 1, "title": "Output schema of the Google Hotels Scraper", "description": "Scraped hotels are stored in the default dataset.", "properties": { "hotels": { "type": "string", "title": "Scraped hotels", "description": "List of scraped hotels with prices, ratings and details.", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1export const GOOGLE_CONSENT_DIALOG_URL = 'https://consent.google.com';2export const CONTENT_LANGUAGE_CODE = 'en';3export const DEFAULT_MAX_REQUESTS_PER_CRAWL = 100;4export const DEFAULT_NUM_OF_ADULTS = 2;5export const DEFAULT_NUM_OF_CHILDREN = 0;6export const MAX_NUM_OF_PEOPLE = 6;1import { Actor, ProxyConfigurationOptions } from 'apify';2import { PlaywrightCrawler } from 'crawlee';3import { createGoogleHotelsRouter } from './routes.js';4import { GoogleHotelsOptions } from './scraper/options.js';5import { CONTENT_LANGUAGE_CODE, DEFAULT_MAX_REQUESTS_PER_CRAWL } from './constants.js';6
7interface Input extends GoogleHotelsOptions {8 proxyConfig: ProxyConfigurationOptions;9 maxRequestsPerCrawl: number;10}11
12// Initialize the Apify SDK13await Actor.init();14
15const input = await Actor.getInput<Input>() ?? {} as Input;16
17// validate inputs format18if (input.checkInDate.match(/^\d{4}-\d{2}-\d{2}$/) === null) {19 await Actor.exit('Invalid check-in date format. Use YYYY-MM-DD.', { exitCode: 1 });20}21if (input.checkOutDate.match(/^\d{4}-\d{2}-\d{2}$/) === null) {22 await Actor.exit('Invalid check-out date format. Use YYYY-MM-DD.', { exitCode: 1 });23}24
25const proxyConfiguration = await Actor.createProxyConfiguration(input.proxyConfig);26const {27 searchQuery,28 maxRequestsPerCrawl = DEFAULT_MAX_REQUESTS_PER_CRAWL,29} = input;30
31const crawler = new PlaywrightCrawler({32 proxyConfiguration,33 maxRequestsPerCrawl,34 requestHandler: createGoogleHotelsRouter(input),35});36
37try {38 await crawler.run([`https://www.google.com/travel/search?q=${encodeURIComponent(searchQuery)}&hl=${CONTENT_LANGUAGE_CODE}`]);39} catch (error) {40 await Actor.exit({ exitCode: 1 });41}42
43// Exit successfully44await Actor.exit();1import { Dataset, createPlaywrightRouter } from 'crawlee';2import { skipGoogleConsent } from './scraper/utils.js';3import { GoogleHotelsOptions } from './scraper/options.js';4import { getDetailsUrls } from './scraper/list.js';5import { getHotelItemData } from './scraper/detail.js';6
7export const createGoogleHotelsRouter = (options: GoogleHotelsOptions) => {8 const router = createPlaywrightRouter();9
10 router.addDefaultHandler(async (ctx) => {11 const { request, page, log, enqueueLinks } = ctx;12 log.info(`enqueueing new URLs`, { url: request.loadedUrl });13
14 // Get rid of the Google consent dialog15 await skipGoogleConsent(request, page);16
17 await getDetailsUrls(ctx, options, async (urls) => {18 await enqueueLinks({19 urls,20 strategy: 'same-domain',21 label: 'detail',22 });23 });24 });25
26 router.addHandler('detail', async (ctx) => {27 const { request, page } = ctx;28 // Get rid of the Google consent dialog29 await skipGoogleConsent(request, page);30 const item = await getHotelItemData(ctx);31 await Dataset.pushData(item);32 });33
34 return router;35};1import { LoadedRequest, PlaywrightCrawlingContext, Request } from 'crawlee';2
3export interface GoogleHotelItemData {4 url: string;5 title: string;6 website?: string;7 address?: string;8 phone?: string;9 photos: string[];10 thumbnail?: string;11 rating: number;12 priceRange?: string;13 reviews: number;14 prices: { provider: string, price: number, link: string }[];15}16
17export const getHotelItemData = async <Context extends PlaywrightCrawlingContext>(ctx: Omit<Context, 'request'> & {18 request: LoadedRequest<Request>;19}): Promise<GoogleHotelItemData> => {20 const { page, log } = ctx;21
22 const title = await page.locator('h1[role="heading"]').last().innerText();23 const url = page.url();24
25 const [pricesTab, reviewsTab, aboutTab, photosTab] = await Promise.all([26 page.waitForSelector('div[id="prices"]'),27 page.waitForSelector('div[id="reviews"]'),28 page.waitForSelector('div[id="details"]'),29 page.waitForSelector('div[id="photos"]'),30 ]);31
32 await pricesTab.click();33 const pricesBtns = await page.locator('button[aria-label^="Visit site for"]').all();34 // const pricesBtns = await page.$$('button[aria-label^="Visit site for"]');35 const prices = (await Promise.all(pricesBtns.map(async (btn) => {36 const row = btn.locator('..').locator('..');37 const link = await row.locator('..').getAttribute('href');38
39 if (link != null) {40 try {41 const provider = await row.locator('div:nth-of-type(1) > div > span > span').first().innerText();42
43 const textPrice = await row.locator('div:nth-of-type(2) > span > span > span > span').first().innerText();44 const price = parseFloat(textPrice.replace(/[^0-9.]/g, ''));45
46 return { provider, price, link: `https://www.google.com/travel${link}` };47 } catch (e) {48 // some providers has multiple prices and structure is different of the others is different,49 // so we need to skip them and use the first (cheapest) price50 return null;51 }52 }53
54 return null;55 }))).filter((price, i, arr) => price !== null && arr.findIndex((o) => o?.provider === price.provider) === i) as GoogleHotelItemData['prices'];56
57 await reviewsTab.click();58 let reviewText: string | undefined;59 if (await page.locator('div[aria-label*="out of"][role="text"]').count()) {60 reviewText = await page.locator('div[aria-label*="out of"][role="text"]').first().getAttribute('aria-label') || undefined;61 }62 let rating = 0;63 let reviews = 0;64 if (reviewText) {65 rating = parseFloat(reviewText.substring(0, reviewText.indexOf('out')).trim());66 reviews = parseInt(reviewText.substring(reviewText.indexOf('from'))67 .replace('from', '')68 .replace('reviews', '')69 .replace(',', '')70 .trim(), 10);71 }72
73 await aboutTab.click();74
75 let address: string | undefined;76 if (await page.locator('span[aria-label*="hotel address is"]').count()) {77 address = await page.locator('span[aria-label*="hotel address is"]').innerText() || undefined;78 }79 let website: string | undefined;80 if (await page.locator('a[aria-label="Website"]').count()) {81 website = await page.locator('a[aria-label="Website"]').getAttribute('href') || undefined;82 }83 let phone: string | undefined;84 if (await page.locator('span[aria-label*="call this hotel"]').count()) {85 phone = await page.locator('span[aria-label*="call this hotel"]').innerText() || undefined;86 }87
88 await photosTab.click();89 await page.waitForTimeout(2000);90 const photosElements = await page.locator('span[id="photos"] img[alt*="Photo"]').all();91 const photos = (await Promise.all(photosElements.map(async (photo) => {92 const src = await photo.getAttribute('src');93 if (src !== null) {94 if (src.startsWith('//')) {95 return `https:${src}`;96 }97 return src;98 }99 return null;100 }))).filter((photo) => photo !== null) as string[];101
102 log.info(`Parsed detail (${title})`, { url: page.url() });103
104 const thumbnail = photos.length ? photos[0] : undefined;105 let priceRange: string | undefined;106 if (prices.length === 1) {107 priceRange = `${prices[0].price}`;108 } else if (prices.length > 1) {109 const minPrice = prices.reduce((min, p) => (p.price < min ? p.price : min), prices[0].price);110 const maxPrice = prices.reduce((max, p) => (p.price > max ? p.price : max), prices[0].price);111 priceRange = `${minPrice} - ${maxPrice}`;112 }113
114 return {115 thumbnail,116 url,117 title,118 website,119 address,120 phone,121 photos,122 rating,123 reviews,124 prices,125 priceRange,126 };127};1import { Page } from 'playwright';2import { LoadedRequest, PlaywrightCrawlingContext, Request } from 'crawlee';3import { waitWhileGoogleLoading } from './utils.js';4import { GoogleHotelsOptions } from './options.js';5import { DEFAULT_NUM_OF_ADULTS, DEFAULT_NUM_OF_CHILDREN, MAX_NUM_OF_PEOPLE } from '../constants.js';6
7// define type for callback function8type EnqueueDetails = (urls: string[]) => Promise<void>;9
10export const getDetailsUrls = async <Context extends PlaywrightCrawlingContext>(ctx: Omit<Context, 'request'> & {11 request: LoadedRequest<Request>;12}, options: GoogleHotelsOptions, enqueueDetails: EnqueueDetails) => {13 const { page, log } = ctx;14 // Wait for the input element to be present and the page to be loaded15 const element = await page.waitForSelector('input[aria-label="Search for places, hotels and more"]');16 log.info(await element.inputValue());17
18 await fillInputForm(page, options);19 await waitWhileGoogleLoading(page);20 await page.waitForTimeout(1000);21
22 // const nextPageButtonSelector = 'main > c-wiz > span > c-wiz > c-wiz:last-of-type > div > button:nth-of-type(2)';23 let hasNextPage = true;24 let pageNumber = 1;25 let totalItems = 0;26 do {27 const items = await page.$$('main > c-wiz > span > c-wiz > c-wiz > div > a');28 log.info(`Found ${items.length} items on the page ${pageNumber}`);29 const urls = await Promise.all(items.map(async (item) => (30 `https://www.google.com${await item.getAttribute('href')}`31 ))) as string[];32
33 if (options.maxResults === undefined) {34 await enqueueDetails(urls);35 } else {36 await enqueueDetails(urls.slice(0, options.maxResults - totalItems));37 }38
39 totalItems += items.length;40 const nextPageButton = page.getByRole('button').filter({ hasText: 'Next' }).first();41 // const nextPageButton = await page.$(nextPageButtonSelector);42 if (nextPageButton !== null && (options.maxResults === undefined || totalItems < options.maxResults!)) {43 await nextPageButton.click();44 await waitWhileGoogleLoading(page);45 await page.waitForTimeout(1000);46 pageNumber++;47 } else {48 hasNextPage = false;49 }50 } while (hasNextPage);51};52
53const fillInputForm = async (page: Page, options: GoogleHotelsOptions) => {54 let checkInElement = await page.waitForSelector('input[aria-label="Check-in"]');55
56 await checkInElement.click();57
58 checkInElement = await page.waitForSelector(59 'div[role="dialog"] > div:nth-of-type(2) > div:nth-of-type(2) > div:nth-of-type(2) > div > div > input[aria-label="Check-in"]',60 );61 const checkOutElement = await page.waitForSelector(62 'div[role="dialog"] > div:nth-of-type(2) > div:nth-of-type(2) > div:nth-of-type(2) > div > div > input[aria-label="Check-out"]',63 );64
65 await checkInElement.fill(options.checkInDate);66 await checkOutElement.click();67 await page.waitForTimeout(1000);68 await checkOutElement.fill(options.checkOutDate);69 await checkOutElement.press('Enter');70
71 const submitButton = await page.waitForSelector('div[role="dialog"] > div:nth-of-type(4) > div > button:nth-of-type(2)');72 await submitButton.click();73
74 const peopleButton = await page.waitForSelector('div[role="button"][aria-label^="Number of travelers"]');75 await peopleButton.click();76 await page.waitForTimeout(1000);77
78 let adults = DEFAULT_NUM_OF_ADULTS;79 let children = DEFAULT_NUM_OF_CHILDREN;80
81 while (adults > options.numberOfAdults && adults > 0) {82 const removeAdultButton = await page.waitForSelector('button[aria-label="Remove adult"]');83 await removeAdultButton.click();84 adults--;85 }86 while (adults < options.numberOfAdults && (adults + children) <= MAX_NUM_OF_PEOPLE) {87 const addAdultButton = await page.waitForSelector('button[aria-label="Add adult"]');88 await addAdultButton.click();89 adults++;90 }91 while (children > options.numberOfChildren && children >= 0) {92 const removeChildButton = await page.waitForSelector('button[aria-label="Remove child"]');93 await removeChildButton.click();94 children--;95 }96 while (children < options.numberOfChildren && (adults + children) <= MAX_NUM_OF_PEOPLE) {97 const addChildButton = await page.waitForSelector('button[aria-label="Add child"]');98 await addChildButton.click();99 children++;100 }101
102 const peopleDoneButton = await page.waitForSelector(103 'div[data-default-adult-num="2"] > div > div > div:nth-of-type(2) > div:nth-of-type(2) > div:nth-of-type(2) > div:nth-of-type(2) > button',104 );105 await peopleDoneButton.click();106
107 const currencyButton = await page.waitForSelector('footer div c-wiz button');108 await currencyButton.click();109 await page.waitForTimeout(1000);110 const requiredCurrency = options.currencyCode;111 const currencyRadio = await page.waitForSelector(`div[role="radio"][data-value="${requiredCurrency.toUpperCase()}"]`);112 await currencyRadio.click();113 const currencyDoneButton = await page.waitForSelector('div[aria-label="Select currency"] > div:nth-of-type(3) > div:nth-of-type(2) > button');114 await currencyDoneButton.click();115};1export interface GoogleHotelsOptions {2 searchQuery: string;3 checkInDate: string;4 checkOutDate: string;5 numberOfAdults: number;6 numberOfChildren: number;7 currencyCode: string;8 maxResults?: number;9}1import { Page } from 'playwright';2import { LoadedRequest, Request } from 'crawlee';3
4/**5 * This function get rid of the Google consent dialog and redirects back to desired page.6 * @param request7 * @param page8 */9export const skipGoogleConsent = async (request: LoadedRequest<Request>, page: Page) => {10 // If the loaded URL is the Google consent dialog, reject all cookies11 if (request.loadedUrl.startsWith('https://consent.google.com')) {12 await page.click('button[aria-label="Reject all"]');13 }14};15
16/**17 * This function waits for the Google loading spinner to disappear.18 * @param page19 */20export const waitWhileGoogleLoading = async (page: Page) => {21 await page.waitForFunction(async () => {22 const loader = document.querySelector('div[aria-label="Loading results"]');23 return !(loader?.checkVisibility() ?? false);24 });25};