Google Shopping
Pricing
from $1.50 / 1,000 results
Searches Google Shopping for a query, scrapes product listings with all identifiers
Pricing
from $1.50 / 1,000 results
Rating
5.0
(1)
Developer
Abinash Sonowal
Maintained by CommunityActor stats
2
Bookmarked
3
Total users
1
Monthly active users
6 days ago
Last modified
Categories
Share
# apify push starts from .gitignore; re-include the build output it ignores,# and skip uploading source that the image doesn't use (Docker runs dist/ only).src/!dist/node_modules/dist/
.env.env.*!.env.example
.idea/.vscode/
.DS_StoreThumbs.db
apify_storage/storage/crawlee_storage/# Apify Node.js base image.FROM apify/actor-node:20
# Install ONLY the runtime dependencies, from the build-generated dist/package.json# (which contains no `workspace:` specs — npm rejects those).COPY dist/package.json ./package.jsonRUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version
# Copy the self-contained bundle (src/ + @gs/shared inlined by esbuild).COPY dist ./dist
CMD ["node", "dist/main.js"]{ "name": "@gs/shopping", "version": "0.0.1", "type": "module", "description": "Scrapes product listings from Google Shopping search results. Also exports its search API for reuse.", "engines": { "node": ">=18" }, "main": "src/search.js", "exports": { ".": "./src/search.js" }, "scripts": { "start": "node dist/main.js", "dev": "node src/main.js" }, "dependencies": { "apify": "^3.2.6", "cheerio": "^1.0.0", "got-scraping": "^4.0.6" }, "devDependencies": { "@gs/client": "workspace:*" }}{ "$schema": "https://apify.com/schemas/v1/actor.ide.json", "actorSpecification": 1, "name": "google-shopping", "title": "Google Shopping Scraper", "description": "Scrapes product listings from Google Shopping search results.", "version": "0.0", "buildTag": "latest", "input": "./input_schema.json", "dockerfile": "../Dockerfile"}{ "actorSpecification": 1, "fields": { "position": { "title": "Position", "type": "integer", "description": "Position in search results" }, "title": { "title": "Title", "type": "string", "nullable": true, "description": "Product name" }, "url": { "title": "URL", "type": "string", "description": "Google Shopping product link" }, "price": { "title": "Price", "type": "string", "nullable": true, "description": "Product price" }, "rating": { "title": "Rating", "type": "number", "nullable": true, "description": "Average rating out of 5" }, "review_count": { "title": "Review count", "type": "integer", "nullable": true, "description": "Total number of reviews" }, "second_hand_condition": { "title": "Condition", "type": "string", "nullable": true, "description": "Second-hand condition tag when present (pre-owned, refurbished, used); null for new items" }, "source": { "title": "Source", "type": "string", "nullable": true, "description": "Merchant name or seller source" }, "image": { "title": "Image", "type": "string", "nullable": true, "description": "Product image URL" } }}{ "$schema": "https://apify.com/schemas/v1/input.ide.json", "title": "Google Shopping Scraper", "type": "object", "schemaVersion": 1, "properties": { "query": { "title": "Search Query", "type": "string", "description": "Google Shopping search query (e.g. wireless headphones)", "default": "wireless headphones", "editor": "textfield" }, "country": { "title": "Country Code", "type": "string", "description": "Google country code for localized results (e.g. us, gb, in)", "default": "in", "editor": "textfield" } }, "required": ["query"]}{ "openapi": "3.0.0", "info": { "title": "Google Shopping Scraper", "version": "1.0.0", "description": "Scrapes product listings from Google Shopping including price, rating, reviews, and store info" }, "paths": { "/run-sync-get-dataset-items": { "post": { "operationId": "runScraper", "summary": "Run scraper and return products", "requestBody": { "required": true, "content": { "application/json": { "schema": { "type": "object", "required": ["query"], "properties": { "query": { "type": "string", "description": "Search query e.g. wireless headphones" }, "country": { "type": "string", "description": "2-letter country code e.g. in, us, gb", "default": "in" } } } } } }, "responses": { "200": { "description": "List of scraped products", "content": { "application/json": { "schema": { "type": "array", "items": { "type": "object", "properties": { "position": { "type": "integer" }, "title": { "type": "string", "nullable": true }, "url": { "type": "string", "format": "uri" }, "price": { "type": "string", "nullable": true }, "second_hand_condition": { "type": "string", "nullable": true, "enum": ["pre-owned", "refurbished", "used", null] }, "rating": { "type": "number", "nullable": true }, "review_count": { "type": "integer", "nullable": true }, "image": { "type": "string", "format": "uri", "nullable": true }, "source": { "type": "string", "nullable": true } } } } } } } } } } }}{ "actorSpecification": 1, "views": { "overview": { "title": "Products", "transformation": { "fields": ["position", "title", "price", "second_hand_condition", "rating", "review_count", "source", "url", "image"] }, "display": { "component": "table", "properties": { "title": { "label": "Title" }, "price": { "label": "Price" }, "second_hand_condition": { "label": "Condition" }, "rating": { "label": "Rating" }, "review_count": { "label": "Reviews" }, "source": { "label": "Source" }, "url": { "label": "URL", "format": "link" } } } } }}1// Bundled by esbuild (src/ + @gs/shared inlined). Do not edit — run `pnpm build`.2
3// shopping/src/main.js4import { Actor as Actor2, log as log2 } from "apify";5
6// shopping/src/search.js7import { Actor } from "apify";8import * as cheerio2 from "cheerio";9
10// client/proxy_http_client.js11import { log } from "apify";12import { gotScraping } from "got-scraping";13var MAX_RETRIES = 3;14var RETRY_BACKOFF_BASE_MS = 1e3;15var DEFAULT_HEADERS = {16 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",17 Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8",18 "Accept-Language": "en-IN,en;q=0.9",19 "Cache-Control": "no-cache",20 Pragma: "no-cache",21 Priority: "u=0, i",22 "Sec-Ch-Ua": '"Google Chrome";v="137", "Chromium";v="137", "Not/A)Brand";v="24"',23 "Sec-Ch-Ua-Mobile": "?0",24 "Sec-Ch-Ua-Platform": '"Windows"',25 "Sec-Fetch-Dest": "document",26 "Sec-Fetch-Mode": "navigate",27 "Sec-Fetch-Site": "none",28 "Sec-Fetch-User": "?1",29 "Upgrade-Insecure-Requests": "1"30};31var sleep = (ms) => new Promise((resolve) => {32 setTimeout(resolve, ms);33});34var ProxyHttpClient = class {35 constructor(proxyUrl, headers = null) {36 this.proxyUrl = proxyUrl;37 this.headers = headers || DEFAULT_HEADERS;38 }39 /**40 * @param {string} url41 * @returns {Promise<[string, string]>} `[htmlBody, finalUrl]`42 */43 async fetch(url) {44 let lastErr;45 for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) {46 try {47 const response = await gotScraping({48 url,49 proxyUrl: this.proxyUrl,50 headers: this.headers,51 useHeaderGenerator: false,52 https: { rejectUnauthorized: false },53 timeout: { request: 3e4 },54 retry: { limit: 0 }55 });56 return [response.body, response.url];57 } catch (err) {58 lastErr = err;59 if (attempt < MAX_RETRIES) {60 const delay = RETRY_BACKOFF_BASE_MS * 2 ** (attempt - 1);61 log.warning(62 `Proxy/HTTP error on attempt ${attempt}/${MAX_RETRIES} (${err.message}). Retrying in ${delay / 1e3}s...`63 );64 await sleep(delay);65 } else {66 log.error(`All ${MAX_RETRIES} attempts failed. Last error: ${err.message}`);67 }68 }69 }70 throw lastErr;71 }72};73
74// client/is_blocked.js75import * as cheerio from "cheerio";76function isBlocked(html) {77 if (html.toLowerCase().includes("g-recaptcha")) return true;78 const text2 = cheerio.load(html).text().toLowerCase();79 return [80 "before you continue",81 "unusual traffic",82 "verify you're human"83 ].some((s) => text2.includes(s));84}85
86// shopping/src/selectors.js87var SELECTORS = {88 /** Product cards: the older `.Ez5pwe` grid and the newer `.YBo8bb` list layout. */89 productCard: ".Ez5pwe, .YBo8bb",90 /** Inner container carrying the data-cid/gid/oid/iid/pid/rds attributes. */91 dataContainer: "[data-cid]",92 /** Product title. */93 title: ".gkQHve",94 /** Price. */95 price: ".lmQWe",96 /** Tag slot next to the price: condition ("Pre-owned") or discount ("5% off ₹69,900"). */97 priceTag: ".W0uRhb",98 /** Merchant / seller source name. */99 source: ".WJMUdc",100 /** Rating stars, carrying an aria-label like "Rated 4.5 out of 5". */101 rating: '[role="img"][aria-label*="Rated"]',102 /** Card containers inside the injected (jsl.dh) HTML, used for the fallback image map. */103 injectedCard: "[data-pid], .MUWJ8c, g-inner-card",104 /** Deferred-image placeholders whose real URL lives in the google.ldi map. */105 deferredImage: 'img[id^="dimg_"]'106};107
108// shopping/src/search.js109var OSHOP_BASE_URL = "https://www.google.com/search";110var SEARCH_BASE_URL = "http://www.google.com/search";111function buildSearchUrl(query, country, start = 0) {112 const params = new URLSearchParams({ q: query, tbm: "shop", hl: "en", gl: country });113 if (start) params.set("start", String(start));114 return `${SEARCH_BASE_URL}?${params.toString()}`;115}116async function searchProducts(proxyConfiguration, query, country) {117 const httpClient = new ProxyHttpClient(await proxyConfiguration.newUrl());118 const [html] = await httpClient.fetch(buildSearchUrl(query, country));119 if (isBlocked(html)) return { products: [], html, blocked: true };120 return { products: parseProducts(html, query, country), html, blocked: false };121}122async function runSearch(query, country) {123 const proxyConfiguration = await Actor.createProxyConfiguration({124 groups: ["GOOGLE_SERP"],125 countryCode: country.toUpperCase()126 });127 return searchProducts(proxyConfiguration, query, country);128}129function decodeJsString(raw) {130 try {131 return JSON.parse(`"${raw.replace(/\\x([0-9A-Fa-f]{2})/g, "\\u00$1")}"`);132 } catch {133 return "";134 }135}136function extractLdiMap(html) {137 const m = html.match(/google\.ldi\s*=\s*(\{.*?\});/s);138 if (!m) return {};139 try {140 return JSON.parse(m[1]);141 } catch {142 return {};143 }144}145function extractInjectedHtml(html) {146 let injected = "";147 const re = /jsl\.dh\([^,]+,\s*"((?:[^"\\]|\\.)*)"\s*\);/g;148 let m;149 while ((m = re.exec(html)) !== null) {150 injected += decodeJsString(m[1]);151 }152 return cheerio2.load(injected);153}154function ratingLabel($card) {155 return $card.find(SELECTORS.rating).first().attr("aria-label") || "";156}157function extractRating($card) {158 const m = ratingLabel($card).match(/Rated\s+([\d.]+)/);159 return m ? parseFloat(m[1]) : null;160}161function extractReviewCount($card) {162 const m = ratingLabel($card).match(/([\d,]+\.?\d*[kKmM]?)\s+(?:user\s+)?reviews?/);163 if (!m) return null;164 const raw = m[1].replace(/,/g, "");165 const suffix = raw.slice(-1).toLowerCase();166 if (suffix === "k") return Math.trunc(parseFloat(raw.slice(0, -1)) * 1e3);167 if (suffix === "m") return Math.trunc(parseFloat(raw.slice(0, -1)) * 1e6);168 return Math.trunc(parseFloat(raw));169}170function extractCondition($card) {171 const tags = $card.find(SELECTORS.priceTag);172 for (let i = 0; i < tags.length; i++) {173 const label = tags.eq(i).text().trim().toLowerCase();174 if (label === "refurbished" || label === "used") return label;175 if (label === "pre-owned" || label === "preowned") return "pre-owned";176 }177 return null;178}179function findImageInCard($, cardEl, ldiMap) {180 let found = null;181 $(cardEl).find("img").each((_, img) => {182 if (found) return;183 for (const url of [$(img).attr("src") || "", $(img).attr("data-src") || ""]) {184 if (url.startsWith("http") && url.includes("encrypted-tbn") && !url.includes("favicon")) {185 found = url;186 return;187 }188 }189 });190 if (found) return found;191 const dimg = $(cardEl).find(SELECTORS.deferredImage).first();192 const id = dimg.attr("id");193 if (id && ldiMap[id]?.startsWith("http")) return ldiMap[id];194 return null;195}196function buildProductUrl(query, country, { headlineOfferDocid, imageDocid, rds, pid, catalogid, gpcid }) {197 const prds = [`headlineOfferDocid:${headlineOfferDocid}`, `imageDocid:${imageDocid}`];198 if (catalogid) prds.push(`catalogid:${catalogid}`, `gpcid:${gpcid}`);199 if (pid) prds.push(`productid:${pid}`, "pvo:25");200 if (rds) prds.push(`rds:${rds}`);201 prds.push("pvt:hg");202 const params = new URLSearchParams({203 ibp: "oshop",204 q: query,205 prds: prds.join(","),206 hl: "en",207 gl: country,208 udm: "28"209 });210 if (pid) params.set("pvorigin", "25");211 return `${OSHOP_BASE_URL}?${params.toString()}`;212}213var text = ($el) => $el.first().text().trim() || null;214function parseProducts(html, query, country) {215 const $ = cheerio2.load(html);216 const ldiMap = extractLdiMap(html);217 const $inj = extractInjectedHtml(html);218 const injectedImages = {};219 $inj(SELECTORS.injectedCard).each((_, card) => {220 const img = findImageInCard($inj, card, ldiMap);221 if (!img) return;222 const pid = $inj(card).attr("data-pid");223 const title = text($inj(card).find(SELECTORS.title));224 if (pid) injectedImages[pid] = img;225 if (title) injectedImages[title] = img;226 });227 const products = [];228 const seen = /* @__PURE__ */ new Set();229 $(SELECTORS.productCard).each((_, cardEl) => {230 const $card = $(cardEl);231 const container = $card.find(SELECTORS.dataContainer).first();232 if (!container.length) return;233 const catalogid = container.attr("data-cid");234 const gpcid = container.attr("data-gid");235 const headlineOfferDocid = container.attr("data-oid");236 const imageDocid = container.attr("data-iid");237 const pid = container.attr("data-pid");238 let rds = container.attr("data-rds");239 if (!rds && gpcid) rds = `PC_${gpcid}|PROD_PC_${gpcid}`;240 const key = `${catalogid}|${pid}|${headlineOfferDocid}`;241 if (seen.has(key)) return;242 seen.add(key);243 let url = "N/A";244 if (catalogid && gpcid && headlineOfferDocid && imageDocid) {245 url = buildProductUrl(query, country, { headlineOfferDocid, imageDocid, rds, catalogid, gpcid });246 } else if (pid && headlineOfferDocid && imageDocid) {247 url = buildProductUrl(query, country, { headlineOfferDocid, imageDocid, rds, pid });248 }249 const title = text($card.find(SELECTORS.title));250 const image = findImageInCard($, cardEl, ldiMap) || pid && injectedImages[pid] || title && injectedImages[title] || null;251 products.push({252 position: products.length + 1,253 title,254 url,255 price: text($card.find(SELECTORS.price)),256 second_hand_condition: extractCondition($card),257 rating: extractRating($card),258 review_count: extractReviewCount($card),259 source: text($card.find(SELECTORS.source)),260 image261 });262 });263 return products;264}265
266// shopping/src/main.js267await Actor2.init();268try {269 const input = await Actor2.getInput() || {};270 const query = input.query;271 const country = input.country || "in";272 if (!query) throw new Error('Input field "query" is required');273 log2.info(`Searching for "${query}" in ${country}`);274 const { products, html, blocked } = await runSearch(query, country);275 log2.info(`Fetched ${html.length} bytes; parsed ${products.length} products`);276 if (blocked) {277 await Actor2.setValue("blocked-page.html", html, { contentType: "text/html" });278 await Actor2.fail('Blocked by Google (captcha / unusual traffic) \u2014 saved HTML as "blocked-page.html"');279 } else if (products.length) {280 await Actor2.pushData(products);281 log2.info(`Pushed ${products.length} products to dataset`);282 } else {283 await Actor2.setValue("search-page.html", html, { contentType: "text/html" });284 log2.warning('0 products parsed \u2014 saved raw HTML to key-value store as "search-page.html"');285 }286} finally {287 await Actor2.exit();288}{ "name": "@gs/shopping", "version": "0.0.1", "type": "module", "dependencies": { "apify": "^3.2.6", "cheerio": "^1.0.0", "got-scraping": "^4.0.6" }}