Pricing
Pay per usage
Pricing
Pay per usage
Rating
0.0
(0)
Developer
020 Om Salunke
Actor stats
0
Bookmarked
6
Total users
3
Monthly active users
10 days ago
Last modified
Categories
Share
{ "build": { "type": "dockerfile" }}# Specify the base Docker image. You can read more about# the available images at https://crawlee.dev/docs/guides/docker-images# You can also use any other image from Docker Hub.FROM apify/actor-node-playwright-chrome:22# Copy just package.json and package-lock.json# to speed up the build using Docker layer cache.COPY package*.json ./
# Install NPM packages, skip optional and development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version
# Next, copy the remaining files and directories with the source code.# Since we do this after NPM install, quick build will be really fast# for most source file changes.COPY . ./
# Run the image.CMD npm start --silent1import { Actor } from 'apify';2import { PlaywrightCrawler, Dataset } from 'crawlee';3
4// Initialize Actor5await Actor.init();6
7// ==================== INPUT CONFIGURATION ====================8const input = await Actor.getInput() ?? {};9
10const {11 pincode = '411001',12 searchUrls = [],13 searchQueries = [],14 maxProductsPerSearch = 100,15 proxyConfiguration = { useApifyProxy: false },16 maxRequestRetries = 3,17 navigationTimeout = 90000,18 headless = false,19 screenshotOnError = true,20 debugMode = false,21 scrollCount = 522} = input;23
24// ==================== CONSTANTS ====================25const USER_AGENTS = [26 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',27 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',28 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15',29 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'30];31
32// Global flag to track if location has been set33let locationSetGlobally = false;34
35// ==================== HELPER FUNCTIONS ====================36
37/**38 * Pick a random item from an array39 */40function pickRandom(arr) {41 return arr[Math.floor(Math.random() * arr.length)];42}43
44/**45 * Parse proxy URL into Playwright's proxy object46 */47function parseProxyUrl(proxyUrl) {48 try {49 const url = new URL(proxyUrl);50 const proxy = {51 server: `${url.protocol}//${url.hostname}${url.port ? `:${url.port}` : ''}`52 };53 if (url.username) proxy.username = decodeURIComponent(url.username);54 if (url.password) proxy.password = decodeURIComponent(url.password);55 return proxy;56 } catch (error) {57 console.error('Invalid proxy URL:', error.message);58 return null;59 }60}61
62/**63 * Delay execution64 */65const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));66
67// ==================== POPUP CLOSING FUNCTION ====================68
69/**70 * Close the location services popup that appears on JioMart71 */72async function closeLocationPopup(page, log) {73 try {74 log.info('🔔 Attempting to close location services popup...');75 76 // Check if popup exists77 const popup = page.locator('div.alcohol-popup').first();78 const popupCount = await popup.count();79 80 if (popupCount === 0) {81 log.info('ℹ️ No location popup detected');82 return true;83 }84 85 log.info('📋 Found location services popup - closing...');86 87 // Try clicking close button first88 const closeButtonSelectors = [89 'button#btn_location_close_icon',90 'button.close-privacy',91 'button.close-icon'92 ];93 94 for (const selector of closeButtonSelectors) {95 try {96 const closeBtn = page.locator(selector).first();97 if (await closeBtn.count() > 0) {98 await closeBtn.click({ timeout: 3000 });99 log.info(`✓ Clicked close button using: ${selector}`);100 await delay(800);101 102 // Verify popup is closed103 const stillVisible = await popup.count();104 if (stillVisible === 0) {105 log.info('✅ Location popup closed successfully');106 return true;107 }108 }109 } catch (error) {110 continue;111 }112 }113 114 // Fallback: click "Select Location Manually" button115 try {116 const selectLocBtn = page.locator('button#select_location_popup').first();117 if (await selectLocBtn.count() > 0) {118 await selectLocBtn.click({ timeout: 3000 });119 log.info('✓ Clicked "Select Location Manually" button');120 await delay(1000);121 log.info('✅ Location popup closed via manual selection');122 return true;123 }124 } catch (error) {125 log.warning(`Failed to click Select Location button: ${error.message}`);126 }127 128 log.warning('⚠️ Could not close location popup, but continuing...');129 return false;130 131 } catch (error) {132 log.error(`❌ Error closing popup: ${error.message}`);133 return false;134 }135}136
137// ==================== LOCATION SETTING FUNCTION ====================138
139/**140 * Set pincode location for JioMart141 */142/**143 * Set pincode location for JioMart - FIXED VERSION144 */145async function setPincodeLocation(page, log, targetPincode) {146 try {147 log.info(`🎯 Setting location to pincode: ${targetPincode}`);148 149 // Wait for page to be fully loaded150 await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});151 await delay(2000);152 153 // === STEP 1: Click on location/pincode button in header ===154 const locationSelectors = [155 'button#btn_pin_code_delivery',156 'button.header-main-pincode-address',157 'span#delivery_city_pincode'158 ];159 160 let locationButtonClicked = false;161 for (const selector of locationSelectors) {162 try {163 const button = page.locator(selector).first();164 if (await button.isVisible({ timeout: 3000 })) {165 await button.click({ timeout: 5000 });166 log.info(`✓ Clicked location button using: ${selector}`);167 locationButtonClicked = true;168 break;169 }170 } catch (error) {171 continue;172 }173 }174 175 if (!locationButtonClicked) {176 log.warning('⚠️ Could not find location button');177 return false;178 }179 180 await delay(2000);181 182 // === STEP 2: Wait for delivery popup to appear ===183 try {184 await page.waitForSelector('div#delivery_popup', { timeout: 5000, state: 'visible' });185 log.info('✓ Delivery popup opened');186 } catch (error) {187 log.warning('⚠️ Delivery popup did not appear');188 return false;189 }190 191 await delay(1500);192 193 // === STEP 3: Wait for initial content and click "Enter a pincode" ===194 // The initial content div needs to be displayed first195 try {196 // Sometimes the div is hidden by default, try to show it197 await page.evaluate(() => {198 const deliveryContent = document.querySelector('#delivery-content');199 if (deliveryContent && deliveryContent.style.display === 'none') {200 deliveryContent.style.display = 'block';201 }202 });203 } catch (e) {204 // Ignore if script fails205 }206 207 await delay(1000);208 209 // Now click the "Enter a pincode" button210 try {211 const enterPincodeBtn = page.locator('button#btn_enter_pincode').first();212 await enterPincodeBtn.waitFor({ state: 'visible', timeout: 5000 });213 await enterPincodeBtn.click({ timeout: 5000 });214 log.info('✓ Clicked "Enter a pincode" button');215 } catch (error) {216 log.error(`❌ Could not click "Enter a pincode" button: ${error.message}`);217 return false;218 }219 220 await delay(2000);221 222 // === STEP 4: Wait for pincode entry form to appear ===223 try {224 await page.waitForSelector('div#delivery_enter_pincode', { timeout: 5000, state: 'visible' });225 log.info('✓ Pincode entry form appeared');226 } catch (error) {227 log.error('❌ Pincode entry form did not appear');228 return false;229 }230 231 await delay(1000);232 233 // === STEP 5: Find and interact with pincode input ===234 const inputField = page.locator('input#rel_pincode').first();235 236 try {237 await inputField.waitFor({ state: 'visible', timeout: 5000 });238 log.info('✓ Found pincode input field');239 } catch (error) {240 log.error('❌ Could not find pincode input field');241 return false;242 }243 244 // === STEP 6: Clear and enter pincode ===245 try {246 // Focus on input247 await inputField.click();248 await delay(500);249 250 // Clear any existing value251 await inputField.fill('');252 await delay(300);253 254 // Triple-click to select all (if any text remains)255 await inputField.click({ clickCount: 3 });256 await delay(200);257 258 // Type the pincode259 await inputField.fill(targetPincode);260 await delay(500);261 262 // Verify the value was set263 const inputValue = await inputField.inputValue();264 log.info(`✓ Entered pincode: ${inputValue}`);265 266 if (inputValue !== targetPincode) {267 log.error(`❌ Pincode mismatch! Expected: ${targetPincode}, Got: ${inputValue}`);268 return false;269 }270 271 // Trigger events to ensure validation272 await page.evaluate((pincode) => {273 const input = document.querySelector('input#rel_pincode');274 if (input) {275 input.value = pincode;276 input.dispatchEvent(new Event('input', { bubbles: true }));277 input.dispatchEvent(new Event('change', { bubbles: true }));278 input.dispatchEvent(new Event('blur', { bubbles: true }));279 }280 }, targetPincode);281 282 log.info('✓ Triggered validation events');283 await delay(2000);284 285 } catch (error) {286 log.error(`❌ Failed to enter pincode: ${error.message}`);287 return false;288 }289 290 // === STEP 7: Wait for location to be detected ===291 try {292 // Wait for success message to appear293 await page.waitForSelector('div#delivery_pin_msg.field-success', { 294 timeout: 5000, 295 state: 'visible' 296 });297 298 const locationMessage = await page.locator('div#delivery_pin_msg').textContent();299 log.info(`✓ Location detected: ${locationMessage.trim()}`);300 } catch (error) {301 log.warning('⚠️ Location message not detected, checking if button is enabled...');302 }303 304 await delay(1000);305 306 // === STEP 8: Click Apply button ===307 try {308 const applyButton = page.locator('button#btn_pincode_submit').first();309 310 // Wait for button to be visible and enabled311 await applyButton.waitFor({ state: 'visible', timeout: 5000 });312 313 // Check if button is disabled314 const isDisabled = await applyButton.isDisabled();315 if (isDisabled) {316 log.warning('⚠️ Apply button is disabled, attempting to enable it...');317 318 // Force enable the button319 await page.evaluate(() => {320 const btn = document.querySelector('button#btn_pincode_submit');321 if (btn) {322 btn.disabled = false;323 btn.removeAttribute('disabled');324 btn.classList.remove('disabled');325 }326 });327 328 await delay(500);329 }330 331 // Click the button332 await applyButton.click({ force: true, timeout: 5000 });333 log.info('✓ Clicked Apply button');334 335 } catch (error) {336 log.error(`❌ Could not click Apply button: ${error.message}`);337 return false;338 }339 340 // === STEP 9: Wait for modal to close ===341 try {342 // Wait for the popup to disappear343 await page.waitForSelector('div#delivery_popup', { 344 state: 'hidden', 345 timeout: 10000 346 });347 log.info('✅ Location modal closed successfully');348 } catch (error) {349 // Check if modal is still visible350 const isStillVisible = await page.locator('div#delivery_popup').isVisible().catch(() => false);351 352 if (isStillVisible) {353 log.warning('⚠️ Modal still visible, trying to close manually...');354 355 // Try clicking the close button356 try {357 await page.locator('button#close_delivery_popup').click({ timeout: 2000 });358 await delay(1000);359 } catch (e) {360 // Try clicking backdrop361 try {362 await page.locator('div.backdrop').click({ timeout: 2000 });363 await delay(1000);364 } catch (e2) {365 log.error('❌ Could not close modal');366 return false;367 }368 }369 } else {370 log.info('✅ Modal closed (not visible)');371 }372 }373 374 // Wait for page to update with new location375 await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {});376 await delay(2000);377 378 // === STEP 10: Verify location is set ===379 try {380 const headerPincode = await page.locator('button#btn_pin_code_delivery, span#delivery_city_pincode')381 .first()382 .textContent()383 .catch(() => '');384 385 if (headerPincode.includes(targetPincode)) {386 log.info(`✅ Location verified in header: ${headerPincode}`);387 } else {388 log.info(`ℹ️ Location set, header shows: ${headerPincode}`);389 }390 } catch (e) {391 // Verification failed but location might still be set392 log.info('ℹ️ Could not verify location in header, but proceeding...');393 }394 395 log.info('✅ Pincode location set successfully');396 return true;397 398 } catch (error) {399 log.error(`❌ Error in setPincodeLocation: ${error.message}`);400 401 // Take screenshot on error for debugging402 try {403 const screenshot = await page.screenshot({ fullPage: true });404 await Actor.setValue(`pincode-error-${Date.now()}.png`, screenshot, { 405 contentType: 'image/png' 406 });407 log.info('📸 Error screenshot saved');408 } catch (e) {409 // Ignore screenshot errors410 }411 412 return false;413 }414}415
416// ==================== SCROLLING FUNCTION ====================417
418/**419 * Auto-scroll to load lazy-loaded content420 */421async function autoScroll(page, log, iterations = 5) {422 try {423 log.info(`🔄 Starting auto-scroll (${iterations} iterations)...`);424 425 for (let i = 0; i < iterations; i++) {426 await page.evaluate(() => window.scrollBy(0, window.innerHeight));427 await delay(1500);428 }429 430 // Scroll back to top431 await page.evaluate(() => window.scrollTo(0, 0));432 await delay(500);433 434 log.info('✓ Auto-scroll completed');435 return true;436 } catch (error) {437 log.warning(`⚠️ Auto-scroll failed: ${error.message}`);438 return false;439 }440}441
442// ==================== DEBUG FUNCTION ====================443
444/**445 * Debug page state with screenshots and HTML446 */447async function debugPageState(page, log, label = 'debug') {448 if (!debugMode) return;449 450 try {451 const timestamp = Date.now();452 453 // Take screenshot454 const screenshot = await page.screenshot({ fullPage: true });455 await Actor.setValue(`${label}-${timestamp}.png`, screenshot, { contentType: 'image/png' });456 457 // Save HTML458 const html = await page.content();459 await Actor.setValue(`${label}-${timestamp}.html`, html, { contentType: 'text/html' });460 461 // Get page info462 const pageInfo = await page.evaluate(() => ({463 url: window.location.href,464 title: document.title,465 viewport: { width: window.innerWidth, height: window.innerHeight },466 elementCounts: {467 productCards: document.querySelectorAll('li.ais-InfiniteHits-item').length,468 productLinks: document.querySelectorAll('a.plp-card-wrapper').length,469 images: document.querySelectorAll('img').length,470 prices: document.querySelectorAll('span.jm-heading-xxs').length471 }472 }));473 474 log.info(`📊 Page state: ${JSON.stringify(pageInfo, null, 2)}`);475 } catch (error) {476 log.error(`Debug failed: ${error.message}`);477 }478}479
480// ==================== WAIT FOR RESULTS ====================481
482/**483 * Wait for search results to appear484 */485async function waitForSearchResults(page, log) {486 const selectors = [487 'li.ais-InfiniteHits-item',488 'a.plp-card-wrapper',489 'div.plp-card-container',490 'div.plp-card-image'491 ];492
493 for (const selector of selectors) {494 try {495 await page.waitForSelector(selector, { timeout: 10000, state: 'visible' });496 const count = await page.locator(selector).count();497 if (count > 0) {498 log.info(`✓ Found ${count} product elements using: ${selector}`);499 await delay(1000);500 return true;501 }502 } catch (error) {503 continue;504 }505 }506
507 // Fallback: check page content508 try {509 const bodyText = await page.textContent('body');510 if (bodyText?.includes('₹') || /\bAdd\b/i.test(bodyText)) {511 log.info('✓ Found product indicators in page content');512 return true;513 }514 } catch (error) {515 // Ignore516 }517
518 log.warning('⚠️ No search results detected');519 return false;520}521
522// ==================== PRODUCT EXTRACTION ====================523
524/**525 * Extract products from JioMart search results526 */527async function extractJioMartProducts(page, log) {528 try {529 log.info('🔍 Extracting products...');530 531 const products = await page.evaluate(() => {532 const extractedProducts = [];533 const productItems = document.querySelectorAll('li.ais-InfiniteHits-item');534 535 productItems.forEach((item, index) => {536 try {537 // Get product link and data538 const productLink = item.querySelector('a.plp-card-wrapper');539 if (!productLink) return;540 541 const productUrl = productLink.href;542 const productId = productLink.getAttribute('data-objid');543 544 // Get GTM data attributes for accurate info545 const gtmData = item.querySelector('.gtmEvents');546 547 // Extract product name548 const nameEl = item.querySelector('div.plp-card-details-name');549 const productName = nameEl?.textContent?.trim() || 550 gtmData?.getAttribute('data-name') || 551 productLink.getAttribute('title');552 553 // Extract image554 const imgEl = item.querySelector('img.lazyloaded, img.lazyautosizes');555 let productImage = imgEl?.src || imgEl?.getAttribute('data-src');556 if (productImage && !productImage.startsWith('http')) {557 productImage = `https://www.jiomart.com${productImage}`;558 }559 560 // Extract current price561 const priceEl = item.querySelector('span.jm-heading-xxs');562 let currentPrice = null;563 if (priceEl) {564 const priceText = priceEl.textContent.trim();565 const match = priceText.match(/₹\s*(\d+(?:,\d+)*(?:\.\d+)?)/);566 currentPrice = match ? parseFloat(match[1].replace(/,/g, '')) : null;567 }568 569 // Fallback to GTM data570 if (!currentPrice && gtmData) {571 const gtmPrice = gtmData.getAttribute('data-price');572 currentPrice = gtmPrice ? parseFloat(gtmPrice) : null;573 }574 575 // Extract original price (strikethrough price)576 const originalPriceEl = item.querySelector('span.line-through');577 let originalPrice = null;578 if (originalPriceEl) {579 const priceText = originalPriceEl.textContent.trim();580 const match = priceText.match(/₹\s*(\d+(?:,\d+)*(?:\.\d+)?)/);581 originalPrice = match ? parseFloat(match[1].replace(/,/g, '')) : null;582 }583 584 // Extract discount percentage585 const discountEl = item.querySelector('span.jm-badge');586 let discountPercentage = null;587 if (discountEl) {588 const discountText = discountEl.textContent.trim();589 const match = discountText.match(/(\d+)%/);590 discountPercentage = match ? parseInt(match[1]) : null;591 }592 593 // Calculate discount if not available594 if (!discountPercentage && currentPrice && originalPrice && originalPrice > currentPrice) {595 discountPercentage = Math.round(((originalPrice - currentPrice) / originalPrice) * 100);596 }597 598 // Extract weight/size from product name or separate field599 let productWeight = null;600 const weightMatch = productName?.match(/(\d+\s*(?:g|kg|ml|l|gm|pack|pcs|piece))/i);601 if (weightMatch) {602 productWeight = weightMatch[1];603 }604 605 // Extract brand606 const brand = gtmData?.getAttribute('data-manu') || null;607 608 // Check if vegetarian609 const vegIcon = item.querySelector('img[src*="icon-veg"]');610 const isVegetarian = vegIcon !== null;611 612 // Check stock status613 const addButton = item.querySelector('button.addtocartbtn');614 const isOutOfStock = addButton?.hasAttribute('disabled') || false;615 616 if (productName && currentPrice) {617 extractedProducts.push({618 productId: productId || `jiomart-${index}`,619 productName,620 productImage,621 currentPrice,622 originalPrice: originalPrice || currentPrice,623 discountPercentage: discountPercentage || 0,624 productWeight,625 brand,626 isVegetarian,627 isOutOfStock,628 productUrl,629 scrapedAt: new Date().toISOString()630 });631 }632 } catch (error) {633 console.error(`Error extracting product ${index}:`, error);634 }635 });636 637 return extractedProducts;638 });639 640 log.info(`✅ Extracted ${products.length} products`);641 642 if (debugMode && products.length > 0) {643 log.info(`Sample product: ${JSON.stringify(products[0], null, 2)}`);644 }645 646 return products;647 } catch (error) {648 log.error(`❌ Error extracting products: ${error.message}`);649 return [];650 }651}652
653// ==================== PROXY CONFIGURATION ====================654
655const proxyConfig = proxyConfiguration?.useApifyProxy 656 ? await Actor.createProxyConfiguration(proxyConfiguration)657 : undefined;658
659const customProxyUrl = proxyConfiguration?.customProxyUrl || 660 proxyConfiguration?.proxyUrl || 661 proxyConfiguration?.proxy;662const launchProxy = customProxyUrl ? parseProxyUrl(customProxyUrl) : null;663
664// ==================== GENERATE SEARCH URLS ====================665
666const allSearchUrls = [667 ...searchUrls,668 ...searchQueries.map(query => 669 `https://www.jiomart.com/search?q=${encodeURIComponent(query)}`670 )671];672
673// ==================== CRAWLER INITIALIZATION ====================674
675const crawler = new PlaywrightCrawler({676 proxyConfiguration: proxyConfig,677 maxRequestRetries,678 navigationTimeoutSecs: navigationTimeout / 1000,679 headless:false,680 681 launchContext: {682 launchOptions: {683 args: [684 '--disable-blink-features=AutomationControlled',685 '--disable-dev-shm-usage',686 '--no-sandbox',687 '--disable-setuid-sandbox',688 '--disable-web-security',689 '--disable-features=IsolateOrigins,site-per-process'690 ],691 ...((!proxyConfig && launchProxy) ? { proxy: launchProxy } : {})692 }693 },694
695 preNavigationHooks: [696 async ({ page, log }) => {697 try {698 const userAgent = pickRandom(USER_AGENTS);699
700 await page.setExtraHTTPHeaders({701 'Accept-Language': 'en-US,en;q=0.9',702 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',703 'User-Agent': userAgent,704 'Cache-Control': 'no-cache',705 'Pragma': 'no-cache'706 });707
708 await page.setViewportSize({ width: 1920, height: 1080 });709
710 // Mask automation711 await page.addInitScript((ua) => {712 Object.defineProperty(navigator, 'webdriver', { get: () => false });713 Object.defineProperty(navigator, 'userAgent', { get: () => ua });714 Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });715 Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });716 717 // Override Chrome detection718 window.chrome = { runtime: {} };719 }, userAgent).catch(() => {});720 721 } catch (error) {722 log.error(`preNavigationHook error: ${error.message}`);723 }724 }725 ],726
727 async requestHandler({ page, request, log }) {728 const { url } = request;729 const isFirstRequest = request.userData?.isFirst || false;730
731 log.info(`🔍 Processing: ${url}`);732
733 try {734 // === STEP 0: Close location services popup ===735 if (isFirstRequest) {736 await closeLocationPopup(page, log);737 await delay(1000);738 }739 740 // === STEP 1: Set location on first request ===741 if (isFirstRequest && !locationSetGlobally) {742 log.info('🎯 First request - setting pincode location');743 744 const locationSet = await setPincodeLocation(page, log, pincode);745 746 if (locationSet) {747 locationSetGlobally = true;748 log.info('✅ Location set successfully');749 750 // Reload page to apply location751 await page.reload({ waitUntil: 'networkidle' });752 await delay(3000);753 } else {754 log.warning('⚠️ Failed to set location, continuing anyway');755 }756 }757 758 // === STEP 2: Wait for page load ===759 await page.waitForLoadState('domcontentloaded');760 await delay(3000);761 762 if (isFirstRequest) {763 await debugPageState(page, log, 'initial');764 }765 766 // === STEP 3: Close any popups/modals ===767 try {768 // Close any remaining delivery popup769 const closeBtn = page.locator('button#close_delivery_popup').first();770 if (await closeBtn.count() > 0 && await closeBtn.isVisible({ timeout: 2000 })) {771 await closeBtn.click();772 await delay(500);773 }774 } catch (error) {775 // No popup776 }777 778 // Close any other modals779 try {780 const backdrop = page.locator('div.backdrop').first();781 if (await backdrop.isVisible({ timeout: 2000 })) {782 await backdrop.click();783 await delay(500);784 }785 } catch (error) {786 // No backdrop787 }788 789 // === STEP 4: Wait for results ===790 const resultsFound = await waitForSearchResults(page, log);791 if (!resultsFound) {792 await debugPageState(page, log, 'no-results');793 log.warning('⚠️ No search results detected');794 }795 796 // === STEP 5: Scroll to load products ===797 await autoScroll(page, log, scrollCount);798 799 if (isFirstRequest) {800 await debugPageState(page, log, 'after-scroll');801 }802 803 // === STEP 6: Extract products ===804 const products = await extractJioMartProducts(page, log);805 806 if (products.length === 0) {807 log.error('❌ No products extracted');808 await debugPageState(page, log, 'no-products');809 return;810 }811 812 // === STEP 7: Save products ===813 const urlParams = new URL(url).searchParams;814 const searchQuery = urlParams.get('q') || urlParams.get('query') || 'direct_url';815 816 const productsToSave = products.slice(0, maxProductsPerSearch).map(product => ({817 ...product,818 searchQuery,819 searchUrl: url,820 platform: 'JioMart',821 pincode822 }));823 824 await Dataset.pushData(productsToSave);825 826 log.info(`✅ Saved ${productsToSave.length} products for "${searchQuery}" (Pincode: ${pincode})`);827
828 } catch (error) {829 log.error(`❌ Error processing ${url}: ${error.message}`);830 831 if (screenshotOnError) {832 try {833 const screenshot = await page.screenshot({ fullPage: true });834 const timestamp = Date.now();835 await Actor.setValue(`error-${timestamp}.png`, screenshot, { contentType: 'image/png' });836 } catch (e) {837 log.error(`Screenshot failed: ${e.message}`);838 }839 }840 841 throw error;842 }843 },844
845 failedRequestHandler: async ({ request, log }) => {846 log.error(`❌ Request failed: ${request.url}`);847 848 const failedUrls = await Actor.getValue('FAILED_URLS') || [];849 failedUrls.push({850 url: request.url,851 timestamp: new Date().toISOString(),852 error: request.errorMessages?.join(', ')853 });854 await Actor.setValue('FAILED_URLS', failedUrls);855 }856});857
858// ==================== START CRAWLER ====================859
860if (allSearchUrls.length > 0) {861 console.log('\n' + '='.repeat(60));862 console.log('🚀 JIOMART SCRAPER STARTED');863 console.log('='.repeat(60));864 console.log(`📍 Pincode: ${pincode}`);865 console.log(`🔍 Search URLs: ${allSearchUrls.length}`);866 console.log(`📊 Max products per search: ${maxProductsPerSearch}`);867 console.log(`📜 Scroll iterations: ${scrollCount}`);868 console.log(`🐛 Debug mode: ${debugMode}`);869 console.log(`👁️ Headless: ${headless}`);870 console.log('='.repeat(60) + '\n');871 872 const searchRequests = allSearchUrls.map((url, index) => ({ 873 url, 874 userData: { isFirst: index === 0 } 875 }));876 877 console.log('🔍 URLs to process:\n');878 searchRequests.forEach((req, idx) => {879 console.log(` ${idx + 1}. ${req.url}${idx === 0 ? ' 📍 (will set location)' : ''}`);880 });881 console.log('');882 883 await crawler.run(searchRequests);884 885 console.log('\n' + '='.repeat(60));886 console.log('✅ SCRAPING COMPLETED');887 console.log('='.repeat(60));888 console.log('📁 Results: storage/datasets/default/');889 console.log('📸 Debug files: storage/key_value_stores/default/');890 console.log('='.repeat(60) + '\n');891} else {892 console.log('\n' + '='.repeat(60));893 console.log('❌ NO SEARCH URLS PROVIDED');894 console.log('='.repeat(60));895 console.log('Please provide either "searchUrls" or "searchQueries" in input.json');896 console.log('='.repeat(60) + '\n');897}898
899await Actor.exit();1import { Actor } from 'apify';2import { PlaywrightCrawler, Dataset } from 'crawlee';3
4// Initialize Actor5await Actor.init();6
7// ==================== INPUT CONFIGURATION ====================8const input = await Actor.getInput() ?? {};9
10const {11 pincode = '411001',12 searchUrls = [],13 searchQueries = [],14 maxProductsPerSearch = 100,15 proxyConfiguration = { useApifyProxy: false },16 maxRequestRetries = 3,17 navigationTimeout = 90000,18 headless = false,19 screenshotOnError = true,20 debugMode = false,21 scrollCount = 522} = input;23
24// ==================== CONSTANTS ====================25const USER_AGENTS = [26 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',27 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',28 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15',29 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'30];31
32// Global flag to track if location has been set33let locationSetGlobally = false;34
35// ==================== HELPER FUNCTIONS ====================36
37/**38 * Pick a random item from an array39 */40function pickRandom(arr) {41 return arr[Math.floor(Math.random() * arr.length)];42}43
44/**45 * Parse proxy URL into Playwright's proxy object46 */47function parseProxyUrl(proxyUrl) {48 try {49 const url = new URL(proxyUrl);50 const proxy = {51 server: `${url.protocol}//${url.hostname}${url.port ? `:${url.port}` : ''}`52 };53 if (url.username) proxy.username = decodeURIComponent(url.username);54 if (url.password) proxy.password = decodeURIComponent(url.password);55 return proxy;56 } catch (error) {57 console.error('Invalid proxy URL:', error.message);58 return null;59 }60}61
62/**63 * Delay execution64 */65const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));66
67// ==================== POPUP CLOSING FUNCTION ====================68
69/**70 * Close the location services popup that appears on JioMart71 */72async function closeLocationPopup(page, log) {73 try {74 log.info('🔔 Attempting to close location services popup...');75 76 // Check if popup exists77 const popup = page.locator('div.alcohol-popup').first();78 const popupCount = await popup.count();79 80 if (popupCount === 0) {81 log.info('ℹ️ No location popup detected');82 return true;83 }84 85 log.info('📋 Found location services popup - closing...');86 87 // Try clicking close button first88 const closeButtonSelectors = [89 'button#btn_location_close_icon',90 'button.close-privacy',91 'button.close-icon'92 ];93 94 for (const selector of closeButtonSelectors) {95 try {96 const closeBtn = page.locator(selector).first();97 if (await closeBtn.count() > 0) {98 await closeBtn.click({ timeout: 3000 });99 log.info(`✓ Clicked close button using: ${selector}`);100 await delay(800);101 102 // Verify popup is closed103 const stillVisible = await popup.count();104 if (stillVisible === 0) {105 log.info('✅ Location popup closed successfully');106 return true;107 }108 }109 } catch (error) {110 continue;111 }112 }113 114 // Fallback: click "Select Location Manually" button115 try {116 const selectLocBtn = page.locator('button#select_location_popup').first();117 if (await selectLocBtn.count() > 0) {118 await selectLocBtn.click({ timeout: 3000 });119 log.info('✓ Clicked "Select Location Manually" button');120 await delay(1000);121 log.info('✅ Location popup closed via manual selection');122 return true;123 }124 } catch (error) {125 log.warning(`Failed to click Select Location button: ${error.message}`);126 }127 128 log.warning('⚠️ Could not close location popup, but continuing...');129 return false;130 131 } catch (error) {132 log.error(`❌ Error closing popup: ${error.message}`);133 return false;134 }135}136
137// ==================== LOCATION SETTING FUNCTION ====================138
139/**140 * Set pincode location for JioMart141 */142/**143 * Set pincode location for JioMart - FIXED VERSION144 */145async function setPincodeLocation(page, log, targetPincode) {146 try {147 log.info(`🎯 Setting location to pincode: ${targetPincode}`);148 149 // Wait for page to be fully loaded150 await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});151 await delay(2000);152 153 // === STEP 1: Click on location/pincode button in header ===154 const locationSelectors = [155 'button#btn_pin_code_delivery',156 'button.header-main-pincode-address',157 'span#delivery_city_pincode'158 ];159 160 let locationButtonClicked = false;161 for (const selector of locationSelectors) {162 try {163 const button = page.locator(selector).first();164 if (await button.isVisible({ timeout: 3000 })) {165 await button.click({ timeout: 5000 });166 log.info(`✓ Clicked location button using: ${selector}`);167 locationButtonClicked = true;168 break;169 }170 } catch (error) {171 continue;172 }173 }174 175 if (!locationButtonClicked) {176 log.warning('⚠️ Could not find location button');177 return false;178 }179 180 await delay(2000);181 182 // === STEP 2: Wait for delivery popup to appear ===183 try {184 await page.waitForSelector('div#delivery_popup', { timeout: 5000, state: 'visible' });185 log.info('✓ Delivery popup opened');186 } catch (error) {187 log.warning('⚠️ Delivery popup did not appear');188 return false;189 }190 191 await delay(1500);192 193 // === STEP 3: Wait for initial content and click "Enter a pincode" ===194 // The initial content div needs to be displayed first195 try {196 // Sometimes the div is hidden by default, try to show it197 await page.evaluate(() => {198 const deliveryContent = document.querySelector('#delivery-content');199 if (deliveryContent && deliveryContent.style.display === 'none') {200 deliveryContent.style.display = 'block';201 }202 });203 } catch (e) {204 // Ignore if script fails205 }206 207 await delay(1000);208 209 // Now click the "Enter a pincode" button210 try {211 const enterPincodeBtn = page.locator('button#btn_enter_pincode').first();212 await enterPincodeBtn.waitFor({ state: 'visible', timeout: 5000 });213 await enterPincodeBtn.click({ timeout: 5000 });214 log.info('✓ Clicked "Enter a pincode" button');215 } catch (error) {216 log.error(`❌ Could not click "Enter a pincode" button: ${error.message}`);217 return false;218 }219 220 await delay(2000);221 222 // === STEP 4: Wait for pincode entry form to appear ===223 try {224 await page.waitForSelector('div#delivery_enter_pincode', { timeout: 5000, state: 'visible' });225 log.info('✓ Pincode entry form appeared');226 } catch (error) {227 log.error('❌ Pincode entry form did not appear');228 return false;229 }230 231 await delay(1000);232 233 // === STEP 5: Find and interact with pincode input ===234 const inputField = page.locator('input#rel_pincode').first();235 236 try {237 await inputField.waitFor({ state: 'visible', timeout: 5000 });238 log.info('✓ Found pincode input field');239 } catch (error) {240 log.error('❌ Could not find pincode input field');241 return false;242 }243 244 // === STEP 6: Clear and enter pincode ===245 try {246 // Focus on input247 await inputField.click();248 await delay(500);249 250 // Clear any existing value251 await inputField.fill('');252 await delay(300);253 254 // Triple-click to select all (if any text remains)255 await inputField.click({ clickCount: 3 });256 await delay(200);257 258 // Type the pincode259 await inputField.fill(targetPincode);260 await delay(500);261 262 // Verify the value was set263 const inputValue = await inputField.inputValue();264 log.info(`✓ Entered pincode: ${inputValue}`);265 266 if (inputValue !== targetPincode) {267 log.error(`❌ Pincode mismatch! Expected: ${targetPincode}, Got: ${inputValue}`);268 return false;269 }270 271 // Trigger events to ensure validation272 await page.evaluate((pincode) => {273 const input = document.querySelector('input#rel_pincode');274 if (input) {275 input.value = pincode;276 input.dispatchEvent(new Event('input', { bubbles: true }));277 input.dispatchEvent(new Event('change', { bubbles: true }));278 input.dispatchEvent(new Event('blur', { bubbles: true }));279 }280 }, targetPincode);281 282 log.info('✓ Triggered validation events');283 await delay(2000);284 285 } catch (error) {286 log.error(`❌ Failed to enter pincode: ${error.message}`);287 return false;288 }289 290 // === STEP 7: Wait for location to be detected ===291 try {292 // Wait for success message to appear293 await page.waitForSelector('div#delivery_pin_msg.field-success', { 294 timeout: 5000, 295 state: 'visible' 296 });297 298 const locationMessage = await page.locator('div#delivery_pin_msg').textContent();299 log.info(`✓ Location detected: ${locationMessage.trim()}`);300 } catch (error) {301 log.warning('⚠️ Location message not detected, checking if button is enabled...');302 }303 304 await delay(1000);305 306 // === STEP 8: Click Apply button ===307 try {308 const applyButton = page.locator('button#btn_pincode_submit').first();309 310 // Wait for button to be visible and enabled311 await applyButton.waitFor({ state: 'visible', timeout: 5000 });312 313 // Check if button is disabled314 const isDisabled = await applyButton.isDisabled();315 if (isDisabled) {316 log.warning('⚠️ Apply button is disabled, attempting to enable it...');317 318 // Force enable the button319 await page.evaluate(() => {320 const btn = document.querySelector('button#btn_pincode_submit');321 if (btn) {322 btn.disabled = false;323 btn.removeAttribute('disabled');324 btn.classList.remove('disabled');325 }326 });327 328 await delay(500);329 }330 331 // Click the button332 await applyButton.click({ force: true, timeout: 5000 });333 log.info('✓ Clicked Apply button');334 335 } catch (error) {336 log.error(`❌ Could not click Apply button: ${error.message}`);337 return false;338 }339 340 // === STEP 9: Wait for modal to close ===341 try {342 // Wait for the popup to disappear343 await page.waitForSelector('div#delivery_popup', { 344 state: 'hidden', 345 timeout: 10000 346 });347 log.info('✅ Location modal closed successfully');348 } catch (error) {349 // Check if modal is still visible350 const isStillVisible = await page.locator('div#delivery_popup').isVisible().catch(() => false);351 352 if (isStillVisible) {353 log.warning('⚠️ Modal still visible, trying to close manually...');354 355 // Try clicking the close button356 try {357 await page.locator('button#close_delivery_popup').click({ timeout: 2000 });358 await delay(1000);359 } catch (e) {360 // Try clicking backdrop361 try {362 await page.locator('div.backdrop').click({ timeout: 2000 });363 await delay(1000);364 } catch (e2) {365 log.error('❌ Could not close modal');366 return false;367 }368 }369 } else {370 log.info('✅ Modal closed (not visible)');371 }372 }373 374 // Wait for page to update with new location375 await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {});376 await delay(2000);377 378 // === STEP 10: Verify location is set ===379 try {380 const headerPincode = await page.locator('button#btn_pin_code_delivery, span#delivery_city_pincode')381 .first()382 .textContent()383 .catch(() => '');384 385 if (headerPincode.includes(targetPincode)) {386 log.info(`✅ Location verified in header: ${headerPincode}`);387 } else {388 log.info(`ℹ️ Location set, header shows: ${headerPincode}`);389 }390 } catch (e) {391 // Verification failed but location might still be set392 log.info('ℹ️ Could not verify location in header, but proceeding...');393 }394 395 log.info('✅ Pincode location set successfully');396 return true;397 398 } catch (error) {399 log.error(`❌ Error in setPincodeLocation: ${error.message}`);400 401 // Take screenshot on error for debugging402 try {403 const screenshot = await page.screenshot({ fullPage: true });404 await Actor.setValue(`pincode-error-${Date.now()}.png`, screenshot, { 405 contentType: 'image/png' 406 });407 log.info('📸 Error screenshot saved');408 } catch (e) {409 // Ignore screenshot errors410 }411 412 return false;413 }414}415
416// ==================== SCROLLING FUNCTION ====================417
418/**419 * Auto-scroll to load lazy-loaded content420 */421async function autoScroll(page, log, iterations = 5) {422 try {423 log.info(`🔄 Starting auto-scroll (${iterations} iterations)...`);424 425 for (let i = 0; i < iterations; i++) {426 await page.evaluate(() => window.scrollBy(0, window.innerHeight));427 await delay(1500);428 }429 430 // Scroll back to top431 await page.evaluate(() => window.scrollTo(0, 0));432 await delay(500);433 434 log.info('✓ Auto-scroll completed');435 return true;436 } catch (error) {437 log.warning(`⚠️ Auto-scroll failed: ${error.message}`);438 return false;439 }440}441
442// ==================== DEBUG FUNCTION ====================443
444/**445 * Debug page state with screenshots and HTML446 */447async function debugPageState(page, log, label = 'debug') {448 if (!debugMode) return;449 450 try {451 const timestamp = Date.now();452 453 // Take screenshot454 const screenshot = await page.screenshot({ fullPage: true });455 await Actor.setValue(`${label}-${timestamp}.png`, screenshot, { contentType: 'image/png' });456 457 // Save HTML458 const html = await page.content();459 await Actor.setValue(`${label}-${timestamp}.html`, html, { contentType: 'text/html' });460 461 // Get page info462 const pageInfo = await page.evaluate(() => ({463 url: window.location.href,464 title: document.title,465 viewport: { width: window.innerWidth, height: window.innerHeight },466 elementCounts: {467 productCards: document.querySelectorAll('li.ais-InfiniteHits-item').length,468 productLinks: document.querySelectorAll('a.plp-card-wrapper').length,469 images: document.querySelectorAll('img').length,470 prices: document.querySelectorAll('span.jm-heading-xxs').length471 }472 }));473 474 log.info(`📊 Page state: ${JSON.stringify(pageInfo, null, 2)}`);475 } catch (error) {476 log.error(`Debug failed: ${error.message}`);477 }478}479
480// ==================== WAIT FOR RESULTS ====================481
482/**483 * Wait for search results to appear484 */485async function waitForSearchResults(page, log) {486 const selectors = [487 'li.ais-InfiniteHits-item',488 'a.plp-card-wrapper',489 'div.plp-card-container',490 'div.plp-card-image'491 ];492
493 for (const selector of selectors) {494 try {495 await page.waitForSelector(selector, { timeout: 10000, state: 'visible' });496 const count = await page.locator(selector).count();497 if (count > 0) {498 log.info(`✓ Found ${count} product elements using: ${selector}`);499 await delay(1000);500 return true;501 }502 } catch (error) {503 continue;504 }505 }506
507 // Fallback: check page content508 try {509 const bodyText = await page.textContent('body');510 if (bodyText?.includes('₹') || /\bAdd\b/i.test(bodyText)) {511 log.info('✓ Found product indicators in page content');512 return true;513 }514 } catch (error) {515 // Ignore516 }517
518 log.warning('⚠️ No search results detected');519 return false;520}521
522// ==================== PRODUCT EXTRACTION ====================523
524/**525 * Extract products from JioMart search results526 */527async function extractJioMartProducts(page, log) {528 try {529 log.info('🔍 Extracting products...');530 531 const products = await page.evaluate(() => {532 const extractedProducts = [];533 const productItems = document.querySelectorAll('li.ais-InfiniteHits-item');534 535 productItems.forEach((item, index) => {536 try {537 // Get product link and data538 const productLink = item.querySelector('a.plp-card-wrapper');539 if (!productLink) return;540 541 const productUrl = productLink.href;542 const productId = productLink.getAttribute('data-objid');543 544 // Get GTM data attributes for accurate info545 const gtmData = item.querySelector('.gtmEvents');546 547 // Extract product name548 const nameEl = item.querySelector('div.plp-card-details-name');549 const productName = nameEl?.textContent?.trim() || 550 gtmData?.getAttribute('data-name') || 551 productLink.getAttribute('title');552 553 // Extract image554 const imgEl = item.querySelector('img.lazyloaded, img.lazyautosizes');555 let productImage = imgEl?.src || imgEl?.getAttribute('data-src');556 if (productImage && !productImage.startsWith('http')) {557 productImage = `https://www.jiomart.com${productImage}`;558 }559 560 // Extract current price561 const priceEl = item.querySelector('span.jm-heading-xxs');562 let currentPrice = null;563 if (priceEl) {564 const priceText = priceEl.textContent.trim();565 const match = priceText.match(/₹\s*(\d+(?:,\d+)*(?:\.\d+)?)/);566 currentPrice = match ? parseFloat(match[1].replace(/,/g, '')) : null;567 }568 569 // Fallback to GTM data570 if (!currentPrice && gtmData) {571 const gtmPrice = gtmData.getAttribute('data-price');572 currentPrice = gtmPrice ? parseFloat(gtmPrice) : null;573 }574 575 // Extract original price (strikethrough price)576 const originalPriceEl = item.querySelector('span.line-through');577 let originalPrice = null;578 if (originalPriceEl) {579 const priceText = originalPriceEl.textContent.trim();580 const match = priceText.match(/₹\s*(\d+(?:,\d+)*(?:\.\d+)?)/);581 originalPrice = match ? parseFloat(match[1].replace(/,/g, '')) : null;582 }583 584 // Extract discount percentage585 const discountEl = item.querySelector('span.jm-badge');586 let discountPercentage = null;587 if (discountEl) {588 const discountText = discountEl.textContent.trim();589 const match = discountText.match(/(\d+)%/);590 discountPercentage = match ? parseInt(match[1]) : null;591 }592 593 // Calculate discount if not available594 if (!discountPercentage && currentPrice && originalPrice && originalPrice > currentPrice) {595 discountPercentage = Math.round(((originalPrice - currentPrice) / originalPrice) * 100);596 }597 598 // Extract weight/size from product name or separate field599 let productWeight = null;600 const weightMatch = productName?.match(/(\d+\s*(?:g|kg|ml|l|gm|pack|pcs|piece))/i);601 if (weightMatch) {602 productWeight = weightMatch[1];603 }604 605 // Extract brand606 const brand = gtmData?.getAttribute('data-manu') || null;607 608 // Check if vegetarian609 const vegIcon = item.querySelector('img[src*="icon-veg"]');610 const isVegetarian = vegIcon !== null;611 612 // Check stock status613 const addButton = item.querySelector('button.addtocartbtn');614 const isOutOfStock = addButton?.hasAttribute('disabled') || false;615 616 if (productName && currentPrice) {617 extractedProducts.push({618 productId: productId || `jiomart-${index}`,619 productName,620 productImage,621 currentPrice,622 originalPrice: originalPrice || currentPrice,623 discountPercentage: discountPercentage || 0,624 productWeight,625 brand,626 isVegetarian,627 isOutOfStock,628 productUrl,629 scrapedAt: new Date().toISOString()630 });631 }632 } catch (error) {633 console.error(`Error extracting product ${index}:`, error);634 }635 });636 637 return extractedProducts;638 });639 640 log.info(`✅ Extracted ${products.length} products`);641 642 if (debugMode && products.length > 0) {643 log.info(`Sample product: ${JSON.stringify(products[0], null, 2)}`);644 }645 646 return products;647 } catch (error) {648 log.error(`❌ Error extracting products: ${error.message}`);649 return [];650 }651}652
653// ==================== PROXY CONFIGURATION ====================654
655const proxyConfig = proxyConfiguration?.useApifyProxy 656 ? await Actor.createProxyConfiguration(proxyConfiguration)657 : undefined;658
659const customProxyUrl = proxyConfiguration?.customProxyUrl || 660 proxyConfiguration?.proxyUrl || 661 proxyConfiguration?.proxy;662const launchProxy = customProxyUrl ? parseProxyUrl(customProxyUrl) : null;663
664// ==================== GENERATE SEARCH URLS ====================665
666const allSearchUrls = [667 ...searchUrls,668 ...searchQueries.map(query => 669 `https://www.jiomart.com/search?q=${encodeURIComponent(query)}`670 )671];672
673// ==================== CRAWLER INITIALIZATION ====================674
675const crawler = new PlaywrightCrawler({676 proxyConfiguration: proxyConfig,677 maxRequestRetries,678 navigationTimeoutSecs: navigationTimeout / 1000,679 headless:false,680 681 launchContext: {682 launchOptions: {683 args: [684 '--disable-blink-features=AutomationControlled',685 '--disable-dev-shm-usage',686 '--no-sandbox',687 '--disable-setuid-sandbox',688 '--disable-web-security',689 '--disable-features=IsolateOrigins,site-per-process'690 ],691 ...((!proxyConfig && launchProxy) ? { proxy: launchProxy } : {})692 }693 },694
695 preNavigationHooks: [696 async ({ page, log }) => {697 try {698 const userAgent = pickRandom(USER_AGENTS);699
700 await page.setExtraHTTPHeaders({701 'Accept-Language': 'en-US,en;q=0.9',702 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',703 'User-Agent': userAgent,704 'Cache-Control': 'no-cache',705 'Pragma': 'no-cache'706 });707
708 await page.setViewportSize({ width: 1920, height: 1080 });709
710 // Mask automation711 await page.addInitScript((ua) => {712 Object.defineProperty(navigator, 'webdriver', { get: () => false });713 Object.defineProperty(navigator, 'userAgent', { get: () => ua });714 Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });715 Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });716 717 // Override Chrome detection718 window.chrome = { runtime: {} };719 }, userAgent).catch(() => {});720 721 } catch (error) {722 log.error(`preNavigationHook error: ${error.message}`);723 }724 }725 ],726
727 async requestHandler({ page, request, log }) {728 const { url } = request;729 const isFirstRequest = request.userData?.isFirst || false;730
731 log.info(`🔍 Processing: ${url}`);732
733 try {734 // === STEP 0: Close location services popup ===735 if (isFirstRequest) {736 await closeLocationPopup(page, log);737 await delay(1000);738 }739 740 // === STEP 1: Set location on first request ===741 if (isFirstRequest && !locationSetGlobally) {742 log.info('🎯 First request - setting pincode location');743 744 const locationSet = await setPincodeLocation(page, log, pincode);745 746 if (locationSet) {747 locationSetGlobally = true;748 log.info('✅ Location set successfully');749 750 // Reload page to apply location751 await page.reload({ waitUntil: 'networkidle' });752 await delay(3000);753 } else {754 log.warning('⚠️ Failed to set location, continuing anyway');755 }756 }757 758 // === STEP 2: Wait for page load ===759 await page.waitForLoadState('domcontentloaded');760 await delay(3000);761 762 if (isFirstRequest) {763 await debugPageState(page, log, 'initial');764 }765 766 // === STEP 3: Close any popups/modals ===767 try {768 // Close any remaining delivery popup769 const closeBtn = page.locator('button#close_delivery_popup').first();770 if (await closeBtn.count() > 0 && await closeBtn.isVisible({ timeout: 2000 })) {771 await closeBtn.click();772 await delay(500);773 }774 } catch (error) {775 // No popup776 }777 778 // Close any other modals779 try {780 const backdrop = page.locator('div.backdrop').first();781 if (await backdrop.isVisible({ timeout: 2000 })) {782 await backdrop.click();783 await delay(500);784 }785 } catch (error) {786 // No backdrop787 }788 789 // === STEP 4: Wait for results ===790 const resultsFound = await waitForSearchResults(page, log);791 if (!resultsFound) {792 await debugPageState(page, log, 'no-results');793 log.warning('⚠️ No search results detected');794 }795 796 // === STEP 5: Scroll to load products ===797 await autoScroll(page, log, scrollCount);798 799 if (isFirstRequest) {800 await debugPageState(page, log, 'after-scroll');801 }802 803 // === STEP 6: Extract products ===804 const products = await extractJioMartProducts(page, log);805 806 if (products.length === 0) {807 log.error('❌ No products extracted');808 await debugPageState(page, log, 'no-products');809 return;810 }811 812 // === STEP 7: Save products ===813 const urlParams = new URL(url).searchParams;814 const searchQuery = urlParams.get('q') || urlParams.get('query') || 'direct_url';815 816 const productsToSave = products.slice(0, maxProductsPerSearch).map(product => ({817 ...product,818 searchQuery,819 searchUrl: url,820 platform: 'JioMart',821 pincode822 }));823 824 await Dataset.pushData(productsToSave);825 826 log.info(`✅ Saved ${productsToSave.length} products for "${searchQuery}" (Pincode: ${pincode})`);827
828 } catch (error) {829 log.error(`❌ Error processing ${url}: ${error.message}`);830 831 if (screenshotOnError) {832 try {833 const screenshot = await page.screenshot({ fullPage: true });834 const timestamp = Date.now();835 await Actor.setValue(`error-${timestamp}.png`, screenshot, { contentType: 'image/png' });836 } catch (e) {837 log.error(`Screenshot failed: ${e.message}`);838 }839 }840 841 throw error;842 }843 },844
845 failedRequestHandler: async ({ request, log }) => {846 log.error(`❌ Request failed: ${request.url}`);847 848 const failedUrls = await Actor.getValue('FAILED_URLS') || [];849 failedUrls.push({850 url: request.url,851 timestamp: new Date().toISOString(),852 error: request.errorMessages?.join(', ')853 });854 await Actor.setValue('FAILED_URLS', failedUrls);855 }856});857
858// ==================== START CRAWLER ====================859
860if (allSearchUrls.length > 0) {861 console.log('\n' + '='.repeat(60));862 console.log('🚀 JIOMART SCRAPER STARTED');863 console.log('='.repeat(60));864 console.log(`📍 Pincode: ${pincode}`);865 console.log(`🔍 Search URLs: ${allSearchUrls.length}`);866 console.log(`📊 Max products per search: ${maxProductsPerSearch}`);867 console.log(`📜 Scroll iterations: ${scrollCount}`);868 console.log(`🐛 Debug mode: ${debugMode}`);869 console.log(`👁️ Headless: ${headless}`);870 console.log('='.repeat(60) + '\n');871 872 const searchRequests = allSearchUrls.map((url, index) => ({ 873 url, 874 userData: { isFirst: index === 0 } 875 }));876 877 console.log('🔍 URLs to process:\n');878 searchRequests.forEach((req, idx) => {879 console.log(` ${idx + 1}. ${req.url}${idx === 0 ? ' 📍 (will set location)' : ''}`);880 });881 console.log('');882 883 await crawler.run(searchRequests);884 885 console.log('\n' + '='.repeat(60));886 console.log('✅ SCRAPING COMPLETED');887 console.log('='.repeat(60));888 console.log('📁 Results: storage/datasets/default/');889 console.log('📸 Debug files: storage/key_value_stores/default/');890 console.log('='.repeat(60) + '\n');891} else {892 console.log('\n' + '='.repeat(60));893 console.log('❌ NO SEARCH URLS PROVIDED');894 console.log('='.repeat(60));895 console.log('Please provide either "searchUrls" or "searchQueries" in input.json');896 console.log('='.repeat(60) + '\n');897}898
899await Actor.exit();1import { Actor } from 'apify';2import { PlaywrightCrawler, Dataset, log } from 'crawlee';3
4// Initialize Actor5await Actor.init();6
7// ==================== INPUT CONFIGURATION ====================8const input = await Actor.getInput() ?? {};9const {10 pincode = '120001', // Default to Mumbai11 searchQueries = ['kurkure'],12 searchUrls = [],13 maxProductsPerSearch = 100,14 maxRequestRetries = 3,15 navigationTimeout = 60000,16 headless = false,17 proxyConfiguration = { useApifyProxy: false },18} = input;19
20// ==================== CONSTANTS & SELECTORS ====================21const SELECTORS = {22 // Location / Pincode23 locationPopup: 'div.alcohol-popup',24 locationCloseBtn: ['button#btn_location_close_icon', 'button.close-privacy', 'button.close-icon'],25 locationManualBtn: 'button#select_location_popup',26 27 headerPincodeBtn: ['button#btn_pin_code_delivery', 'button.header-main-pincode-address', 'span#delivery_city_pincode'],28 deliveryPopup: 'div#delivery_popup',29 enterPincodeBtn: 'button#btn_enter_pincode',30 pincodeInputWrapper: 'div#delivery_enter_pincode',31 pincodeInput: 'input#rel_pincode',32 applyPincodeBtn: 'button#btn_pincode_submit',33 locationSuccessMsg: 'div#delivery_pin_msg.field-success',34 closeDeliveryPopup: 'button#close_delivery_popup',35 36 // Products37 productItem: 'li.ais-InfiniteHits-item',38 productLink: 'a.plp-card-wrapper',39 productName: 'div.plp-card-details-name',40 productImage: 'img.lazyloaded, img.lazyautosizes',41 currentPrice: 'span.jm-heading-xxs',42 originalPrice: 'span.line-through',43 discountBadge: 'span.jm-badge',44 addToCartBtn: 'button.addtocartbtn',45 vegIcon: 'img[src*="icon-veg"]',46 47 // Search / Listing48 searchResultsContainer: 'ul.ais-InfiniteHits-list',49 noResults: 'div.no-results', // Hypothetical selector, adjust if known50};51
52// ==================== HELPER FUNCTIONS ====================53
54const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));55
56/**57 * Close the initial location popup if it appears.58 */59async function closeLocationPopup(page, log) {60 try {61 const popup = page.locator(SELECTORS.locationPopup).first();62 if (await popup.count() === 0) return;63
64 log.info('🔔 Location popup detected. Attempting to close...');65
66 // Try close buttons with reduced timeout67 for (const selector of SELECTORS.locationCloseBtn) {68 const btn = page.locator(selector).first();69 if (await btn.isVisible({ timeout: 1000 }).catch(() => false)) {70 await btn.click();71 log.info(`✓ Closed popup using ${selector}`);72 return;73 }74 }75
76 // Try manual select button as fallback77 const manualBtn = page.locator(SELECTORS.locationManualBtn).first();78 if (await manualBtn.isVisible({ timeout: 1000 }).catch(() => false)) {79 await manualBtn.click();80 log.info('✓ Closed popup using "Select Location Manually"');81 }82 } catch (error) {83 log.warning(`⚠️ Failed to close location popup: ${error.message}`);84 }85}86
87/**88 * Set the pincode to ensure correct pricing and availability.89 */90async function setPincode(page, log, targetPincode) {91 log.info(`📍 Setting pincode to: ${targetPincode}`);92
93 try {94 // 1. Open Delivery Popup95 let opened = false;96 for (const selector of SELECTORS.headerPincodeBtn) {97 const btn = page.locator(selector).first();98 if (await btn.isVisible()) {99 await btn.click();100 opened = true;101 break;102 }103 }104 if (!opened) throw new Error('Could not find header location button');105
106 await page.waitForSelector(SELECTORS.deliveryPopup, { timeout: 5000 });107
108 // 2. Click "Enter Pincode" if needed (sometimes it shows saved addresses)109 const enterBtn = page.locator(SELECTORS.enterPincodeBtn).first();110 if (await enterBtn.isVisible()) {111 await enterBtn.click();112 }113
114 // 3. Enter Pincode115 await page.waitForSelector(SELECTORS.pincodeInput, { timeout: 5000 });116 const input = page.locator(SELECTORS.pincodeInput).first();117 await input.fill(targetPincode);118 119 // Trigger events to ensure validation logic runs120 await input.evaluate(e => {121 e.dispatchEvent(new Event('input', { bubbles: true }));122 e.dispatchEvent(new Event('change', { bubbles: true }));123 });124
125 // 4. Apply126 const applyBtn = page.locator(SELECTORS.applyPincodeBtn).first();127 await applyBtn.waitFor({ state: 'visible' });128 129 // Check if disabled, sometimes needs a moment130 if (await applyBtn.isDisabled()) {131 await delay(300);132 }133 await applyBtn.click();134
135 // 5. Verify Success (reduced timeout)136 try {137 await page.waitForSelector(SELECTORS.locationSuccessMsg, { timeout: 3000 });138 log.info('✅ Pincode applied successfully.');139 } catch (e) {140 log.warning('⚠️ Success message not seen, but continuing...');141 }142
143 // 6. Close Popup (if it doesn't auto-close)144 if (await page.locator(SELECTORS.deliveryPopup).isVisible({ timeout: 1000 }).catch(() => false)) {145 const closeBtn = page.locator(SELECTORS.closeDeliveryPopup).first();146 if (await closeBtn.isVisible({ timeout: 500 }).catch(() => false)) await closeBtn.click();147 else await page.keyboard.press('Escape');148 }149 150 // Wait for page to update with new location (reduced from 2000ms)151 await delay(500);152
153 } catch (error) {154 log.error(`❌ Failed to set pincode: ${error.message}`);155 // We continue even if this fails, as we might still get some data156 }157}158
159/**160 * Scroll a fixed number of times to load products - OPTIMIZED.161 */162async function autoScroll(page, log, iterations = 10) {163 log.info(`🔄 Auto-scrolling ${iterations} times...`);164 165 await page.evaluate(async (iterations) => {166 const distance = 800; // Increased distance per scroll167 const delay = 300; // Reduced from 1000ms to 300ms168 169 for (let i = 0; i < iterations; i++) {170 window.scrollBy(0, distance);171 await new Promise(resolve => setTimeout(resolve, delay));172 173 // Scroll to bottom occasionally to trigger infinite scroll174 if (i % 4 === 0) {175 window.scrollTo(0, document.body.scrollHeight);176 await new Promise(resolve => setTimeout(resolve, delay));177 }178 }179 180 // Final scroll to bottom to ensure all products loaded181 window.scrollTo(0, document.body.scrollHeight);182 await new Promise(resolve => setTimeout(resolve, 500));183 }, iterations);184 185 log.info('✓ Auto-scroll finished');186}187
188// ==================== CRAWLER SETUP ====================189
190const proxyConfig = proxyConfiguration?.useApifyProxy 191 ? await Actor.createProxyConfiguration(proxyConfiguration)192 : undefined;193
194// Track if pincode has been set for this session195let pincodeSet = false;196
197const crawler = new PlaywrightCrawler({198 proxyConfiguration: proxyConfig,199 maxRequestRetries,200 navigationTimeoutSecs: navigationTimeout / 1000,201 headless,202 maxConcurrency: 2, // Allow parallel processing of multiple queries203 204 // Browser launch options205 launchContext: {206 launchOptions: {207 args: [208 '--disable-blink-features=AutomationControlled',209 '--no-sandbox',210 '--disable-dev-shm-usage',211 ],212 }213 },214
215 async requestHandler({ page, request, log }) {216 log.info(`Processing ${request.url}`);217
218 // Only handle location setup once per session219 if (!pincodeSet) {220 await closeLocationPopup(page, log);221 await setPincode(page, log, pincode);222 pincodeSet = true;223 224 // Wait for page to stabilize after location change225 await delay(800);226 }227
228 // Wait for results with reduced timeout229 try {230 await page.waitForSelector(SELECTORS.productItem, { timeout: 10000 });231 } catch (e) {232 log.warning('⚠️ No products found or timeout waiting for selector.');233 // Quick check for no results234 const noResultsVisible = await page.locator(SELECTORS.noResults).isVisible({ timeout: 1000 }).catch(() => false);235 if (noResultsVisible) {236 log.info('No results found for this query.');237 return;238 }239 }240
241 await autoScroll(page, log, 10);242
243 // Extract Data244 const products = await page.$$eval(SELECTORS.productItem, (items, { selectors, requestData }) => {245 return items.map(item => {246 try {247 const linkEl = item.querySelector(selectors.productLink);248 const nameEl = item.querySelector(selectors.productName);249 const imgEl = item.querySelector(selectors.productImage);250 const priceEl = item.querySelector(selectors.currentPrice);251 const origPriceEl = item.querySelector(selectors.originalPrice);252 const discountEl = item.querySelector(selectors.discountBadge);253 const addBtn = item.querySelector(selectors.addToCartBtn);254 const vegIcon = item.querySelector(selectors.vegIcon);255 256 // Helper to clean price257 const parsePrice = (txt) => {258 if (!txt) return null;259 const match = txt.match(/[\d,.]+/);260 return match ? parseFloat(match[0].replace(/,/g, '')) : null;261 };262
263 const rawPrice = priceEl?.textContent?.trim();264 const rawOrigPrice = origPriceEl?.textContent?.trim();265 const currentPrice = parsePrice(rawPrice);266 const originalPrice = parsePrice(rawOrigPrice) || currentPrice;267 268 // Calculate discount269 let discountPercentage = 0;270 if (discountEl) {271 const match = discountEl.textContent.trim().match(/(\d+)%/);272 discountPercentage = match ? parseInt(match[1]) : 0;273 } else if (originalPrice && currentPrice && originalPrice > currentPrice) {274 discountPercentage = Math.round(((originalPrice - currentPrice) / originalPrice) * 100);275 }276
277 const productName = nameEl?.textContent?.trim() || linkEl?.getAttribute('title') || '';278 279 // Extract weight from name280 let productWeight = null;281 const weightMatch = productName.match(/(\d+\s*(?:g|kg|ml|l|gm|pack|pcs|piece))/i);282 if (weightMatch) {283 productWeight = weightMatch[1];284 }285
286 // Try to get brand from GTM data if available287 const gtmData = item.querySelector('.gtmEvents');288 const brand = gtmData?.getAttribute('data-manu') || 'JioMart'; // Default or extract289
290 return {291 productId: linkEl?.getAttribute('data-objid') || '',292 productName,293 productImage: imgEl?.src || imgEl?.getAttribute('data-src') || '',294 currentPrice,295 originalPrice,296 discountPercentage,297 productWeight,298 brand,299 isVegetarian: !!vegIcon,300 isOutOfStock: addBtn ? addBtn.hasAttribute('disabled') : false,301 productUrl: linkEl ? linkEl.href : '',302 scrapedAt: new Date().toISOString(),303 searchQuery: requestData.query,304 searchUrl: window.location.href,305 platform: "JioMart",306 pincode: requestData.pincode307 };308 } catch (e) {309 return null;310 }311 }).filter(p => p && p.productName && p.currentPrice); // Filter invalid items312 }, { selectors: SELECTORS, requestData: request.userData });313
314 log.info(`Found ${products.length} products.`);315
316 // Push to dataset317 await Dataset.pushData(products);318 },319
320 failedRequestHandler({ request, log }) {321 log.error(`Request ${request.url} failed too many times.`);322 },323});324
325// ==================== EXECUTION ====================326
327const startUrls = [328 ...searchQueries.map(query => ({329 url: `https://www.jiomart.com/search?q=${encodeURIComponent(query)}`,330 userData: { 331 query,332 pincode333 }334 })),335 ...searchUrls.map(url => ({336 url,337 userData: {338 query: 'direct_url',339 pincode340 }341 }))342];343
344log.info(`Starting crawler for ${startUrls.length} queries...`);345await crawler.run(startUrls);346log.info('Crawler finished.');347
348// Exit Actor349await Actor.exit();1import { Actor } from 'apify';2import { PlaywrightCrawler, Dataset } from 'crawlee';3
4// Initialize Actor5await Actor.init();6
7// ==================== INPUT CONFIGURATION ====================8const input = await Actor.getInput() ?? {};9
10const {11 pincode = '411001',12 searchUrls = [],13 searchQueries = [],14 maxProductsPerSearch = 100,15 proxyConfiguration = { useApifyProxy: false },16 maxRequestRetries = 3,17 navigationTimeout = 90000,18 headless = false,19 screenshotOnError = true,20 debugMode = false,21 scrollCount = 522} = input;23
24// ==================== CONSTANTS ====================25const USER_AGENTS = [26 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',27 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36',28 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15',29 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'30];31
32// Global flag to track if location has been set33let locationSetGlobally = false;34
35// ==================== HELPER FUNCTIONS ====================36
37/**38 * Pick a random item from an array39 */40function pickRandom(arr) {41 return arr[Math.floor(Math.random() * arr.length)];42}43
44/**45 * Parse proxy URL into Playwright's proxy object46 */47function parseProxyUrl(proxyUrl) {48 try {49 const url = new URL(proxyUrl);50 const proxy = {51 server: `${url.protocol}//${url.hostname}${url.port ? `:${url.port}` : ''}`52 };53 if (url.username) proxy.username = decodeURIComponent(url.username);54 if (url.password) proxy.password = decodeURIComponent(url.password);55 return proxy;56 } catch (error) {57 console.error('Invalid proxy URL:', error.message);58 return null;59 }60}61
62/**63 * Delay execution64 */65const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));66
67// ==================== POPUP CLOSING FUNCTION ====================68
69/**70 * Close the location services popup that appears on JioMart71 */72async function closeLocationPopup(page, log) {73 try {74 log.info('🔔 Attempting to close location services popup...');75 76 // Check if popup exists77 const popup = page.locator('div.alcohol-popup').first();78 const popupCount = await popup.count();79 80 if (popupCount === 0) {81 log.info('ℹ️ No location popup detected');82 return true;83 }84 85 log.info('📋 Found location services popup - closing...');86 87 // Try clicking close button first88 const closeButtonSelectors = [89 'button#btn_location_close_icon',90 'button.close-privacy',91 'button.close-icon'92 ];93 94 for (const selector of closeButtonSelectors) {95 try {96 const closeBtn = page.locator(selector).first();97 if (await closeBtn.count() > 0) {98 await closeBtn.click({ timeout: 3000 });99 log.info(`✓ Clicked close button using: ${selector}`);100 await delay(800);101 102 // Verify popup is closed103 const stillVisible = await popup.count();104 if (stillVisible === 0) {105 log.info('✅ Location popup closed successfully');106 return true;107 }108 }109 } catch (error) {110 continue;111 }112 }113 114 // Fallback: click "Select Location Manually" button115 try {116 const selectLocBtn = page.locator('button#select_location_popup').first();117 if (await selectLocBtn.count() > 0) {118 await selectLocBtn.click({ timeout: 3000 });119 log.info('✓ Clicked "Select Location Manually" button');120 await delay(1000);121 log.info('✅ Location popup closed via manual selection');122 return true;123 }124 } catch (error) {125 log.warning(`Failed to click Select Location button: ${error.message}`);126 }127 128 log.warning('⚠️ Could not close location popup, but continuing...');129 return false;130 131 } catch (error) {132 log.error(`❌ Error closing popup: ${error.message}`);133 return false;134 }135}136
137// ==================== LOCATION SETTING FUNCTION ====================138
139/**140 * Set pincode location for JioMart141 */142/**143 * Set pincode location for JioMart - FIXED VERSION144 */145async function setPincodeLocation(page, log, targetPincode) {146 try {147 log.info(`🎯 Setting location to pincode: ${targetPincode}`);148 149 // Wait for page to be fully loaded150 await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {});151 await delay(2000);152 153 // === STEP 1: Click on location/pincode button in header ===154 const locationSelectors = [155 'button#btn_pin_code_delivery',156 'button.header-main-pincode-address',157 'span#delivery_city_pincode'158 ];159 160 let locationButtonClicked = false;161 for (const selector of locationSelectors) {162 try {163 const button = page.locator(selector).first();164 if (await button.isVisible({ timeout: 3000 })) {165 await button.click({ timeout: 5000 });166 log.info(`✓ Clicked location button using: ${selector}`);167 locationButtonClicked = true;168 break;169 }170 } catch (error) {171 continue;172 }173 }174 175 if (!locationButtonClicked) {176 log.warning('⚠️ Could not find location button');177 return false;178 }179 180 await delay(2000);181 182 // === STEP 2: Wait for delivery popup to appear ===183 try {184 await page.waitForSelector('div#delivery_popup', { timeout: 5000, state: 'visible' });185 log.info('✓ Delivery popup opened');186 } catch (error) {187 log.warning('⚠️ Delivery popup did not appear');188 return false;189 }190 191 await delay(1500);192 193 // === STEP 3: Wait for initial content and click "Enter a pincode" ===194 // The initial content div needs to be displayed first195 try {196 // Sometimes the div is hidden by default, try to show it197 await page.evaluate(() => {198 const deliveryContent = document.querySelector('#delivery-content');199 if (deliveryContent && deliveryContent.style.display === 'none') {200 deliveryContent.style.display = 'block';201 }202 });203 } catch (e) {204 // Ignore if script fails205 }206 207 await delay(1000);208 209 // Now click the "Enter a pincode" button210 try {211 const enterPincodeBtn = page.locator('button#btn_enter_pincode').first();212 await enterPincodeBtn.waitFor({ state: 'visible', timeout: 5000 });213 await enterPincodeBtn.click({ timeout: 5000 });214 log.info('✓ Clicked "Enter a pincode" button');215 } catch (error) {216 log.error(`❌ Could not click "Enter a pincode" button: ${error.message}`);217 return false;218 }219 220 await delay(2000);221 222 // === STEP 4: Wait for pincode entry form to appear ===223 try {224 await page.waitForSelector('div#delivery_enter_pincode', { timeout: 5000, state: 'visible' });225 log.info('✓ Pincode entry form appeared');226 } catch (error) {227 log.error('❌ Pincode entry form did not appear');228 return false;229 }230 231 await delay(1000);232 233 // === STEP 5: Find and interact with pincode input ===234 const inputField = page.locator('input#rel_pincode').first();235 236 try {237 await inputField.waitFor({ state: 'visible', timeout: 5000 });238 log.info('✓ Found pincode input field');239 } catch (error) {240 log.error('❌ Could not find pincode input field');241 return false;242 }243 244 // === STEP 6: Clear and enter pincode ===245 try {246 // Focus on input247 await inputField.click();248 await delay(500);249 250 // Clear any existing value251 await inputField.fill('');252 await delay(300);253 254 // Triple-click to select all (if any text remains)255 await inputField.click({ clickCount: 3 });256 await delay(200);257 258 // Type the pincode259 await inputField.fill(targetPincode);260 await delay(500);261 262 // Verify the value was set263 const inputValue = await inputField.inputValue();264 log.info(`✓ Entered pincode: ${inputValue}`);265 266 if (inputValue !== targetPincode) {267 log.error(`❌ Pincode mismatch! Expected: ${targetPincode}, Got: ${inputValue}`);268 return false;269 }270 271 // Trigger events to ensure validation272 await page.evaluate((pincode) => {273 const input = document.querySelector('input#rel_pincode');274 if (input) {275 input.value = pincode;276 input.dispatchEvent(new Event('input', { bubbles: true }));277 input.dispatchEvent(new Event('change', { bubbles: true }));278 input.dispatchEvent(new Event('blur', { bubbles: true }));279 }280 }, targetPincode);281 282 log.info('✓ Triggered validation events');283 await delay(2000);284 285 } catch (error) {286 log.error(`❌ Failed to enter pincode: ${error.message}`);287 return false;288 }289 290 // === STEP 7: Wait for location to be detected ===291 try {292 // Wait for success message to appear293 await page.waitForSelector('div#delivery_pin_msg.field-success', { 294 timeout: 5000, 295 state: 'visible' 296 });297 298 const locationMessage = await page.locator('div#delivery_pin_msg').textContent();299 log.info(`✓ Location detected: ${locationMessage.trim()}`);300 } catch (error) {301 log.warning('⚠️ Location message not detected, checking if button is enabled...');302 }303 304 await delay(1000);305 306 // === STEP 8: Click Apply button ===307 try {308 const applyButton = page.locator('button#btn_pincode_submit').first();309 310 // Wait for button to be visible and enabled311 await applyButton.waitFor({ state: 'visible', timeout: 5000 });312 313 // Check if button is disabled314 const isDisabled = await applyButton.isDisabled();315 if (isDisabled) {316 log.warning('⚠️ Apply button is disabled, attempting to enable it...');317 318 // Force enable the button319 await page.evaluate(() => {320 const btn = document.querySelector('button#btn_pincode_submit');321 if (btn) {322 btn.disabled = false;323 btn.removeAttribute('disabled');324 btn.classList.remove('disabled');325 }326 });327 328 await delay(500);329 }330 331 // Click the button332 await applyButton.click({ force: true, timeout: 5000 });333 log.info('✓ Clicked Apply button');334 335 } catch (error) {336 log.error(`❌ Could not click Apply button: ${error.message}`);337 return false;338 }339 340 // === STEP 9: Wait for modal to close ===341 try {342 // Wait for the popup to disappear343 await page.waitForSelector('div#delivery_popup', { 344 state: 'hidden', 345 timeout: 10000 346 });347 log.info('✅ Location modal closed successfully');348 } catch (error) {349 // Check if modal is still visible350 const isStillVisible = await page.locator('div#delivery_popup').isVisible().catch(() => false);351 352 if (isStillVisible) {353 log.warning('⚠️ Modal still visible, trying to close manually...');354 355 // Try clicking the close button356 try {357 await page.locator('button#close_delivery_popup').click({ timeout: 2000 });358 await delay(1000);359 } catch (e) {360 // Try clicking backdrop361 try {362 await page.locator('div.backdrop').click({ timeout: 2000 });363 await delay(1000);364 } catch (e2) {365 log.error('❌ Could not close modal');366 return false;367 }368 }369 } else {370 log.info('✅ Modal closed (not visible)');371 }372 }373 374 // Wait for page to update with new location375 await page.waitForLoadState('networkidle', { timeout: 5000 }).catch(() => {});376 await delay(2000);377 378 // === STEP 10: Verify location is set ===379 try {380 const headerPincode = await page.locator('button#btn_pin_code_delivery, span#delivery_city_pincode')381 .first()382 .textContent()383 .catch(() => '');384 385 if (headerPincode.includes(targetPincode)) {386 log.info(`✅ Location verified in header: ${headerPincode}`);387 } else {388 log.info(`ℹ️ Location set, header shows: ${headerPincode}`);389 }390 } catch (e) {391 // Verification failed but location might still be set392 log.info('ℹ️ Could not verify location in header, but proceeding...');393 }394 395 log.info('✅ Pincode location set successfully');396 return true;397 398 } catch (error) {399 log.error(`❌ Error in setPincodeLocation: ${error.message}`);400 401 // Take screenshot on error for debugging402 try {403 const screenshot = await page.screenshot({ fullPage: true });404 await Actor.setValue(`pincode-error-${Date.now()}.png`, screenshot, { 405 contentType: 'image/png' 406 });407 log.info('📸 Error screenshot saved');408 } catch (e) {409 // Ignore screenshot errors410 }411 412 return false;413 }414}415
416// ==================== SCROLLING FUNCTION ====================417
418/**419 * Auto-scroll to load lazy-loaded content420 */421async function autoScroll(page, log, iterations = 5) {422 try {423 log.info(`🔄 Starting auto-scroll (${iterations} iterations)...`);424 425 for (let i = 0; i < iterations; i++) {426 await page.evaluate(() => window.scrollBy(0, window.innerHeight));427 await delay(1500);428 }429 430 // Scroll back to top431 await page.evaluate(() => window.scrollTo(0, 0));432 await delay(500);433 434 log.info('✓ Auto-scroll completed');435 return true;436 } catch (error) {437 log.warning(`⚠️ Auto-scroll failed: ${error.message}`);438 return false;439 }440}441
442// ==================== DEBUG FUNCTION ====================443
444/**445 * Debug page state with screenshots and HTML446 */447async function debugPageState(page, log, label = 'debug') {448 if (!debugMode) return;449 450 try {451 const timestamp = Date.now();452 453 // Take screenshot454 const screenshot = await page.screenshot({ fullPage: true });455 await Actor.setValue(`${label}-${timestamp}.png`, screenshot, { contentType: 'image/png' });456 457 // Save HTML458 const html = await page.content();459 await Actor.setValue(`${label}-${timestamp}.html`, html, { contentType: 'text/html' });460 461 // Get page info462 const pageInfo = await page.evaluate(() => ({463 url: window.location.href,464 title: document.title,465 viewport: { width: window.innerWidth, height: window.innerHeight },466 elementCounts: {467 productCards: document.querySelectorAll('li.ais-InfiniteHits-item').length,468 productLinks: document.querySelectorAll('a.plp-card-wrapper').length,469 images: document.querySelectorAll('img').length,470 prices: document.querySelectorAll('span.jm-heading-xxs').length471 }472 }));473 474 log.info(`📊 Page state: ${JSON.stringify(pageInfo, null, 2)}`);475 } catch (error) {476 log.error(`Debug failed: ${error.message}`);477 }478}479
480// ==================== WAIT FOR RESULTS ====================481
482/**483 * Wait for search results to appear484 */485async function waitForSearchResults(page, log) {486 const selectors = [487 'li.ais-InfiniteHits-item',488 'a.plp-card-wrapper',489 'div.plp-card-container',490 'div.plp-card-image'491 ];492
493 for (const selector of selectors) {494 try {495 await page.waitForSelector(selector, { timeout: 10000, state: 'visible' });496 const count = await page.locator(selector).count();497 if (count > 0) {498 log.info(`✓ Found ${count} product elements using: ${selector}`);499 await delay(1000);500 return true;501 }502 } catch (error) {503 continue;504 }505 }506
507 // Fallback: check page content508 try {509 const bodyText = await page.textContent('body');510 if (bodyText?.includes('₹') || /\bAdd\b/i.test(bodyText)) {511 log.info('✓ Found product indicators in page content');512 return true;513 }514 } catch (error) {515 // Ignore516 }517
518 log.warning('⚠️ No search results detected');519 return false;520}521
522// ==================== PRODUCT EXTRACTION ====================523
524/**525 * Extract products from JioMart search results526 */527async function extractJioMartProducts(page, log) {528 try {529 log.info('🔍 Extracting products...');530 531 const products = await page.evaluate(() => {532 const extractedProducts = [];533 const productItems = document.querySelectorAll('li.ais-InfiniteHits-item');534 535 productItems.forEach((item, index) => {536 try {537 // Get product link and data538 const productLink = item.querySelector('a.plp-card-wrapper');539 if (!productLink) return;540 541 const productUrl = productLink.href;542 const productId = productLink.getAttribute('data-objid');543 544 // Get GTM data attributes for accurate info545 const gtmData = item.querySelector('.gtmEvents');546 547 // Extract product name548 const nameEl = item.querySelector('div.plp-card-details-name');549 const productName = nameEl?.textContent?.trim() || 550 gtmData?.getAttribute('data-name') || 551 productLink.getAttribute('title');552 553 // Extract image554 const imgEl = item.querySelector('img.lazyloaded, img.lazyautosizes');555 let productImage = imgEl?.src || imgEl?.getAttribute('data-src');556 if (productImage && !productImage.startsWith('http')) {557 productImage = `https://www.jiomart.com${productImage}`;558 }559 560 // Extract current price561 const priceEl = item.querySelector('span.jm-heading-xxs');562 let currentPrice = null;563 if (priceEl) {564 const priceText = priceEl.textContent.trim();565 const match = priceText.match(/₹\s*(\d+(?:,\d+)*(?:\.\d+)?)/);566 currentPrice = match ? parseFloat(match[1].replace(/,/g, '')) : null;567 }568 569 // Fallback to GTM data570 if (!currentPrice && gtmData) {571 const gtmPrice = gtmData.getAttribute('data-price');572 currentPrice = gtmPrice ? parseFloat(gtmPrice) : null;573 }574 575 // Extract original price (strikethrough price)576 const originalPriceEl = item.querySelector('span.line-through');577 let originalPrice = null;578 if (originalPriceEl) {579 const priceText = originalPriceEl.textContent.trim();580 const match = priceText.match(/₹\s*(\d+(?:,\d+)*(?:\.\d+)?)/);581 originalPrice = match ? parseFloat(match[1].replace(/,/g, '')) : null;582 }583 584 // Extract discount percentage585 const discountEl = item.querySelector('span.jm-badge');586 let discountPercentage = null;587 if (discountEl) {588 const discountText = discountEl.textContent.trim();589 const match = discountText.match(/(\d+)%/);590 discountPercentage = match ? parseInt(match[1]) : null;591 }592 593 // Calculate discount if not available594 if (!discountPercentage && currentPrice && originalPrice && originalPrice > currentPrice) {595 discountPercentage = Math.round(((originalPrice - currentPrice) / originalPrice) * 100);596 }597 598 // Extract weight/size from product name or separate field599 let productWeight = null;600 const weightMatch = productName?.match(/(\d+\s*(?:g|kg|ml|l|gm|pack|pcs|piece))/i);601 if (weightMatch) {602 productWeight = weightMatch[1];603 }604 605 // Extract brand606 const brand = gtmData?.getAttribute('data-manu') || null;607 608 // Check if vegetarian609 const vegIcon = item.querySelector('img[src*="icon-veg"]');610 const isVegetarian = vegIcon !== null;611 612 // Check stock status613 const addButton = item.querySelector('button.addtocartbtn');614 const isOutOfStock = addButton?.hasAttribute('disabled') || false;615 616 if (productName && currentPrice) {617 extractedProducts.push({618 productId: productId || `jiomart-${index}`,619 productName,620 productImage,621 currentPrice,622 originalPrice: originalPrice || currentPrice,623 discountPercentage: discountPercentage || 0,624 productWeight,625 brand,626 isVegetarian,627 isOutOfStock,628 productUrl,629 scrapedAt: new Date().toISOString()630 });631 }632 } catch (error) {633 console.error(`Error extracting product ${index}:`, error);634 }635 });636 637 return extractedProducts;638 });639 640 log.info(`✅ Extracted ${products.length} products`);641 642 if (debugMode && products.length > 0) {643 log.info(`Sample product: ${JSON.stringify(products[0], null, 2)}`);644 }645 646 return products;647 } catch (error) {648 log.error(`❌ Error extracting products: ${error.message}`);649 return [];650 }651}652
653// ==================== PROXY CONFIGURATION ====================654
655const proxyConfig = proxyConfiguration?.useApifyProxy 656 ? await Actor.createProxyConfiguration(proxyConfiguration)657 : undefined;658
659const customProxyUrl = proxyConfiguration?.customProxyUrl || 660 proxyConfiguration?.proxyUrl || 661 proxyConfiguration?.proxy;662const launchProxy = customProxyUrl ? parseProxyUrl(customProxyUrl) : null;663
664// ==================== GENERATE SEARCH URLS ====================665
666const allSearchUrls = [667 ...searchUrls,668 ...searchQueries.map(query => 669 `https://www.jiomart.com/search?q=${encodeURIComponent(query)}`670 )671];672
673// ==================== CRAWLER INITIALIZATION ====================674
675const crawler = new PlaywrightCrawler({676 proxyConfiguration: proxyConfig,677 maxRequestRetries,678 navigationTimeoutSecs: navigationTimeout / 1000,679 headless:false,680 681 launchContext: {682 launchOptions: {683 args: [684 '--disable-blink-features=AutomationControlled',685 '--disable-dev-shm-usage',686 '--no-sandbox',687 '--disable-setuid-sandbox',688 '--disable-web-security',689 '--disable-features=IsolateOrigins,site-per-process'690 ],691 ...((!proxyConfig && launchProxy) ? { proxy: launchProxy } : {})692 }693 },694
695 preNavigationHooks: [696 async ({ page, log }) => {697 try {698 const userAgent = pickRandom(USER_AGENTS);699
700 await page.setExtraHTTPHeaders({701 'Accept-Language': 'en-US,en;q=0.9',702 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',703 'User-Agent': userAgent,704 'Cache-Control': 'no-cache',705 'Pragma': 'no-cache'706 });707
708 await page.setViewportSize({ width: 1920, height: 1080 });709
710 // Mask automation711 await page.addInitScript((ua) => {712 Object.defineProperty(navigator, 'webdriver', { get: () => false });713 Object.defineProperty(navigator, 'userAgent', { get: () => ua });714 Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3, 4, 5] });715 Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'] });716 717 // Override Chrome detection718 window.chrome = { runtime: {} };719 }, userAgent).catch(() => {});720 721 } catch (error) {722 log.error(`preNavigationHook error: ${error.message}`);723 }724 }725 ],726
727 async requestHandler({ page, request, log }) {728 const { url } = request;729 const isFirstRequest = request.userData?.isFirst || false;730
731 log.info(`🔍 Processing: ${url}`);732
733 try {734 // === STEP 0: Close location services popup ===735 if (isFirstRequest) {736 await closeLocationPopup(page, log);737 await delay(1000);738 }739 740 // === STEP 1: Set location on first request ===741 if (isFirstRequest && !locationSetGlobally) {742 log.info('🎯 First request - setting pincode location');743 744 const locationSet = await setPincodeLocation(page, log, pincode);745 746 if (locationSet) {747 locationSetGlobally = true;748 log.info('✅ Location set successfully');749 750 // Reload page to apply location751 await page.reload({ waitUntil: 'networkidle' });752 await delay(3000);753 } else {754 log.warning('⚠️ Failed to set location, continuing anyway');755 }756 }757 758 // === STEP 2: Wait for page load ===759 await page.waitForLoadState('domcontentloaded');760 await delay(3000);761 762 if (isFirstRequest) {763 await debugPageState(page, log, 'initial');764 }765 766 // === STEP 3: Close any popups/modals ===767 try {768 // Close any remaining delivery popup769 const closeBtn = page.locator('button#close_delivery_popup').first();770 if (await closeBtn.count() > 0 && await closeBtn.isVisible({ timeout: 2000 })) {771 await closeBtn.click();772 await delay(500);773 }774 } catch (error) {775 // No popup776 }777 778 // Close any other modals779 try {780 const backdrop = page.locator('div.backdrop').first();781 if (await backdrop.isVisible({ timeout: 2000 })) {782 await backdrop.click();783 await delay(500);784 }785 } catch (error) {786 // No backdrop787 }788 789 // === STEP 4: Wait for results ===790 const resultsFound = await waitForSearchResults(page, log);791 if (!resultsFound) {792 await debugPageState(page, log, 'no-results');793 log.warning('⚠️ No search results detected');794 }795 796 // === STEP 5: Scroll to load products ===797 await autoScroll(page, log, scrollCount);798 799 if (isFirstRequest) {800 await debugPageState(page, log, 'after-scroll');801 }802 803 // === STEP 6: Extract products ===804 const products = await extractJioMartProducts(page, log);805 806 if (products.length === 0) {807 log.error('❌ No products extracted');808 await debugPageState(page, log, 'no-products');809 return;810 }811 812 // === STEP 7: Save products ===813 const urlParams = new URL(url).searchParams;814 const searchQuery = urlParams.get('q') || urlParams.get('query') || 'direct_url';815 816 const productsToSave = products.slice(0, maxProductsPerSearch).map(product => ({817 ...product,818 searchQuery,819 searchUrl: url,820 platform: 'JioMart',821 pincode822 }));823 824 await Dataset.pushData(productsToSave);825 826 log.info(`✅ Saved ${productsToSave.length} products for "${searchQuery}" (Pincode: ${pincode})`);827
828 } catch (error) {829 log.error(`❌ Error processing ${url}: ${error.message}`);830 831 if (screenshotOnError) {832 try {833 const screenshot = await page.screenshot({ fullPage: true });834 const timestamp = Date.now();835 await Actor.setValue(`error-${timestamp}.png`, screenshot, { contentType: 'image/png' });836 } catch (e) {837 log.error(`Screenshot failed: ${e.message}`);838 }839 }840 841 throw error;842 }843 },844
845 failedRequestHandler: async ({ request, log }) => {846 log.error(`❌ Request failed: ${request.url}`);847 848 const failedUrls = await Actor.getValue('FAILED_URLS') || [];849 failedUrls.push({850 url: request.url,851 timestamp: new Date().toISOString(),852 error: request.errorMessages?.join(', ')853 });854 await Actor.setValue('FAILED_URLS', failedUrls);855 }856});857
858// ==================== START CRAWLER ====================859
860if (allSearchUrls.length > 0) {861 console.log('\n' + '='.repeat(60));862 console.log('🚀 JIOMART SCRAPER STARTED');863 console.log('='.repeat(60));864 console.log(`📍 Pincode: ${pincode}`);865 console.log(`🔍 Search URLs: ${allSearchUrls.length}`);866 console.log(`📊 Max products per search: ${maxProductsPerSearch}`);867 console.log(`📜 Scroll iterations: ${scrollCount}`);868 console.log(`🐛 Debug mode: ${debugMode}`);869 console.log(`👁️ Headless: ${headless}`);870 console.log('='.repeat(60) + '\n');871 872 const searchRequests = allSearchUrls.map((url, index) => ({ 873 url, 874 userData: { isFirst: index === 0 } 875 }));876 877 console.log('🔍 URLs to process:\n');878 searchRequests.forEach((req, idx) => {879 console.log(` ${idx + 1}. ${req.url}${idx === 0 ? ' 📍 (will set location)' : ''}`);880 });881 console.log('');882 883 await crawler.run(searchRequests);884 885 console.log('\n' + '='.repeat(60));886 console.log('✅ SCRAPING COMPLETED');887 console.log('='.repeat(60));888 console.log('📁 Results: storage/datasets/default/');889 console.log('📸 Debug files: storage/key_value_stores/default/');890 console.log('='.repeat(60) + '\n');891} else {892 console.log('\n' + '='.repeat(60));893 console.log('❌ NO SEARCH URLS PROVIDED');894 console.log('='.repeat(60));895 console.log('Please provide either "searchUrls" or "searchQueries" in input.json');896 console.log('='.repeat(60) + '\n');897}898
899await Actor.exit();1const fs = require('fs');2
3let content = fs.readFileSync('d:\\\\creatosaurus-intership\\\\scarpping\\\\jiomart-scrapper\\\\index.js', 'utf8');4
5// Configuration optimizations6content = content.replace('maxRequestRetries = 3', 'maxRequestRetries = 2');7content = content.replace('navigationTimeout = 90000', 'navigationTimeout = 30000');8content = content.replace('scrollCount = 5', 'scrollCount = 3');9
10// Delay optimizations - be very specific11content = content.replace(/await delay\(2000\);/g, 'await delay(500);');12content = content.replace(/await delay\(1500\);/g, 'await delay(400);');13content = content.replace(/await delay\(1000\);/g, 'await delay(300);');14content = content.replace(/await delay\(3000\);/g, 'await delay(800);');15content = content.replace(/await delay\(800\);/g, 'await delay(300);');16
17// Timeout optimizations18content = content.replace(/timeout: 10000/g, 'timeout: 5000');19
20fs.writeFileSync('d:\\\\creatosaurus-intership\\\\scarpping\\\\jiomart-scrapper\\\\index.js', content);21console.log('Optimizations applied successfully!');$content = Get-Content "d:\creatosaurus-intership\scarpping\jiomart-scrapper\index.js" -Raw
# Configuration optimizations$content = $content -replace 'maxRequestRetries = 3', 'maxRequestRetries = 2'$content = $content -replace 'navigationTimeout = 90000', 'navigationTimeout = 30000'$content = $content -replace 'scrollCount = 5', 'scrollCount = 3'
# Delay optimizations - be very specific with patterns$content = $content -replace 'await delay\(2000\);', 'await delay(500);'$content = $content -replace 'await delay\(1500\);', 'await delay(400);'$content = $content -replace 'await delay\(1000\);', 'await delay(300);'$content = $content -replace 'await delay\(3000\);', 'await delay(800);'$content = $content -replace 'await delay\(800\);', 'await delay(300);'
# Timeout optimizations$content = $content -replace 'timeout: 10000', 'timeout: 5000'
# Scrolling optimization - reduce delay in scroll loop$content = $content -replace 'for \(let i = 0; i < iterations; i\+\+\) \{\s+await page\.evaluate\(\(\) => window\.scrollBy\(0, window\.innerHeight\)\);\s+await delay\(400\);', 'for (let i = 0; i < iterations; i++) {`r`n await page.evaluate(() => window.scrollBy(0, window.innerHeight));`r`n await delay(500);'
$content | Set-Content "d:\creatosaurus-intership\scarpping\jiomart-scrapper\index.js" -NoNewlineWrite-Host "Optimizations applied successfully!"{ "name": "blinkit-scrapper", "type": "module", "version": "1.0.0", "main": "index.js", "scripts": { "start": "node index.js", "test": "echo \"Error: no test specified\" && exit 1" }, "keywords": [], "author": "", "license": "ISC", "description": "", "dependencies": { "apify": "^3.5.2", "crawlee": "^3.15.3", "nodemon": "^3.1.11", "playwright": "^1.30.0" }}1import { Actor } from 'apify';2import { PlaywrightCrawler, Dataset, log } from 'crawlee';3
4// Initialize Actor5await Actor.init();6
7// ==================== INPUT CONFIGURATION ====================8const input = await Actor.getInput() ?? {};9const {10 pincode = '411001',11 searchQueries = ['milk'],12 searchUrls = [],13 maxProductsPerSearch = 100,14 maxRequestRetries = 3,15 navigationTimeout = 60000,16 headless = true,17 proxyConfiguration = { useApifyProxy: false },18 scrollCount = 40, // Increased default scroll count19} = input;20
21// ==================== CONSTANTS & SELECTORS ====================22const SELECTORS = {23 // Location / Pincode24 locationButton: [25 'button[aria-label="Select Location"]',26 'button.__4y7HY',27 'div.a0Ppr button'28 ],29 locationModal: 'div[data-testid="address-modal"]',30 searchInput: 'div[data-testid="address-search-input"] input[type="text"]',31 searchResultItem: 'div[data-testid="address-search-item"]',32 33 // Products34 productLink: 'a.B4vNQ',35 productCard: 'div.cavQgJ.cTH4Df', // Fallback if link doesn't contain everything36 37 // Product Details (Inside card)38 productName: [39 'div[data-slot-id="ProductName"] span',40 'div.cQAjo6.ch5GgP span',41 'h3', 42 'h2'43 ],44 productImage: 'img',45 priceSpan: 'span', // We'll iterate spans to find price46 packSize: '[data-slot-id="PackSize"] span',47 rating: '[data-slot-id="RatingInformation"]',48 sponsorTag: '[data-slot-id="SponsorTag"]',49 50 // Search / Listing51 searchResultsContainer: 'div.grid', // Generic grid container, might need adjustment52};53
54// ==================== HELPER FUNCTIONS ====================55
56const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));57
58/**59 * Sets the pincode location on the page.60 */61async function setPincode(page, log, targetPincode) {62 try {63 log.info(`🎯 Setting location to pincode: ${targetPincode}`);64 65 await page.waitForLoadState('domcontentloaded');66 await delay(1500);67 68 let clicked = false;69 for (const selector of SELECTORS.locationButton) {70 try {71 const button = page.locator(selector).first();72 if (await button.count() > 0) {73 await button.click({ timeout: 3000 });74 log.info(`✓ Clicked location button: ${selector}`);75 clicked = true;76 break;77 }78 } catch (e) {79 continue;80 }81 }82 83 if (!clicked) {84 log.warning('⚠️ Location button not found');85 return false;86 }87 88 await delay(1000);89 90 try {91 await page.waitForSelector(SELECTORS.locationModal, { timeout: 5000 });92 } catch (e) {93 log.warning('⚠️ Location modal not detected');94 return false;95 }96 97 await delay(800);98 99 const searchInput = page.locator(SELECTORS.searchInput).first();100 101 if (await searchInput.count() === 0) {102 log.error('❌ Search input not found in modal');103 return false;104 }105 106 await searchInput.focus();107 await delay(200);108 await page.keyboard.press('Control+A');109 await page.keyboard.press('Backspace');110 await delay(200);111 await searchInput.type(targetPincode, { delay: 80 });112 113 await delay(1500);114 115 try {116 await page.waitForSelector(SELECTORS.searchResultItem, { timeout: 5000 });117 } catch (e) {118 log.error('❌ No address results appeared');119 return false;120 }121 122 const firstAddress = page.locator(SELECTORS.searchResultItem).first();123 124 if (await firstAddress.count() > 0) {125 await firstAddress.click({ force: true });126 await delay(1500);127 128 const modalStillOpen = await page.locator(SELECTORS.locationModal).count();129 if (modalStillOpen === 0) {130 log.info('✅ Location set successfully');131 return true;132 }133 }134 135 return false;136 } catch (error) {137 log.error(`❌ Error setting pincode: ${error.message}`);138 return false;139 }140}141
142/**143 * Auto-scrolls the page to load dynamic content.144 * Scrolls to bottom, waits for height change, and retries if needed.145 */146async function autoScroll(page, log, maxScrolls = 20) {147 try {148 log.info(`🔄 Auto-scrolling up to ${maxScrolls} times...`);149 150 let previousHeight = await page.evaluate('document.body.scrollHeight');151 let noChangeCount = 0;152 153 for (let i = 0; i < maxScrolls; i++) {154 await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));155 await delay(2000); // Wait for content to load156 157 const newHeight = await page.evaluate('document.body.scrollHeight');158 159 if (newHeight === previousHeight) {160 noChangeCount++;161 // Try scrolling up a bit and back down to trigger observers162 if (noChangeCount === 1) {163 await page.evaluate(() => window.scrollBy(0, -500));164 await delay(500);165 await page.evaluate(() => window.scrollTo(0, document.body.scrollHeight));166 await delay(1000);167 }168 169 if (noChangeCount >= 3) {170 log.info(`✓ Reached bottom or no new content after ${i + 1} scrolls`);171 break;172 }173 } else {174 noChangeCount = 0;175 previousHeight = newHeight;176 log.info(` - Scroll ${i + 1}/${maxScrolls}: Content loaded (Height: ${newHeight})`);177 }178 }179 180 // Scroll back to top to ensure all elements are rendered/hydrated if needed181 await page.evaluate(() => window.scrollTo(0, 0));182 await delay(500);183 184 } catch (error) {185 log.warning(`Auto-scroll failed: ${error.message}`);186 }187}188
189/**190 * Waits for search results to appear on the page.191 */192async function waitForSearchResults(page, log) {193 try {194 await page.waitForSelector(SELECTORS.productLink, { timeout: 10000 });195 const count = await page.locator(SELECTORS.productLink).count();196 if (count > 0) {197 await delay(500);198 return true;199 }200 } catch (e) {201 // Fallback check202 try {203 const bodyText = await page.evaluate(() => document.body.innerText || '');204 if (bodyText.includes('₹') || /\bADD\b/i.test(bodyText)) {205 return true;206 }207 } catch (err) {208 // Ignore209 }210 }211 212 log.warning('No search results found');213 return false;214}215
216function pickRandom(arr) {217 return arr[Math.floor(Math.random() * arr.length)];218}219
220// ==================== CRAWLER SETUP ====================221
222const USER_AGENTS = [223 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',224 'Mozilla/5.0 (Macintosh; Intel Mac OS X 13_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15',225 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/119.0.0.0 Safari/537.36'226];227
228const proxyConfig = proxyConfiguration?.useApifyProxy 229 ? await Actor.createProxyConfiguration(proxyConfiguration)230 : undefined;231
232const crawler = new PlaywrightCrawler({233 proxyConfiguration: proxyConfig,234 maxRequestRetries,235 navigationTimeoutSecs: navigationTimeout / 1000,236 headless,237 238 launchContext: {239 launchOptions: {240 args: [241 '--disable-blink-features=AutomationControlled',242 '--disable-dev-shm-usage',243 '--no-sandbox',244 '--disable-setuid-sandbox',245 '--disable-gpu',246 '--disable-software-rasterizer'247 ],248 }249 },250
251 preNavigationHooks: [252 async ({ page, log }) => {253 try {254 const ua = pickRandom(USER_AGENTS);255
256 await page.setExtraHTTPHeaders({257 'Accept-Language': 'en-US,en;q=0.9',258 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',259 'User-Agent': ua260 });261
262 await page.setViewportSize({ width: 1920, height: 1080 });263
264 await page.addInitScript((ua) => {265 Object.defineProperty(navigator, 'webdriver', { get: () => false });266 Object.defineProperty(navigator, 'userAgent', { get: () => ua });267 }, ua).catch(() => {});268 } catch (e) {269 log.error(`preNavigationHook error: ${e.message}`);270 }271 }272 ],273
274 async requestHandler({ page, request, log }) {275 const { url } = request;276 const isFirstRequest = request.userData?.isFirst || false;277
278 log.info(`🔍 Processing: ${url}`);279
280 try {281 // Handle Location282 if (isFirstRequest) {283 await setPincode(page, log, pincode);284 await delay(1500);285 }286 287 await page.waitForLoadState('domcontentloaded');288 await delay(2000);289 290 // Close popups291 try {292 const closeButton = page.locator('button[aria-label*="Close"]').first();293 if (await closeButton.count() > 0) {294 await closeButton.click({ timeout: 1500 });295 }296 } catch (e) {297 // No popup298 }299 300 const resultsFound = await waitForSearchResults(page, log);301 302 if (!resultsFound) {303 log.warning('⚠️ No search results detected');304 return;305 }306 307 await autoScroll(page, log, scrollCount);308 309 // Extract Data310 const products = await page.evaluate((selectors) => {311 const productCards = [];312 const productLinks = document.querySelectorAll(selectors.productLink);313
314 function textOrNull(el) {315 return el ? (el.textContent || '').trim() : null;316 }317
318 productLinks.forEach((link, index) => {319 try {320 const productUrl = link.href;321 const urlMatch = productUrl.match(/\/pn\/([^/]+)\/pvid\/([^/]+)/) || 322 productUrl.match(/\/(?:p|product)\/([^/]+)\/([^/]+)/);323 const productSlug = urlMatch?.[1] || null;324 const productId = urlMatch?.[2] || `zepto-${index}`;325
326 const card = link.querySelector(selectors.productCard) || link;327
328 // Name extraction329 let productName = null;330 for (const sel of selectors.productName) {331 const el = card.querySelector(sel);332 if (el && textOrNull(el)) {333 productName = textOrNull(el);334 break;335 }336 }337 if (!productName) {338 productName = link.getAttribute('title') || 339 link.querySelector('img')?.alt || null;340 }341
342 // Image343 const imgEl = card.querySelector(selectors.productImage) || link.querySelector('img');344 const productImage = imgEl?.src || imgEl?.getAttribute('data-src') || null;345
346 // Price347 let currentPrice = null;348 const spans = Array.from(card.querySelectorAll(selectors.priceSpan));349 for (const s of spans) {350 const match = (s.textContent || '').match(/₹\s*(\d+(?:,\d+)*(?:\.\d+)?)/);351 if (match) {352 currentPrice = parseFloat(match[1].replace(/,/g, ''));353 break;354 }355 }356
357 // Original price358 let originalPrice = null;359 const origSpan = spans.find(s => 360 /(MRP|strike|original)/i.test(s.className || ''));361 if (origSpan) {362 const match = (origSpan.textContent || '').match(/₹\s*(\d+(?:,\d+)*(?:\.\d+)?)/);363 if (match) originalPrice = parseFloat(match[1].replace(/,/g, ''));364 }365
366 // Discount367 let discountPercentage = null;368 if (currentPrice && originalPrice && originalPrice > currentPrice) {369 discountPercentage = Math.round(((originalPrice - currentPrice) / originalPrice) * 100);370 }371
372 // Pack size373 const packSizeEl = card.querySelector(selectors.packSize);374 const productWeight = packSizeEl ? textOrNull(packSizeEl) : null;375
376 // Rating377 let rating = null;378 const ratingEl = card.querySelector(selectors.rating);379 if (ratingEl) {380 const match = (ratingEl.textContent || '').match(/(\d+\.\d+)/);381 if (match) rating = parseFloat(match[1]);382 }383
384 const isSponsored = !!card.querySelector(selectors.sponsorTag);385 const isOutOfStock = card.getAttribute?.('data-is-out-of-stock') === 'true';386
387 if (productName || currentPrice || productImage) {388 productCards.push({389 productId,390 productSlug,391 productName,392 productImage,393 currentPrice,394 originalPrice,395 discountPercentage,396 productWeight,397 rating,398 isSponsored,399 isOutOfStock,400 productUrl,401 scrapedAt: new Date().toISOString()402 });403 }404 } catch (err) {405 // console.error(`Error processing product ${index}:`, err);406 }407 });408
409 return productCards;410 }, SELECTORS);411 412 if (products.length === 0) {413 log.error('❌ No products extracted');414 return;415 }416 417 const urlParams = new URL(url).searchParams;418 const searchQuery = urlParams.get('query') || request.userData.query;419 420 const productsToSave = products.slice(0, maxProductsPerSearch).map(product => ({421 ...product,422 searchQuery,423 searchUrl: url,424 platform: 'Zepto',425 pincode426 }));427 428 await Dataset.pushData(productsToSave);429 430 log.info(`✅ Saved ${productsToSave.length} products for "${searchQuery}"`);431
432 } catch (error) {433 log.error(`❌ Error: ${error.message}`);434 throw error;435 }436 },437
438 failedRequestHandler: async ({ request, log }) => {439 log.error(`❌ Request failed: ${request.url}`);440 }441});442
443// ==================== EXECUTION ====================444
445const startUrls = [446 ...searchQueries.map(query => ({447 url: `https://www.zepto.com/search?query=${encodeURIComponent(query)}`,448 userData: { 449 query,450 isFirst: true // Mark as first to trigger location set451 }452 })),453 ...searchUrls.map(url => ({454 url,455 userData: {456 query: 'direct_url',457 isFirst: true458 }459 }))460];461
462if (startUrls.length > 0) {463 log.info(`🚀 Starting Zepto Scraper with ${startUrls.length} URLs`);464 465 // We only mark the VERY first request as 'isFirst' for location setting if we want to be strict,466 // but since we might need to set location per session, and we have maxConcurrency: 1 (implied or default),467 // we can try to set it on the first one.468 // However, the logic above checks `request.userData.isFirst`.469 // Let's ensure only the first one has it if we want to avoid repeated attempts, 470 // OR we rely on the `locationSetGlobally` flag if we were using a global variable.471 // But `PlaywrightCrawler` creates new pages.472 // The provided code used a global `locationSetGlobally`.473 // We can use a global variable here too since we are in a single process.474 475 // Let's modify the startUrls to only have isFirst on the first item476 startUrls.forEach((u, i) => {477 u.userData.isFirst = (i === 0);478 });479
480 await crawler.run(startUrls);481 482 log.info('✅ Scraping completed successfully!');483} else {484 log.error('❌ No search URLs or queries provided!');485}486
487// Exit Actor488await Actor.exit();{ "actorSpecification": 1, "name": "jiomart-scrapper", "version": "1.0", "buildTag": "latest", "environmentVariables": {}}