Pricing
from $20.00 / 1,000 results
Pricing
from $20.00 / 1,000 results
Rating
0.0
(0)
Developer
ruly optimism
Actor stats
0
Bookmarked
5
Total users
2
Monthly active users
17 days ago
Last modified
Categories
Share
# Python__pycache__/*.py[cod]*$py.class*.so.Pythonenv/venv/.venv/
# IDE.idea/.vscode/*.swp*.swo
# Apifystorage/apify_storage/.actor/
# Logslogs/*.log
# Outputoutput/
# Cookies (included for Apify deployment)# cookies.json
# Environment.env.env.local
# OS.DS_StoreThumbs.db
# Added by Apify CLIstoragenode_modules.venv# SHEIN Product Scraper - Apify Actor# Base image with Python, Chrome, and SeleniumFROM apify/actor-python-selenium:3.11
# Set working directoryWORKDIR /usr/src/app
# Copy requirements first for cachingCOPY requirements.txt ./
# Install Python dependenciesRUN pip install --no-cache-dir -r requirements.txt
# Copy source codeCOPY . ./
# Explicitly copy cookies.json (ensure it's included)COPY cookies.json ./cookies.json
# Set environment variablesENV PYTHONUNBUFFERED=1ENV PYTHONPATH=/usr/src/app
# Run the actor using __main__.pyCMD ["python", "-m", "src.main"]1"""2Entry point for the Apify actor (for local apify run)3"""4import asyncio5import sys6import os7
8# Add current directory to path9sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))10
11from src.main import main12
13if __name__ == "__main__":14 asyncio.run(main()){ "actorSpecification": 1, "name": "shein-product-scraper", "title": "SHEIN Product Scraper", "description": "Extract comprehensive product data from SHEIN with 95%+ success rate. Features 3-layer captcha bypass, auto-recovery from crashes, and optimized performance. Get prices, sizes, colors, specs, images, and SKUs.", "version": "1.2", "buildTag": "latest", "dockerfile": "./Dockerfile", "readme": "./README.md", "input": "./INPUT_SCHEMA.json", "output": "./dataset_schema.json", "defaultRunOptions": { "memoryMbytes": 1024 }, "storages": { "dataset": { "actorSpecification": 1, "title": "SHEIN Products Dataset", "views": { "products": { "title": "Products", "transformation": { "fields": ["sku", "title", "sale_price", "retail_price", "discount_percentage", "color", "url", "scrape_time_seconds"] }, "display": { "component": "table", "properties": { "sku": { "label": "SKU", "format": "text" }, "title": { "label": "Product Title", "format": "text" }, "sale_price": { "label": "Sale Price", "format": "text" }, "retail_price": { "label": "Retail Price", "format": "text" }, "discount_percentage": { "label": "Discount %", "format": "number" }, "color": { "label": "Color", "format": "text" }, "scrape_time_seconds": { "label": "Time (s)", "format": "number" } } } } } } }}[ { "domain": ".il.shein.com", "expiry": 1797628478, "httpOnly": false, "name": "_pin_unauth", "path": "/", "sameSite": "Lax", "secure": false, "value": "dWlkPVpUWmlOVFl5TkRZdFlXVTVOaTAwTjJZeUxUa3pOamN0Wm1KbFlUY3pOemt3WmpZeQ" }, { "domain": "il.shein.com", "expiry": 1781644471, "httpOnly": false, "name": "g_state", "path": "/", "sameSite": "Lax", "secure": false, "value": "{\"i_l\":0,\"i_ll\":1766092471051,\"i_b\":\"faacCbxKatbTyZo5V8JWyweqOJtnuLgLkXat4NCg2Ic\",\"i_e\":{\"enable_itp_optimization\":0}}" }, { "domain": "il.shein.com", "expiry": 1768313939, "httpOnly": true, "name": "AT", "path": "/", "sameSite": "Lax", "secure": true, "value": "MDEwMDE.eyJiIjo3LCJnIjoxNzY1NzIxOTM4LCJyIjoiUUplTUROIiwidCI6MX0.43ae0e0a93eb3d6d" }, { "domain": "il.shein.com", "expiry": 1766222064, "httpOnly": true, "name": "sessionID_shein", "path": "/", "sameSite": "Lax", "secure": true, "value": "s%3ATxm1FJT7vLsBhZFZu0bXKlZOPhw9jkn3.PKjzteNEd7V1MCkY7BVMIU8U8%2FRqWJdHbzmef9sHTkM" }]1# Apify SDK2apify>=1.6.03apify-client>=1.6.04
5# Selenium & Chrome6selenium>=4.15.07undetected-chromedriver>=3.5.08selenium-stealth>=1.0.69selenium-wire>=5.1.010
11# Utilities12click>=8.1.013requests>=2.31.014
15# Async support16asyncio>=3.4.317
18blinker==1.7.01#!/usr/bin/env python32"""3Local test script for SHEIN scraper (runs without Apify SDK)4Use this to test the scraping logic before deploying to Apify.5
6Usage:7 python test_local.py # With proxy (default)8 python test_local.py "https://il.shein.com/product-url.html"9 PROXY_MODE=false python test_local.py # Disable proxy (use current IP)10 11Note: Proxies are ENABLED by default for Israel domain (il.shein.com).12"""13
14import sys15import os16import time17import json18import random19from datetime import datetime20
21# Add src to path22sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))23
24from src.browser import (25 get_browser, close_browser, close_popup, load_cookies,26 get_proxies_from_api, get_random_proxy, get_next_proxy,27 should_use_proxies_com28)29from src.scraper import scrape_product, check_captcha, handle_captcha, CaptchaDetected30from src.parser import clean_url31
32
33def run_local_test(urls, cookies_file="cookies.json"):34 """Run the scraper locally without Apify SDK"""35 36 print(f"Starting scraper for {len(urls)} URL(s)")37 38 # Stats39 stats = {40 "total": len(urls),41 "success": 0,42 "failed": 0,43 "captcha_bypasses": 0,44 "start_time": datetime.now()45 }46 47 results = []48 driver = None49 current_proxy = None50 use_proxies_com = False51 52 try:53 # Proxy ENABLED by default (set PROXY_MODE=false to disable)54 proxy_mode = os.environ.get('PROXY_MODE', 'true').lower() == 'true'55 56 # Use Smartproxy for Israel domain when enabled57 if proxy_mode and should_use_proxies_com(urls):58 proxies = get_proxies_from_api(proxy_name="shein")59 if proxies:60 use_proxies_com = True61 current_proxy = get_random_proxy()62 63 print(f"Proxy: {'enabled' if current_proxy else 'disabled'}")64 65 # Initialize browser66 headless = os.environ.get('HEADLESS', 'true').lower() != 'false'67 print("Initializing browser...")68 driver = get_browser(proxy=current_proxy, headless=headless)69 70 # Load cookies71 if not os.path.exists(cookies_file):72 parent_cookies = os.path.join(os.path.dirname(os.path.dirname(__file__)), "cookies.json")73 if os.path.exists(parent_cookies):74 cookies_file = parent_cookies75 76 # Determine base domain from URLs77 base_domain = "https://il.shein.com"78 if urls and 'ar.shein.com' in urls[0].lower():79 base_domain = "https://ar.shein.com"80 81 cookies_loaded = load_cookies(driver, cookie_file=cookies_file)82 if not cookies_loaded:83 driver.get(base_domain)84 time.sleep(1)85 86 # Wait for session87 time.sleep(1)88 close_popup(driver)89 close_popup(driver)90 91 # Check session92 if check_captcha(driver):93 print("Session blocked")94 else:95 print("Session ready")96 97 # Scrape each URL98 for idx, url in enumerate(urls, 1):99 url_start = time.time()100 101 print(f"[{idx}/{len(urls)}] Processing...")102 103 retry_count = 0104 max_retries = 3105 skip_navigation = False106 107 while retry_count < max_retries:108 try:109 product_data = scrape_product(110 driver,111 url,112 timeout=60,113 include_images=True,114 include_reviews=True,115 skip_navigation=skip_navigation116 )117 118 if product_data:119 scrape_time = round(time.time() - url_start, 2)120 product_data['url'] = clean_url(url)121 product_data['scrape_time_seconds'] = scrape_time122 product_data['scraped_at'] = datetime.now().isoformat()123 124 results.append(product_data)125 stats["success"] += 1126 print(f"✓ Done ({scrape_time}s)")127 break128 else:129 raise Exception("No data returned")130 131 except CaptchaDetected as e:132 bypass_success = handle_captcha(driver, url)133 134 if bypass_success:135 skip_navigation = True # Already on product URL136 stats["captcha_bypasses"] += 1137 continue138 else:139 if use_proxies_com:140 close_browser(driver)141 time.sleep(1)142 current_proxy = get_next_proxy(current_proxy)143 driver = get_browser(proxy=current_proxy, headless=False)144 load_cookies(driver, cookie_file=cookies_file)145 time.sleep(1)146 close_popup(driver)147 148 if not check_captcha(driver):149 print(" ✓ Captcha bypassed (Strategy 3: IP rotation)")150 skip_navigation = False151 stats["captcha_bypasses"] += 1152 continue153 else:154 print(" ✗ Strategy 3 failed, retrying...")155 156 retry_count += 1157 skip_navigation = False158 159 except Exception as e:160 retry_count += 1161 skip_navigation = False162 163 if retry_count >= max_retries:164 stats["failed"] += 1165 results.append({166 "url": url,167 "error": str(e),168 "status": "failed"169 })170 171 # Delay between requests172 if idx < len(urls):173 time.sleep(1)174 175 # Final stats176 stats["end_time"] = datetime.now()177 duration = (stats["end_time"] - stats["start_time"]).total_seconds()178 179 print(f"Complete: {stats['success']}/{stats['total']} in {duration:.1f}s")180 181 # Save results182 output_file = f"output/local_test_{int(time.time())}.json"183 os.makedirs("output", exist_ok=True)184 185 with open(output_file, 'w', encoding='utf-8') as f:186 json.dump({187 "stats": {188 "total": stats["total"],189 "success": stats["success"],190 "failed": stats["failed"],191 "duration_seconds": duration192 },193 "results": results194 }, f, indent=2, ensure_ascii=False)195 196 return results197 198 except Exception as e:199 print(f"Error: {e}")200 201 finally:202 if driver:203 close_browser(driver)204
205
206if __name__ == "__main__":207 # Default test URL (Israel domain - use PROXY_MODE=true for Smartproxy)208 test_urls = [209 "https://il.shein.com/SHEIN-BAE-Sexy-Lace-Sheer-Camisole-Top-p-70811510.html"210 ]211 212 # Use command line URL if provided213 if len(sys.argv) > 1:214 test_urls = sys.argv[1:]215 216 run_local_test(test_urls)1"""2SHEIN Product Scraper - Apify Actor3
4Fast and reliable scraping of SHEIN product pages.5"""6
7from .browser import get_browser, close_browser, close_popup8from .scraper import scrape_product, handle_captcha, CaptchaDetected9from .parser import parse_shein_product10from .utils import load_cookies_from_string, setup_proxy, clean_url11
12__version__ = "1.0.0"13__author__ = "Your Name"14
15__all__ = [16 'get_browser',17 'close_browser',18 'close_popup',19 'scrape_product',20 'handle_captcha',21 'CaptchaDetected',22 'parse_shein_product',23 'load_cookies_from_string',24 'setup_proxy',25 'clean_url'26]1"""2Browser configuration and management for SHEIN scraper3Supports Smartproxy (Israel) and Apify Proxy for reliable scraping with full captcha bypass.4"""5
6import undetected_chromedriver as uc7from selenium_stealth import stealth8from selenium.webdriver.common.by import By9import time10import json11import os12import random13import tempfile14import logging15
16# Suppress selenium-wire noise17logging.getLogger('seleniumwire').setLevel(logging.ERROR)18logging.getLogger('hpack').setLevel(logging.ERROR)19logging.getLogger('urllib3').setLevel(logging.ERROR)20
21# Try to import selenium-wire for proxy auth22try:23 from seleniumwire import webdriver as wire_webdriver24 SELENIUMWIRE_AVAILABLE = True25except ImportError:26 SELENIUMWIRE_AVAILABLE = False27
28
29# Fingerprint profile (FIXED for cookie compatibility - MUST match cookies!)30FINGERPRINT = {31 "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",32 "platform": "Win32",33 "vendor": "Google Inc.",34 "webgl_vendor": "Intel Inc.",35 "renderer": "Intel Iris OpenGL Engine"36}37
38# =====================================================39# SMARTPROXY CONFIGURATION (for Israel domain only)40# Rotating residential proxy - NEW IP per session41# =====================================================42SMARTPROXY_SERVER = "proxy.smartproxy.net"43SMARTPROXY_PORT = "3120"44# SMARTPROXY_USERNAME_BASE = "smart-v1zmwetcduzk_area-IL"45
46SMARTPROXY_USERNAME_BASE = "smart-v1zmwetcduzk"47SMARTPROXY_PASSWORD = "PcnWBokjVDGawLA9"48
49_proxy_initialized = False50_current_session_id = None51
52
53def _generate_session_id():54 """Generate unique session ID for new IP"""55 import uuid56 return str(uuid.uuid4())[:8]57
58
59def _build_proxy_url(session_id=None):60 """Build Smartproxy URL with optional session for IP rotation"""61 if session_id:62 # Adding session ID forces new IP63 username = f"{SMARTPROXY_USERNAME_BASE}_session-{session_id}"64 else:65 username = SMARTPROXY_USERNAME_BASE66 return f"http://{username}:{SMARTPROXY_PASSWORD}@{SMARTPROXY_SERVER}:{SMARTPROXY_PORT}"67
68
69def is_israel_domain(url):70 """Check if URL is for Israel SHEIN domain"""71 return 'il.shein.com' in url.lower() if url else False72
73
74def is_supported_domain(url):75 """Check if URL is for a supported SHEIN domain (IL, AR, etc.)"""76 if not url:77 return False78 url_lower = url.lower()79 supported = ['il.shein.com', 'ar.shein.com']80 return any(domain in url_lower for domain in supported)81
82
83def should_use_smartproxy(urls):84 """85 Determine if we should use Smartproxy based on URLs86 Use for supported domains (il.shein.com, ar.shein.com)87 """88 if not urls:89 return False90 for url in urls:91 if is_supported_domain(url):92 return True93 return False94
95
96# Keep old function names for compatibility97should_use_proxies_com = should_use_smartproxy98
99
100def get_proxies_from_api(proxy_name=None):101 """Initialize Smartproxy (no API call needed - single endpoint)"""102 global _proxy_initialized, _current_session_id103 _proxy_initialized = True104 _current_session_id = _generate_session_id()105 return [_build_proxy_url(_current_session_id)]106
107
108def get_random_proxy(exclude_proxy=None):109 """Get Smartproxy URL with NEW session ID (new IP)"""110 global _current_session_id111 _current_session_id = _generate_session_id()112 return _build_proxy_url(_current_session_id)113
114
115def get_next_proxy(current_proxy=None):116 """Get Smartproxy URL with NEW session ID (forces new IP)"""117 global _current_session_id118 _current_session_id = _generate_session_id()119 return _build_proxy_url(_current_session_id)120
121
122def create_proxy_auth_extension(proxy_host, proxy_user, proxy_pass):123 """124 Create a Chrome extension that handles proxy authentication automatically.125 This avoids the auth popup that Chrome shows for authenticated proxies.126 127 Returns:128 Path to the extension directory129 """130 import tempfile131 import zipfile132 133 # Parse host and port134 if ':' in proxy_host:135 host, port = proxy_host.rsplit(':', 1)136 else:137 host = proxy_host138 port = "80"139 140 manifest_json = """141{142 "version": "1.0.0",143 "manifest_version": 2,144 "name": "Proxy Auth Extension",145 "permissions": [146 "proxy",147 "tabs",148 "unlimitedStorage",149 "storage",150 "<all_urls>",151 "webRequest",152 "webRequestBlocking"153 ],154 "background": {155 "scripts": ["background.js"]156 }157}158"""159
160 background_js = """161var config = {162 mode: "fixed_servers",163 rules: {164 singleProxy: {165 scheme: "http",166 host: "%s",167 port: parseInt(%s)168 },169 bypassList: ["localhost"]170 }171};172
173chrome.proxy.settings.set({value: config, scope: "regular"}, function() {});174
175function callbackFn(details) {176 return {177 authCredentials: {178 username: "%s",179 password: "%s"180 }181 };182}183
184chrome.webRequest.onAuthRequired.addListener(185 callbackFn,186 {urls: ["<all_urls>"]},187 ['blocking']188);189""" % (host, port, proxy_user, proxy_pass)190
191 # Create extension in temp directory192 ext_dir = tempfile.mkdtemp(prefix="proxy_auth_")193 194 with open(os.path.join(ext_dir, "manifest.json"), 'w') as f:195 f.write(manifest_json)196 197 with open(os.path.join(ext_dir, "background.js"), 'w') as f:198 f.write(background_js)199 200 return ext_dir201
202
203def get_browser(proxy=None, headless=True):204 """205 Initialize and return a configured Chrome browser instance206 207 Args:208 proxy: Proxy string - can be:209 - proxies.com format: "http://user:pass@ip:port" 210 - Apify format: "http://user:pass@proxy.apify.com:8000"211 - Simple format: "ip:port" or "http://ip:port"212 headless: Run in headless mode (default True for Apify)213 214 Returns:215 WebDriver instance216 """217 # Parse proxy to extract auth credentials and host218 proxy_host = None219 proxy_user = None220 proxy_pass = None221 222 if proxy:223 # Parse proxy URL to extract credentials224 # Format: http://user:pass@ip:port or socks5://user:pass@ip:port or ip:port225 if '@' in proxy:226 # Has authentication227 # Split: protocol://user:pass@host:port228 if '://' in proxy:229 protocol, rest = proxy.split('://', 1)230 auth_part, host_part = rest.rsplit('@', 1)231 if ':' in auth_part:232 proxy_user, proxy_pass = auth_part.split(':', 1)233 proxy_host = host_part234 else:235 # user:pass@host:port (no protocol)236 auth_part, host_part = proxy.rsplit('@', 1)237 if ':' in auth_part:238 proxy_user, proxy_pass = auth_part.split(':', 1)239 proxy_host = host_part240 pass # Authenticated proxy241 else:242 # No authentication - just host:port243 if '://' in proxy:244 proxy_host = proxy.split('://', 1)[1]245 else:246 proxy_host = proxy247 248 249 # Initialize driver250 driver = None251 252 try:253 # For authenticated proxy, use selenium-wire (handles auth at network level)254 if proxy_user and proxy_pass and proxy_host:255 if not SELENIUMWIRE_AVAILABLE:256 raise RuntimeError("selenium-wire required for authenticated proxy. pip install selenium-wire")257 from seleniumwire import webdriver as sw258 from selenium.webdriver.chrome.options import Options259 from selenium.webdriver.chrome.service import Service260 import subprocess261 262 options = Options()263 options.page_load_strategy = 'eager' # Don't wait for all resources264 options.add_argument("--window-size=1920,1080")265 options.add_argument(f"user-agent={FINGERPRINT['user_agent']}")266 options.add_argument("--no-sandbox")267 options.add_argument("--disable-dev-shm-usage")268 options.add_argument("--disable-gpu")269 options.add_argument("--disable-infobars")270 options.add_argument("--ignore-certificate-errors")271 options.add_argument("--ignore-ssl-errors")272 options.add_argument("--blink-settings=imagesEnabled=false")273 # Block more resources to save proxy bandwidth274 options.add_argument("--disable-remote-fonts")275 options.add_argument("--disable-plugins")276 options.add_argument("--disable-extensions")277 options.add_argument("--disable-default-apps")278 # Speed up Chrome startup279 options.add_argument("--disable-background-networking")280 options.add_argument("--disable-background-timer-throttling")281 options.add_argument("--disable-backgrounding-occluded-windows")282 options.add_argument("--disable-breakpad")283 options.add_argument("--disable-component-extensions-with-background-pages")284 options.add_argument("--disable-features=TranslateUI")285 options.add_argument("--disable-hang-monitor")286 options.add_argument("--disable-ipc-flooding-protection")287 options.add_argument("--disable-renderer-backgrounding")288 options.add_argument("--disable-sync")289 options.add_argument("--metrics-recording-only")290 options.add_argument("--no-first-run")291 options.add_argument("--safebrowsing-disable-auto-update")292 # Stability fixes for renderer crashes293 options.add_argument("--disable-software-rasterizer")294 options.add_argument("--disable-setuid-sandbox")295 options.add_argument("--disable-features=VizDisplayCompositor")296 # Linux/Apify-only stability flags (crash on macOS)297 import platform298 if platform.system() == "Linux":299 options.add_argument("--single-process")300 options.add_argument("--memory-pressure-off")301 if headless:302 options.add_argument("--headless=new")303 304 # Selenium-wire proxy config - use upstream proxy without intercepting305 proxy_url = f"http://{proxy_user}:{proxy_pass}@{proxy_host}"306 sw_options = {307 'proxy': {308 'http': proxy_url,309 'https': proxy_url,310 'no_proxy': 'localhost,127.0.0.1'311 },312 'disable_capture': True, # Don't intercept, just route313 'verify_ssl': False,314 'suppress_connection_errors': True,315 'connection_timeout': 30, # Increased for TLS handshake316 'request_storage_max_size': 0, # Don't store requests317 'disable_encoding': True, # Don't modify encoding318 }319 320 driver = sw.Chrome(options=options, seleniumwire_options=sw_options)321 # Set timeouts to prevent hanging322 driver.set_page_load_timeout(45) # Reduced - fail faster323 driver.set_script_timeout(20)324 325 # No proxy or no auth - use undetected-chromedriver326 else:327 options = uc.ChromeOptions()328 options.page_load_strategy = 'eager'329 options.add_argument("--window-size=1920,1080")330 options.add_argument(f"user-agent={FINGERPRINT['user_agent']}")331 options.add_argument("--no-sandbox")332 options.add_argument("--disable-dev-shm-usage")333 options.add_argument("--disable-infobars")334 options.add_argument("--disable-notifications")335 options.add_argument("--disable-popup-blocking")336 options.add_argument("--blink-settings=imagesEnabled=false")337 options.add_argument("--disable-blink-features=AutomationControlled")338 # Speed up Chrome startup339 options.add_argument("--disable-background-networking")340 options.add_argument("--disable-background-timer-throttling")341 options.add_argument("--disable-breakpad")342 options.add_argument("--disable-features=TranslateUI")343 options.add_argument("--disable-hang-monitor")344 options.add_argument("--disable-sync")345 options.add_argument("--no-first-run")346 # Stability fixes for renderer crashes347 options.add_argument("--disable-software-rasterizer")348 options.add_argument("--disable-setuid-sandbox")349 # Linux/Apify-only stability flags (crash on macOS)350 import platform351 if platform.system() == "Linux":352 options.add_argument("--single-process")353 options.add_argument("--memory-pressure-off")354 if headless:355 options.add_argument("--headless=new")356 options.add_argument("--disable-gpu")357 if proxy_host:358 options.add_argument(f"--proxy-server=http://{proxy_host}")359 360 driver = uc.Chrome(options=options, version_main=143)361 # Set timeouts to prevent hanging362 driver.set_page_load_timeout(45) # Reduced from 60363 driver.set_script_timeout(20) # Reduced from 30364 365 except Exception as e:366 raise367 368 # Apply stealth settings369 try:370 stealth(driver,371 languages=["en-US", "en"],372 vendor=FINGERPRINT["vendor"],373 platform=FINGERPRINT["platform"],374 webgl_vendor=FINGERPRINT["webgl_vendor"],375 renderer=FINGERPRINT["renderer"],376 fix_hairline=True,377 webdriver=False378 )379 except:380 pass381 382 # Block heavy resources to save proxy bandwidth383 try:384 driver.execute_cdp_cmd('Network.setBlockedURLs', {385 'urls': [386 '*.woff', '*.woff2', '*.ttf', '*.otf', # Fonts387 '*.mp4', '*.webm', '*.avi', # Videos388 '*google-analytics*', '*googletagmanager*', # Analytics389 '*facebook*', '*fbcdn*', # FB tracking390 '*doubleclick*', '*adsense*', # Ads391 '*.gif', # Animated images392 ]393 })394 driver.execute_cdp_cmd('Network.enable', {})395 except:396 pass397 398 # Inject popup suppression CSS (proactive)399 try:400 driver.execute_cdp_cmd('Page.addScriptToEvaluateOnNewDocument', {401 'source': """402 // Inject CSS to hide popups immediately on page load403 const style = document.createElement('style');404 style.textContent = `405 .coupon-dialog,406 .coupon-dialog__coupon-content,407 .sui-dialog,408 .popup-overlay,409 .modal-overlay,410 .dialog-overlay,411 [class*="coupon-dialog"],412 [class*="promotion-dialog"],413 [role="dialog"] {414 display: none !important;415 visibility: hidden !important;416 opacity: 0 !important;417 }418 `;419 document.head.appendChild(style);420 421 // Also prevent body scroll lock422 document.addEventListener('DOMContentLoaded', function() {423 document.body.style.overflow = 'auto !important';424 document.body.style.position = 'static !important';425 });426 """427 })428 except:429 pass430 431 # Set timeouts432 driver.implicitly_wait(10)433 driver.set_page_load_timeout(60)434 driver.set_script_timeout(30)435 436 return driver437
438
439def close_browser(driver):440 """Safely close the browser"""441 try:442 driver.quit()443 except:444 pass445
446
447def close_popup(driver):448 """Close any popups on the page (fast, non-blocking)"""449 try:450 driver.execute_script("""451 const selectors = '.coupon-dialog, .sui-dialog, [role="dialog"], .popup-overlay, .modal-overlay, .she-mask, [class*="coupon"], [class*="modal"], [class*="popup"]';452 document.querySelectorAll(selectors).forEach(el => {453 el.style.display = 'none';454 el.style.visibility = 'hidden';455 });456 document.querySelectorAll('.sui-dialog__close, .sui-icon-close, .icon-close, [aria-label="close"], [aria-label="Close"]').forEach(btn => {457 try { btn.click(); } catch(e) {}458 });459 document.body.style.overflow = 'auto';460 document.body.style.position = 'static';461 """)462 return True463 except:464 return False465
466
467def load_cookies(driver, cookies_json=None, cookie_file="cookies.json"):468 """Load cookies from JSON string or file to establish session."""469 cookies = None470 471 # Try from JSON string first (Apify input)472 if cookies_json:473 try:474 if isinstance(cookies_json, str):475 cookies = json.loads(cookies_json)476 elif isinstance(cookies_json, list):477 cookies = cookies_json478 except:479 pass480 481 # Try from file482 if not cookies:483 possible_paths = [cookie_file, f"/usr/src/app/{cookie_file}", f"./{cookie_file}"]484 for path in possible_paths:485 if os.path.exists(path):486 try:487 with open(path, 'r') as f:488 cookies = json.load(f)489 break490 except:491 pass492 493 if not cookies:494 return False495 496 # Navigate to domain first497 driver.get("https://il.shein.com")498 time.sleep(2)499 500 # Set cookies501 success_count = 0502 for cookie in cookies:503 try:504 driver.execute_cdp_cmd('Network.setCookie', {505 'name': cookie.get('name'),506 'value': cookie.get('value'),507 'domain': '.shein.com',508 'path': '/'509 })510 success_count += 1511 except:512 pass513 514 # Refresh to apply515 if success_count > 0:516 driver.refresh()517 time.sleep(1)518 519 return success_count > 01"""2SHEIN Product Scraper - Apify Actor3
4A fast and reliable scraper for extracting product data from SHEIN.5
6Features:7- 3-layer captcha bypass system8- Auto-recovery from browser crashes9- Optimized Chrome startup10- Page load timeouts (60s max)11- Smart error handling12
13Captcha Bypass Strategies:141. Strategy 1: Main domain redirect (~2-3s)152. Strategy 2: Wait & retry (~4-5s)163. Strategy 3: IP rotation with new proxy (~40-50s)17
18Version: 1.219"""20
21import asyncio22import json23import os24import time25import random26from datetime import datetime27
28# Apify SDK29from apify import Actor30
31# Local imports32from .browser import (33 get_browser, close_browser, close_popup, load_cookies,34 get_proxies_from_api, get_random_proxy, get_next_proxy,35 should_use_proxies_com36)37from .scraper import scrape_product, check_captcha, handle_captcha, CaptchaDetected38from .parser import parse_shein_product, clean_url39
40
41async def main():42 """Main entry point for the Apify actor"""43 44 async with Actor:45 # Get input46 actor_input = await Actor.get_input() or {}47 48 urls = actor_input.get('urls', [])49 max_retries = actor_input.get('maxRetries', 2)50 max_captcha_retries = actor_input.get('maxCaptchaRetries', 1) # 1 IP rotation max, then skip51 proxy_config = actor_input.get('proxyConfiguration', {})52 cookies_json = actor_input.get('cookiesJson', '')53 include_images = actor_input.get('includeImages', True)54 include_reviews = actor_input.get('includeReviews', True)55 timeout = actor_input.get('timeout', 30) # 30s default (faster fail)56 delay_ms = actor_input.get('delayBetweenRequests', 1000)57 58 if not urls:59 Actor.log.error("No URLs provided")60 return61 62 # Limit to 5 URLs per run for stability63 MAX_URLS = 564 if len(urls) > MAX_URLS:65 Actor.log.warning(f"Too many URLs ({len(urls)}). Limited to {MAX_URLS}. Run multiple times for more.")66 urls = urls[:MAX_URLS]67 68 Actor.log.info(f"Scraping {len(urls)} URL(s)")69 70 # Track stats71 stats = {72 "total": len(urls),73 "success": 0,74 "failed": 0,75 "captcha_bypasses": 0,76 "start_time": datetime.now().isoformat()77 }78 79 # =====================================================80 # PROXY SETUP81 # Default: ENABLED (use Smartproxy for Israel domain)82 # Set useProxy=false in input to disable83 # =====================================================84 proxy_configuration = None85 current_proxy_url = None86 use_proxies_com = False87 88 # Proxy enabled by default, can be disabled via input or env89 proxy_enabled = (90 os.environ.get('PROXY_MODE', 'true').lower() == 'true' and91 actor_input.get('useProxy', True) != False92 )93 94 if proxy_enabled:95 if should_use_proxies_com(urls):96 proxies = get_proxies_from_api(proxy_name="shein")97 if proxies:98 use_proxies_com = True99 current_proxy_url = get_random_proxy()100 101 if not use_proxies_com and proxy_config.get('useApifyProxy', False):102 proxy_groups = proxy_config.get('apifyProxyGroups', ['RESIDENTIAL'])103 proxy_configuration = await Actor.create_proxy_configuration(groups=proxy_groups)104 current_proxy_url = await proxy_configuration.new_url()105 106 Actor.log.info(f"Proxy: {'enabled' if current_proxy_url else 'disabled'}")107 Actor.log.info("Initializing browser...")108 driver = None109 captcha_retry_count = 0110 111 try:112 driver = get_browser(113 proxy=current_proxy_url,114 headless=True115 )116 117 # Load cookies118 cookies_loaded = load_cookies(driver, cookies_json)119 if not cookies_loaded:120 driver.get("https://il.shein.com")121 time.sleep(1)122 123 # Wait for session124 time.sleep(1)125 try:126 close_popup(driver)127 close_popup(driver)128 except:129 pass130 131 # Check session132 if check_captcha(driver):133 Actor.log.error("Session blocked")134 else:135 Actor.log.info("Session ready")136 137 # Scrape URLs138 MAX_TIME_PER_URL = 60 # Fail URL if taking longer than 60s - fail fast139 140 for idx, url in enumerate(urls, 1):141 try: # URL-level try/catch to ensure we always continue142 url_start_time = time.time()143 Actor.log.info(f"[{idx}/{len(urls)}] Processing...")144 145 retry_count = 0146 success = False147 skip_navigation = False148 149 while retry_count < max_retries and not success:150 # Time limit check151 if time.time() - url_start_time > MAX_TIME_PER_URL:152 Actor.log.warning(f"URL timeout ({MAX_TIME_PER_URL}s) - skipping")153 stats["failed"] += 1154 await Actor.push_data({155 "url": clean_url(url),156 "error": "Time limit exceeded",157 "status": "failed",158 "scraped_at": datetime.now().isoformat()159 })160 break161 try:162 # Scrape the product163 product_data = scrape_product(164 driver, 165 url,166 timeout=timeout,167 include_images=include_images,168 include_reviews=include_reviews,169 skip_navigation=skip_navigation170 )171 172 if product_data:173 # Add metadata174 scrape_time = round(time.time() - url_start_time, 2)175 product_data['url'] = clean_url(url)176 product_data['scrape_time_seconds'] = scrape_time177 product_data['scraped_at'] = datetime.now().isoformat()178 179 # Push to dataset180 await Actor.push_data(product_data)181 182 stats["success"] += 1183 success = True184 Actor.log.info(f"✓ Done ({scrape_time}s)")185 else:186 raise Exception("No data returned")187 188 except CaptchaDetected as e:189 captcha_retry_count += 1190 191 if captcha_retry_count > max_captcha_retries:192 Actor.log.warning("Blocked - skipping URL")193 stats["failed"] += 1194 await Actor.push_data({195 "url": clean_url(url),196 "error": "Captcha/blocked",197 "status": "failed",198 "scraped_at": datetime.now().isoformat()199 })200 break201 202 bypass_success = handle_captcha(driver, url)203 204 if bypass_success:205 skip_navigation = True # Already on product URL206 stats["captcha_bypasses"] += 1207 captcha_retry_count = 0208 continue209 else:210 # Check time limit before IP rotation211 if time.time() - url_start_time > MAX_TIME_PER_URL:212 Actor.log.warning(f"URL timeout ({MAX_TIME_PER_URL}s) - skipping")213 stats["failed"] += 1214 await Actor.push_data({215 "url": clean_url(url),216 "error": "Timeout during captcha handling",217 "status": "failed",218 "scraped_at": datetime.now().isoformat()219 })220 break221 222 # Rotate IP (with error handling)223 try:224 close_browser(driver)225 except:226 pass227 await asyncio.sleep(1)228 229 old_proxy = current_proxy_url230 if use_proxies_com:231 current_proxy_url = get_next_proxy(old_proxy)232 elif proxy_configuration:233 current_proxy_url = await proxy_configuration.new_url()234 235 try:236 driver = get_browser(proxy=current_proxy_url, headless=True)237 load_cookies(driver, cookies_json)238 time.sleep(2) # Longer wait after new IP239 try:240 close_popup(driver)241 except:242 pass243 244 if not check_captcha(driver):245 print(" ✓ Captcha bypassed (Strategy 3: IP rotation)")246 skip_navigation = False247 stats["captcha_bypasses"] += 1248 captcha_retry_count = 0249 await asyncio.sleep(random.uniform(2, 4)) # Human-like delay250 continue251 else:252 print(" ✗ Strategy 3 failed, retrying...")253 retry_count += 1254 skip_navigation = False255 continue256 except Exception as ip_error:257 print(f" ✗ IP rotation failed: {str(ip_error)[:50]}")258 retry_count += 1259 # Try to get a fresh browser for next attempt260 await asyncio.sleep(2)261 try:262 driver = get_browser(proxy=current_proxy_url, headless=True)263 load_cookies(driver, cookies_json)264 except:265 pass266 skip_navigation = False267 continue268 269 except Exception as e:270 error_msg = str(e)271 retry_count += 1272 273 # Check if it's a timeout or browser crash - need to restart browser274 if "timeout" in error_msg.lower() or "renderer" in error_msg.lower() or "session" in error_msg.lower():275 Actor.log.warning(f"Browser crashed, restarting...")276 stats["failed"] += 1277 await Actor.push_data({278 "url": url,279 "error": "Browser timeout/crash",280 "status": "failed",281 "scraped_at": datetime.now().isoformat()282 })283 284 # Restart browser for next URLs (with error handling)285 try:286 close_browser(driver)287 except:288 pass289 await asyncio.sleep(2)290 try:291 driver = get_browser(proxy=current_proxy_url, headless=True)292 load_cookies(driver, cookies_json)293 time.sleep(1)294 except Exception as restart_error:295 Actor.log.warning(f"Browser restart failed, will retry...")296 await asyncio.sleep(3)297 try:298 driver = get_browser(proxy=current_proxy_url, headless=True)299 load_cookies(driver, cookies_json)300 except:301 Actor.log.error("Could not restart browser")302 break # Skip to next URL303 304 if retry_count < max_retries:305 try:306 if check_captcha(driver):307 skip_navigation = False308 raise CaptchaDetected(f"Captcha at {url}")309 except:310 pass311 time.sleep(1)312 skip_navigation = False313 else:314 stats["failed"] += 1315 Actor.log.warning(f"Failed: {url[:50]}")316 await Actor.push_data({317 "url": url,318 "error": error_msg[:200],319 "status": "failed",320 "scraped_at": datetime.now().isoformat()321 })322 323 # Delay between requests324 if idx < len(urls):325 delay_seconds = delay_ms / 1000326 await asyncio.sleep(delay_seconds)327 328 except Exception as url_error:329 # Catch ANY error for this URL and continue to next330 Actor.log.warning(f"URL failed: {str(url_error)[:80]}")331 stats["failed"] += 1332 await Actor.push_data({333 "url": url,334 "error": str(url_error)[:200],335 "status": "failed",336 "scraped_at": datetime.now().isoformat()337 })338 # Try to restart browser for next URL339 try:340 close_browser(driver)341 except:342 pass343 await asyncio.sleep(2)344 try:345 driver = get_browser(proxy=current_proxy_url, headless=True)346 load_cookies(driver, cookies_json)347 except:348 Actor.log.warning("Could not restart browser")349 continue # Always continue to next URL350 351 # =====================================================352 # FINAL STATS353 # =====================================================354 stats["end_time"] = datetime.now().isoformat()355 stats["duration_seconds"] = round(356 (datetime.fromisoformat(stats["end_time"]) - 357 datetime.fromisoformat(stats["start_time"])).total_seconds(), 2358 )359 360 Actor.log.info(f"\n{'='*50}")361 Actor.log.info(f"✅ Scraping Complete!")362 Actor.log.info(f"Complete: {stats['success']}/{stats['total']} in {stats['duration_seconds']}s")363 364 # Save stats to key-value store365 await Actor.set_value('STATS', stats)366 367 except Exception as e:368 Actor.log.error(f"Error: {str(e)}")369 370 finally:371 if driver:372 close_browser(driver)373
374
375if __name__ == "__main__":376 asyncio.run(main())1"""2SHEIN product data parser3
4Complete parser for SHEIN product JSON (gbRawData).5Handles the full data structure with all variants, sizes, specs, etc.6"""7
8import json9import re10
11
12def clean_url(url):13 """Clean URL - remove query parameters after .html"""14 if not url:15 return url16 # Find .html and truncate everything after it17 match = re.search(r'(.*?\.html)', url)18 if match:19 return match.group(1)20 return url21
22
23def parse_shein_product(json_data):24 """25 Complete parser for SHEIN product JSON26 Returns formatted output matching the required structure27 28 Args:29 json_data: The gbRawData JavaScript object (dict or JSON string)30 31 Returns:32 dict: Structured product data33 """34 if isinstance(json_data, str):35 data = json.loads(json_data)36 else:37 data = json_data38 39 modules = data.get('modules', {})40 product_info = modules.get('productInfo', {})41 sale_attr = modules.get('saleAttr', {})42 price_info = modules.get('priceInfo', {})43 44 def fix_url(url):45 """Add http: prefix to URLs starting with //"""46 if url and url.startswith('//'):47 return 'http:' + url48 return url49 50 # Basic product info51 goods_id = product_info.get('goods_id', '')52 cat_id = product_info.get('cat_id', '')53 goods_sn = product_info.get('goods_sn', '')54 url = f"https://il.shein.com/product-p-{goods_id}-cat-{cat_id}.html"55 56 # Main image57 main_image = fix_url(product_info.get('goods_img', ''))58 59 # Get all images (convert 900x to 750x999)60 skc_images = product_info.get('currentSkcImgInfo', {}).get('skcImages', [])61 images = [fix_url(img.replace('_thumbnail_900x.webp', '_thumbnail_750x999.webp')) 62 for img in skc_images]63 64 # Get color from product details65 color = "Unknown"66 product_details = product_info.get('productDescriptionInfo', {}).get('productDetails', [])67 for detail in product_details:68 if detail.get('attr_name') == 'Color':69 color = detail.get('attr_value')70 break71 72 # Parse prices73 retail_price_data = price_info.get('retailPrice', {})74 sale_price_data = price_info.get('salePrice', {})75 76 retail_price = {77 "amount": float(retail_price_data.get('amount', 0)),78 "amount_with_symbol": retail_price_data.get('amountWithSymbol', ''),79 "usd_amount": float(retail_price_data.get('usdAmount', 0)),80 "usd_amount_with_symbol": retail_price_data.get('usdAmountWithSymbol', '')81 }82 83 sale_price = {84 "amount": float(sale_price_data.get('amount', 0)),85 "amount_with_symbol": sale_price_data.get('amountWithSymbol', ''),86 "usd_amount": float(sale_price_data.get('usdAmount', 0)),87 "usd_amount_with_symbol": sale_price_data.get('usdAmountWithSymbol', '')88 }89 90 # Calculate discount91 has_discount = retail_price['amount'] != sale_price['amount']92 discount_percentage = 093 if has_discount and retail_price['amount'] > 0:94 discount_percentage = round(((retail_price['amount'] - sale_price['amount']) 95 / retail_price['amount']) * 100)96 97 # Parse sizes98 multi_level = sale_attr.get('multiLevelSaleAttribute', {})99 skc_sale_attr = multi_level.get('skc_sale_attr', [])100 101 sizes = []102 # If skc_sale_attr is missing or empty, checking saleAttr structure103 if not skc_sale_attr:104 # Sometimes it's in sizeInfo -> sizeInfo105 size_info_list = sale_attr.get('sizeInfo', {}).get('sizeInfo', [])106 for s in size_info_list:107 size_obj = {108 "attr_value_name": s.get('attr_value_name', ''),109 "attr_value_name_en": s.get('attr_value_name_en', ''),110 "is_sold_out": False, # Basic assumption if listed111 "attr_value_local_size": [],112 "size_chart": []113 }114 # Attempt to populate size chart data if embedded plain115 chart_data = {}116 for key, val in s.items():117 if key not in ['attr_id', 'attr_name', 'attr_value_id', 'attr_value_name']:118 chart_data[key] = val119 if chart_data:120 size_obj['size_chart'].append(chart_data) # Simplification121 sizes.append(size_obj)122
123 # Standard loop over skc_sale_attr if present124 for attr in skc_sale_attr:125 if attr.get('isSize') == '1' or attr.get('is_size'):126 for attr_value in attr.get('attr_value_list', []):127 size_obj = {128 "attr_value_name": attr_value.get('attr_value_name', ''),129 "attr_value_name_en": attr_value.get('attr_value_name_en', ''),130 "is_sold_out": False,131 "attr_value_local_size": attr_value.get('attr_value_local_size', []),132 "size_chart": []133 }134 135 # Get size chart136 for desc_popup in attr_value.get('attrDescPopUp', []):137 for bind_data in desc_popup.get('bindAttrData', []):138 size_obj['size_chart'].append({139 "attr_name_value_cm": bind_data.get('attrNameValueCm', ''),140 "attr_name_value_inch": bind_data.get('attrNameValueInch', ''),141 "attr_name_value_key": bind_data.get('attrNameKey', ''),142 "attr_name_value": bind_data.get('attrNameValue', '')143 })144 145 sizes.append(size_obj)146 147 # Parse specs148 specs = []149 for detail in product_details:150 specs.append({151 "id": str(detail.get('attr_id', '')),152 "name": detail.get('attr_name', ''),153 "name_en": detail.get('attr_name_en', ''),154 "value": detail.get('attr_value', ''),155 "value_en": detail.get('attr_value_en', ''),156 "value_id": str(detail.get('attr_value_id', ''))157 })158 159 # Parse SKU list160 sku_list_data = multi_level.get('sku_list', [])161 sku_list = []162 for sku in sku_list_data:163 price_data = sku.get('price', {})164 sku_obj = {165 "retail_price": {166 "amount": float(price_data.get('retailPrice', {}).get('amount', 0)),167 "amount_with_symbol": price_data.get('retailPrice', {}).get('amountWithSymbol', ''),168 "usd_amount": float(price_data.get('retailPrice', {}).get('usdAmount', 0)),169 "usd_amount_with_symbol": price_data.get('retailPrice', {}).get('usdAmountWithSymbol', '')170 },171 "sale_price": {172 "amount": float(price_data.get('salePrice', {}).get('amount', 0)),173 "amount_with_symbol": price_data.get('salePrice', {}).get('amountWithSymbol', ''),174 "usd_amount": float(price_data.get('salePrice', {}).get('usdAmount', 0)),175 "usd_amount_with_symbol": price_data.get('salePrice', {}).get('usdAmountWithSymbol', '')176 },177 "sku_code": sku.get('sku_code', ''),178 "sku_sale_attr": [179 {180 "id": str(attr.get('attr_id', '')),181 "name": attr.get('attr_name', ''),182 "name_en": attr.get('attr_name_en', ''),183 "value_en": attr.get('attr_value_name_en', ''),184 "value_id": str(attr.get('attr_value_id', ''))185 }186 for attr in sku.get('sku_sale_attr', [])187 ]188 }189 sku_list.append(sku_obj)190 191 # Parse second_sale_attrs192 second_sale_attrs = []193 for attr in skc_sale_attr:194 attr_obj = {195 "id": str(attr.get('attr_id', '')),196 "name": attr.get('attr_name', ''),197 "name_en": attr.get('attr_name', ''),198 "values": [199 {200 "id": str(val.get('attr_value_id', '')),201 "value": val.get('attr_value_name', ''),202 "value_en": val.get('attr_value_name_en', ''),203 "is_sold_out": False204 }205 for val in attr.get('attr_value_list', [])206 ]207 }208 second_sale_attrs.append(attr_obj)209 210 # Build description211 description = "\n".join([f"{spec['name']}: {spec['value']}" for spec in specs]) + "\n"212 213 # Category info214 category_ids = str(product_info.get('cateIds', '')).split(',')215 category_name = product_info.get('cate_name', '')216 217 # Build variant218 variant = {219 "product_id": goods_id,220 "url": url,221 "title": product_info.get('goods_name', ''),222 "sku": goods_sn,223 "main_image": main_image,224 "discount_percentage": discount_percentage,225 "retail_price": retail_price,226 "sale_price": sale_price,227 "has_discount": has_discount,228 "main_sale_attr": {},229 "second_sale_attrs": second_sale_attrs,230 "color": color,231 "specs": specs,232 "images": images,233 "sizes": sizes,234 "sku_list": sku_list,235 "skc_sale_attr": [236 {237 "attr_id": str(attr.get('attr_id', '')),238 "attr_name": attr.get('attr_name', ''),239 "is_size": attr.get('isSize') == '1' or attr.get('is_size', False),240 "attr_value_list": [241 {242 "attr_value_id": str(val.get('attr_value_id', '')),243 "attr_value_name": val.get('attr_value_name', ''),244 "attr_value_name_en": val.get('attr_value_name_en', '')245 }246 for val in attr.get('attr_value_list', [])247 ]248 }249 for attr in skc_sale_attr250 ]251 }252 253 # Final output254 return {255 "main_image": main_image,256 "product_id": goods_id,257 "sku": goods_sn,258 "url": url,259 "title": product_info.get('goods_name', ''),260 "images": images,261 "color": color,262 "retail_price": retail_price,263 "sale_price": sale_price,264 "has_discount": has_discount,265 "discount_percentage": discount_percentage,266 "sizes": sizes,267 "specs": specs,268 "variants": [variant],269 "main_sale_attr": {},270 "second_sale_attrs": second_sale_attrs,271 "sku_list": sku_list,272 "skc_sale_attr": variant['skc_sale_attr'],273 "category_ids": category_ids,274 "category_name": category_name,275 "category_id": cat_id,276 "description": description277 }1"""2SHEIN product scraping logic with FULL captcha bypass (3 strategies)3"""4
5import time6import json7import re8import random9import os10from .browser import close_popup, close_browser, get_browser, load_cookies11from .parser import parse_shein_product12
13
14class CaptchaDetected(Exception):15 """Raised when captcha is detected"""16 pass17
18
19def check_captcha(driver):20 """Check if current page is a captcha page."""21 try:22 current_url = driver.current_url23 if "/risk/challenge" in current_url or "captcha_type=" in current_url:24 return True25 return False26 except:27 return False28
29
30def scrape_product(driver, url, timeout=60, include_images=True, include_reviews=True, skip_navigation=False):31 """32 Scrape a single SHEIN product page using same session/cookies.33 34 Args:35 driver: Selenium WebDriver (already has cookies loaded)36 url: Product URL37 timeout: Maximum time to wait for data38 include_images: Include image URLs39 include_reviews: Include review data40 skip_navigation: Skip navigation (already on page after captcha bypass)41 42 Returns:43 dict: Product data or None44 """45 46 # Navigate to URL (skip if already on page)47 if not skip_navigation:48 target_short = url.split('/')[-1][:40] if '/' in url else url[:40]49 print(f" Target: {target_short}...")50 try:51 driver.get(url)52 except Exception as e:53 if "timeout" in str(e).lower():54 raise Exception(f"Page load timeout")55 raise56 time.sleep(0.5) # Reduced from 1s57 58 # Show state59 state = _get_url_type(driver, url)60 print(f" State: {state}")61 62 # Check for captcha63 if check_captcha(driver):64 raise CaptchaDetected("Blocked")65 66 # Poll for data67 max_poll_time = timeout68 poll_interval = 0.569 elapsed = 070 js_data = None71 72 while elapsed < max_poll_time:73 # Check for captcha74 if check_captcha(driver):75 raise CaptchaDetected(f"Captcha detected at {url}")76 77 # Try to get data78 try:79 js_data = driver.execute_script("""80 if (window.gbRawData) return window.gbRawData;81 if (window.productIntroData) return window.productIntroData;82 return null;83 """)84 except:85 pass86 87 if js_data:88 break89 90 # Close popups periodically91 if elapsed > 0 and elapsed % 10 == 0:92 try:93 close_popup(driver)94 except:95 pass96 97 time.sleep(poll_interval)98 elapsed += poll_interval99 100 # Fallback to regex101 if not js_data:102 try:103 html = driver.page_source104 match = re.search(r'window\.gbRawData\s*=\s*({.*?});', html, re.DOTALL)105 if match:106 js_data = json.loads(match.group(1))107 except:108 pass109 110 if not js_data:111 return None112 113 # Parse the data114 result = parse_shein_product(js_data)115 116 # Optionally remove images/reviews117 if not include_images:118 result.pop('images', None)119 if not include_reviews:120 result.pop('rating', None)121 result.pop('review_count', None)122 123 return result124
125
126def _get_url_type(driver, target_url=None):127 """Get current URL type: TARGET, BLOCKED, or MAIN"""128 try:129 current = driver.current_url130 if "/risk/challenge" in current or "captcha_type=" in current:131 return "BLOCKED"132 elif target_url and target_url[:50] in current:133 return "TARGET"134 elif current.rstrip('/').endswith('.shein.com') or current.count('/') <= 3:135 return "MAIN"136 return "TARGET"137 except:138 return "UNKNOWN"139
140
141def handle_captcha(driver, url):142 """Attempt to bypass captcha. Returns True if successful, False if needs IP rotation."""143 # Detect domain from URL144 if 'ar.shein.com' in url.lower():145 initial_domain = "https://ar.shein.com"146 else:147 initial_domain = "https://il.shein.com"148 149 target_short = url.split('/')[-1][:40] if '/' in url else url[:40]150 print(f" Target: {target_short}...")151 print(f" State: {_get_url_type(driver, url)}")152 153 # Strategy 1: Return to main domain then target (fast - only 1 attempt)154 try:155 driver.get(initial_domain)156 time.sleep(0.3)157 try:158 close_popup(driver)159 except:160 pass161 162 if not check_captcha(driver):163 driver.get(url)164 time.sleep(0.3)165 state = _get_url_type(driver, url)166 print(f" State: {state}")167 if not check_captcha(driver):168 print(f" ✓ Bypassed (Strategy 1)")169 return True170 except Exception as e:171 print(f" ✗ Strategy 1 error: {str(e)[:30]}")172 173 # Strategy 1 failed - go straight to IP rotation (skip Strategy 2 to save time)174 print(f" State: BLOCKED")175 print(" ✗ Rotating IP...")176 return False1"""2Utility functions for SHEIN scraper3"""4
5import json6
7
8def load_cookies_from_string(cookies_json):9 """10 Parse cookies from JSON string11 12 Args:13 cookies_json: JSON string of cookies array14 15 Returns:16 list: List of cookie dictionaries17 """18 try:19 cookies = json.loads(cookies_json)20 if isinstance(cookies, list):21 return cookies22 except:23 pass24 return []25
26
27def setup_proxy(proxy_config):28 """29 Setup proxy configuration30 31 Args:32 proxy_config: Proxy configuration dict33 34 Returns:35 str: Proxy URL or None36 """37 if not proxy_config:38 return None39 40 # Apify proxy41 if proxy_config.get('useApifyProxy'):42 return "apify" # Will be handled by Apify SDK43 44 # Custom proxy45 proxy_urls = proxy_config.get('proxyUrls', [])46 if proxy_urls:47 return proxy_urls[0]48 49 return None50
51
52def clean_url(url):53 """54 Clean and validate SHEIN product URL55 56 Args:57 url: Raw URL string58 59 Returns:60 str: Cleaned URL or None if invalid61 """62 if not url:63 return None64 65 url = url.strip()66 67 # Validate it's a SHEIN URL68 if 'shein.com' not in url.lower():69 return None70 71 # Remove tracking parameters (optional)72 # This keeps the URL cleaner in output73 if '?' in url:74 base_url = url.split('?')[0]75 # Keep only the product ID parameter if present76 if '-p-' in base_url:77 return base_url + '.html' if not base_url.endswith('.html') else base_url78 79 return url80
81
82def format_price(price_str):83 """84 Format price string consistently85 86 Args:87 price_str: Raw price string88 89 Returns:90 str: Formatted price91 """92 if not price_str:93 return None94 95 # Remove extra whitespace96 price_str = ' '.join(price_str.split())97 98 return price_str99
100
101def calculate_discount(original, sale):102 """103 Calculate discount percentage104 105 Args:106 original: Original price (float or string)107 sale: Sale price (float or string)108 109 Returns:110 int: Discount percentage111 """112 try:113 # Extract numbers from price strings114 import re115 orig_match = re.search(r'[\d,.]+', str(original))116 sale_match = re.search(r'[\d,.]+', str(sale))117 118 if orig_match and sale_match:119 orig_val = float(orig_match.group().replace(',', ''))120 sale_val = float(sale_match.group().replace(',', ''))121 122 if orig_val > sale_val:123 discount = int(((orig_val - sale_val) / orig_val) * 100)124 return discount125 except:126 pass127 128 return 0