Pricing
Pay per usage
Pricing
Pay per usage
Rating
0.0
(0)
Developer
Purva Rajyaguru
Actor stats
0
Bookmarked
2
Total users
1
Monthly active users
4 days ago
Last modified
Categories
Share
{ "name": "klar-skincare-scraper", "version": "1.0.0", "type": "module", "description": "Scrapes skincare product details and translates to any language", "main": "src/main.js", "scripts": { "start": "node src/main.js", "test": "echo \"No tests yet\"" }, "dependencies": { "@crawlee/playwright": "^3.12.5", "apify": "^3.2.6", "lingo.dev": "^0.117.10", "playwright": "*" }, "author": "Klar", "devDependencies": { "apify-cli": "^1.1.1" }}# Use Apify's Playwright Chrome image (browsers pre-installed)FROM apify/actor-node-playwright-chrome:20
COPY package*.json ./RUN npm install --omit=dev --omit=optional
COPY . ./CMD ["npm", "start"]{ "actorSpecification": 1, "name": "klar-skincare-scraper", "title": "Klar Skincare Scraper", "description": "Searches Google for skincare products, scrapes detailed info, and translates to any language using Lingo.dev", "version": "1.0", "buildTag": "latest", "input": "./input_schema.json", "output": "./output_schema.json", "dockerfile": "./Dockerfile"}{ "title": "Klar Skincare Scraper Input", "type": "object", "schemaVersion": 1, "properties": { "searchQuery": { "title": "Search Query", "type": "string", "description": "Product name or search term to find on Google", "editor": "textfield", "prefill": "LANEIGE Water Sleeping Mask" }, "maxResults": { "title": "Max Results", "type": "integer", "description": "Maximum number of product pages to scrape (1 recommended for speed)", "default": 1, "minimum": 1, "maximum": 10 }, "targetLanguage": { "title": "Target Language", "type": "string", "description": "Language code to translate results to (e.g., en, hi, es, fr, de)", "default": "en", "editor": "select", "enum": ["en", "hi", "es", "fr", "de", "pt", "it", "ko", "ja", "zh", "ar", "ru", "th", "vi", "id", "tr", "nl", "pl", "bn", "ta", "te"], "enumTitles": ["English", "Hindi", "Spanish", "French", "German", "Portuguese", "Italian", "Korean", "Japanese", "Chinese", "Arabic", "Russian", "Thai", "Vietnamese", "Indonesian", "Turkish", "Dutch", "Polish", "Bengali", "Tamil", "Telugu"] }, "lingoApiKey": { "title": "Lingo.dev API Key", "type": "string", "description": "Your Lingo.dev API key for translation (required if targetLanguage is not 'en')", "editor": "textfield", "isSecret": true } }, "required": ["searchQuery"]}{ "actorOutputSchemaVersion": 1, "title": "Output schema of Klar Skincare Scraper", "description": "Contains scraped product information with translations", "properties": { "products": { "type": "string", "title": "Products", "description": "Scraped skincare product data stored in the default dataset", "template": "{{links.apiDefaultDatasetUrl}}/items" }, "summary": { "type": "string", "title": "Summary", "description": "Run summary with search query, target language, and total results count", "template": "{{links.apiDefaultKeyValueStoreUrl}}/records/OUTPUT" } }}1import { PlaywrightCrawler, Dataset, createPlaywrightRouter } from '@crawlee/playwright';2import { Actor } from 'apify';3
4await Actor.init();5
6const { searchQuery, maxResults = 1, targetLanguage = 'en', lingoApiKey } = (await Actor.getInput()) ?? {};7
8if (!searchQuery) throw new Error('searchQuery is required');9
10// Initialize Lingo.dev for translation if needed11let lingoDotDev = null;12if (targetLanguage !== 'en' && lingoApiKey) {13 const { LingoDotDevEngine } = await import('lingo.dev/sdk');14 lingoDotDev = new LingoDotDevEngine({ apiKey: lingoApiKey });15}16
17// Helper function to translate text18async function translateText(text, targetLang) {19 if (!text || targetLang === 'en' || !lingoDotDev) return text;20 21 try {22 const result = await lingoDotDev.localizeText(23 { text },24 { sourceLocale: 'en', targetLocale: targetLang }25 );26 return result.text || text;27 } catch (error) {28 console.error('Translation error:', error.message);29 return text;30 }31}32
33// Trusted domains that are fast and have good product data34const TRUSTED_DOMAINS = [35 'amazon.com', 'sephora.com', 'ulta.com', 'target.com', 'walmart.com',36 'dermstore.com', 'skinstore.com', 'cultbeauty.com', 'lookfantastic.com',37 'yesstyle.com', 'sokoglam.com', 'beautylish.com', 'nordstrom.com',38 'macys.com', 'bloomingdales.com', 'bergdorfgoodman.com', 'saksfifthavenue.com',39 'thefaceshop.com', 'innisfree.com', 'etudehouse.com', 'laneige.com',40 'cosrx.com', 'paula'41];42
43const productUrls = [];44const router = createPlaywrightRouter();45
46router.addHandler('GOOGLE', async ({ page, log }) => {47 log.info('Parsing Google search results...');48 49 await page.waitForSelector('div#search', { timeout: 10000 }).catch(() => {});50 51 const links = await page.$$eval('div#search a[href^="http"]', (anchors) => {52 const blocked = [53 'google.com', 'youtube.com', 'youtu.be', 'reddit.com', 'quora.com',54 'instagram.com', 'tiktok.com', 'facebook.com', 'twitter.com', 'x.com',55 'pinterest.com', 'linkedin.com', 'tumblr.com', 'snapchat.com',56 'wikipedia.org', 'wikihow.com', 'forum.', 'community.'57 ];58 return anchors59 .map(a => a.href)60 .filter(href => href && !blocked.some(domain => href.includes(domain)));61 });62 63 const unique = [...new Set(links)];64 log.info(`Found ${unique.length} URLs`);65 66 // Prioritize trusted domains67 const trusted = unique.filter(url => 68 TRUSTED_DOMAINS.some(domain => url.toLowerCase().includes(domain))69 );70 const others = unique.filter(url => 71 !TRUSTED_DOMAINS.some(domain => url.toLowerCase().includes(domain))72 );73 74 // Take trusted first, then others75 const prioritized = [...trusted, ...others];76 log.info(`Prioritized: ${trusted.length} trusted, ${others.length} others`);77 78 productUrls.push(...prioritized.slice(0, maxResults));79});80
81router.addHandler('PRODUCT', async ({ request, page, log }) => {82 log.info(`Scraping: ${request.url}`);83 84 await page.waitForLoadState('domcontentloaded');85 86 const title = await page.title();87 const description = await page.$eval('meta[name="description"]', el => el.content).catch(() => '');88 const bodyText = await page.$eval('body', el => el.innerText.toLowerCase()).catch(() => '');89 90 let ingredients = '';91 const ingredientEl = await page.$('[class*="ingredient"], [id*="ingredient"]').catch(() => null);92 if (ingredientEl) {93 ingredients = await ingredientEl.innerText().catch(() => '');94 ingredients = ingredients.substring(0, 800);95 }96 97 let howToUse = '';98 const howToUseSelectors = [99 '[class*="how-to-use"]', '[class*="howToUse"]', '[class*="usage"]',100 '[class*="directions"]', '[id*="how-to-use"]', '[id*="usage"]', '[id*="directions"]'101 ];102 for (const selector of howToUseSelectors) {103 const el = await page.$(selector).catch(() => null);104 if (el) {105 const text = await el.innerText().catch(() => '');106 if (text && text.length > 20 && text.length < 1000) {107 howToUse = text.substring(0, 500);108 break;109 }110 }111 }112 113 const skinTypes = ['oily', 'dry', 'combination', 'sensitive', 'normal', 'all skin types']114 .filter(k => bodyText.includes(k));115 116 const benefits = ['hydrating', 'moisturizing', 'brightening', 'anti-aging', 'soothing', 'calming', 'firming', 'nourishing', 'refreshing']117 .filter(k => bodyText.includes(k));118 119 const priceMatch = bodyText.match(/\$[\d,.]+/);120 const price = priceMatch ? priceMatch[0] : '';121 122 // Translate description and howToUse if target language is not English123 // Smart Hybrid: Keep title and ingredients in English124 let translatedDescription = description;125 let translatedHowToUse = howToUse;126 127 if (targetLanguage !== 'en' && lingoDotDev) {128 log.info(`Translating to ${targetLanguage}...`);129 [translatedDescription, translatedHowToUse] = await Promise.all([130 translateText(description, targetLanguage),131 translateText(howToUse, targetLanguage)132 ]);133 }134 135 await Dataset.pushData({136 url: request.url,137 title: title.substring(0, 200),138 description: translatedDescription.substring(0, 500),139 descriptionOriginal: description.substring(0, 500),140 ingredients,141 howToUse: translatedHowToUse,142 howToUseOriginal: howToUse,143 skinTypes: [...new Set(skinTypes)],144 benefits: [...new Set(benefits)],145 price,146 targetLanguage,147 scrapedAt: new Date().toISOString()148 });149 150 log.info(`Done: ${title.substring(0, 40)}...`);151});152
153const proxyConfiguration = await Actor.createProxyConfiguration().catch(() => null);154
155const crawler = new PlaywrightCrawler({156 proxyConfiguration,157 requestHandlerTimeoutSecs: 30,158 navigationTimeoutSecs: 15,159 maxRequestRetries: 1,160 requestHandler: router,161 headless: true,162});163
164const searchUrl = `https://www.google.com/search?q=${encodeURIComponent(searchQuery + ' skincare')}`;165await crawler.run([{ url: searchUrl, label: 'GOOGLE' }]);166
167if (productUrls.length > 0) {168 await crawler.run(productUrls.map(url => ({ url, label: 'PRODUCT' })));169}170
171const dataset = await Dataset.open();172const { items } = await dataset.getData();173
174await Actor.setValue('OUTPUT', {175 searchQuery,176 targetLanguage,177 totalResults: items.length,178 products: items179});180
181await Actor.exit();