.actor

# Specify the base Docker image. You can read more about
# the available images at https://crawlee.dev/docs/guides/docker-images
# You can also use any other image from Docker Hub.
FROM apify/actor-node-puppeteer-chrome:16

# Copy just package.json and package-lock.json
# to speed up the build using Docker layer cache.
COPY --chown=myuser package*.json ./

# Install NPM packages, skip optional and development dependencies to
# keep the image small. Avoid logging too much and print the dependency
# tree for debugging
RUN npm --quiet set progress=false \
    && npm install --omit=dev --omit=optional \
    && echo "Installed NPM packages:" \
    && (npm list --omit=dev --all || true) \
    && echo "Node.js version:" \
    && node --version \
    && echo "NPM version:" \
    && npm --version \
    && rm -r ~/.npm

# Next, copy the remaining files and directories with the source code.
# Since we do this after NPM install, quick build will be really fast
# for most source file changes.
COPY --chown=myuser . ./


# Run the image. If you know you won't need headful browsers,
# you can remove the XVFB start script for a micro perf gain.
CMD ./start_xvfb_and_run_cmd.sh && npm start --silent


.actor/Dockerfile

Crawlee and Puppeteer project in JavaScript.

my-actor

Project Puppeteer Crawler JavaScript

.actor/actor.json

Country

Number of parallel threads crawling the products

Max concerency

Select proxies to be used by your crawler.

Proxy configuration

Enter Shopee product Urls

PlaywrightCrawler Template

.actor/input_schema.json

/**
 * This template is a production ready boilerplate for developing with `PuppeteerCrawler`.
 * Use this to bootstrap your projects using the most up-to-date code.
 * If you're looking for examples or want to learn more, see README.
 */



import { Actor } from 'apify';
import { PlaywrightCrawler,CheerioCrawler,RequestList } from 'crawlee';
import { BrowserPool, PlaywrightPlugin } from '@crawlee/browser-pool';
import FingerprintGenerator from 'fingerprint-generator';
import { FingerprintInjector } from 'fingerprint-injector';
import playwright from 'playwright';


async function getprs(input,page)
{
	let mcon = input.mcon===undefined?1:input.mcon;
	let urlar = input.starturls;
	if(urlar===undefined)
	{
		return 'No Shopee url to crawl';
	}
	else
	{		
		
		let proxyop = {};
		let proxy = input.proxy;
		if(proxy.useApifyProxy===true)
		{
			if(proxy.apifyProxyGroups!== undefined)
			{
				proxyop.groups = proxy.apifyProxyGroups;
			}
			if(proxy.apifyProxyCountry!== undefined)
			{
				proxyop.countryCode = proxy.apifyProxyCountry;
			}
		}
		else
		{
			if(proxy.proxyUrls!==undefined && proxy.proxyUrls.length>0)
			{
				proxyop.proxyUrls = proxy.proxyUrls;
			}
		}
		const proxyConfiguration = await Actor.createProxyConfiguration(proxyop);
		const requestList = await RequestList.open(null, urlar);
		let firesult = [];
		const crawler = new CheerioCrawler({
			proxyConfiguration,
			useSessionPool: false,
			persistCookiesPerSession: false, 
			requestList,
			minConcurrency: 1,
			maxConcurrency: mcon,
			maxRequestRetries: 3,
			requestHandlerTimeoutSecs: 30,
			maxRequestsPerCrawl: 0,
			preNavigationHooks: [
				async (crawlingContext, gotOptions) => {
					let request = crawlingContext.request;
					let url = request.url;
					let urlO;
					try
					{
						urlO = new URL(url);
					}
					catch(e)
					{
						return false;
						//do nothing
					}
					let hostname = urlO.hostname;	
					let referer = 'https://'+hostname+'/';
					if(request.userData.url===undefined)
					{
						request.userData.url = url;
						url = url.split('?')[0].split('#')[0];						
						let itemid = -1;
						let shopid = -1;											
						let host = hostname.split('.')[0];				
						if(host.toLowerCase()==='shopee')
						{
							let lastidx = url.lastIndexOf("-i.");
							if(lastidx>0)
							{
								let ids = url.substring(lastidx+3).split('.');
								if(ids.length===2)
								{
									shopid = ids[0].replace(/\D/g,'');;
									itemid = ids[1].replace(/\D/g,'');;
								}
							}
							else
							{
								lastidx = url.lastIndexOf("/product/");
								if(lastidx>0)
								{
									let ids = url.substring(lastidx+9).split('/');
									if(ids.length===2)
									{
										shopid = ids[0].replace(/\D/g,'');;
										itemid = ids[1].replace(/\D/g,'');;
									}
								}					
							}
							if(shopid!==-1&&itemid!==-1)
							{
								referer = 'https://'+hostname+'/product/'+shopid+'/'+itemid;
								request.url = 'https://'+hostname+'/api/v4/item/get?itemid='+itemid+'&shopid='+shopid;
							}
						}
						else
						{
							return false;
						}						
					}
					else
					{
						referer = request.userData.url;
					}
					let acstring = await page.evaluate(()  =>
					{	
						let acstring = window.ssdk00oQOOooO00QoQO(300,1);	
						return (acstring[0]);
					});
					request.headers = {		
							"user-agent":'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36',			
							"af-ac-enc-dat":acstring,							
							"accept": "application/json",
							"accept-language": "en-US,en;q=0.9",
							"content-type": "application/json",
							"sec-ch-ua": "\"Google Chrome\";v=\"112\", \"Not(A:Brand\";v=\"8\", \"Chromium\";v=\"112\"",
							"sec-ch-ua-mobile": "?0",
							"sec-ch-ua-platform": "\"Windows\"",
							"sec-fetch-dest": "empty",
							"sec-fetch-mode": "cors",
							"sec-fetch-site": "same-origin",
							"x-api-source": "pc",
							"x-requested-with": "XMLHttpRequest",
							"x-shopee-language": "vi",
							"cookie": "__LOCALE__null=VN; ",
							"Referrer-Policy": "strict-origin-when-cross-origin",
							'conection': 'close',
							// 'dnt': '1',
							'accept-encoding': 'gzip, deflate, br', 
							"Referer": referer,
						};	
				},
			],
			async requestHandler({ request, response, body, contentType, $,json })
			{
				if(response.statusCode===403|| (json!==null && json.action_type!==undefined))
				{ 
					request.userData.data = json;
					throw ({message:'Get data failed. Maybe because of IP being blocked.'});
				}
				else
				{
					try
					{
						let data1 = JSON.parse(body.toString());
						if(data1.data.item_has_video===undefined)
						{
							throw 'Error in data returned';
						}
						firesult.push({"Status":'Succeeded',Error:'No',Url:request.userData.url,Data:json});
					}
					catch(e)
					{
						firesult.push({"Status":'Failed',Error:'Seem like not Shopee Product Url',Url:request.userData.url,Data:body.toString()});
					}
				}
			},

			// This function is called if the page processing failed more than maxRequestRetries + 1 times.
			failedRequestHandler({ request,Error  })
			{
				firesult.push({"Status":'Failed',Error:Error,Url:request.userData.url,Data:request.userData.data});
				console.log(`Request ${request.url} failed.`);
			},	

		});
		await crawler.run();
		await Actor.pushData(firesult);
		return true;
	}
}
async function init1()
{
	await Actor.init();
	const input = await Actor.getInput();
	let basesp = 'https://shopee.vn/';
	switch(input.country)
	{
		case 'Singapore':
			basesp = 'https://shopee.sg/';
		break;
		
		case 'Indonesia':
			basesp = 'https://shopee.co.id/';
		break;
		
		case 'Taiwan':
			basesp = 'https://shopee.tw/';
		break;
		
		case 'Thailand':
			basesp = 'https://shopee.co.th/';
		break;
		
		case 'Malaysia':
			basesp = 'https://shopee.com.my/';
		break;
		
		case 'Vietnam':
			basesp = 'https://shopee.vn/';
		break;
		
		case 'Philippines':
			basesp = 'https://shopee.ph/';
		break;
		
		case 'Brazil':
			basesp = 'https://shopee.com.br/';
		break;
		
		case 'México':
			basesp = 'https://shopee.com.mx/';
		break;
		
		case 'Colombia':
			basesp = 'https://shopee.com.co/';
		break;
		
		case 'Chile':
			basesp = 'https://shopee.cl/';
		break;
	}
	const pluginOptions = {
		launchOptions: {
			headless: true,
			channel: 'chrome',
		},
	};

	const playwrightPlugin = new PlaywrightPlugin(playwright.chromium, pluginOptions);
	const fingerprintGenerator = new FingerprintGenerator({
		devices: ['desktop'],
		browsers: ["firefox","chrome","safari"],
		operatingSystems: ['windows','linux',],
	});

	const { fingerprint } = fingerprintGenerator.getFingerprint();
	const fingerprintInjector = new FingerprintInjector({ fingerprint });

	const browserPoolOptions = {
		useFingerprints:false,
		preLaunchHooks: [(pageId, launchContext) => {
			const { useIncognitoPages, launchOptions } = launchContext;

			if (useIncognitoPages) {
				return;
			}

			launchContext.launchOptions = {
				...launchOptions,
				userAgent: fingerprint.userAgent,
				viewport: {
					width: fingerprint.screen.width,
					height: fingerprint.screen.height,
				},

			};
		}],
		prePageCreateHooks: [
			(pageId, browserController, pageOptions) => {
				const { launchContext } = browserController;

				if (launchContext.useIncognitoPages && pageOptions) {
					pageOptions.userAgent = fingerprint.userAgent;
					pageOptions.viewport = {
						width: fingerprint.screen.width,
						height: fingerprint.screen.height,
					};
				}
			},
		],
		postPageCreateHooks: [
			async (page, browserController) => {
				const { browserPlugin, launchContext } = browserController;

				if (browserPlugin instanceof PlaywrightPlugin) {
					const { useIncognitoPages, isFingerprintInjected } = launchContext;

					if (isFingerprintInjected) {
						return;
					}
					//console.log('Injecting fingerprint to playwright');

					const context = page.context();
					await fingerprintInjector.attachFingerprintToPlaywright(context, fingerprint);

					if (!useIncognitoPages)
					{
						launchContext.extend({ isFingerprintInjected: true });
					}
				} else if (browserPlugin instanceof PuppeteerPlugin) {
					//console.log('Injecting fingerprint to puppeteer');
					await fingerprintInjector.attachFingerprintToPuppeteer(page, fingerprint);
				}
			},
		],
	};

	const crawler = new PlaywrightCrawler({browserPoolOptions,
		launchContext: {
			useChrome: false,
			useIncognitoPages:true,
			userDataDir:'./mycache',
			launcher:playwright.chromium,
			launchOptions: {
				headless: false,
				//ignoreDefaultArgs:true,
				//args:[],
			},
		},
	requestHandlerTimeoutSecs:5000,
		maxRequestsPerCrawl: 1,
		
		async requestHandler({ request, page, enqueueLinks }) {
			await page.waitForLoadState('networkidle',{timeout:60000});				
			await getprs(input,page);
		},
		failedRequestHandler({ request })
		{
			console.log(`Request failed too many times. Seem like low network.`);
		},
	});

	// Run the crawler and wait for it to finish.
	await crawler.run([basesp]);
	
	console.log('Mass Shopee Crawling Finished');

	await Actor.exit();
}

init1();

src/main.js

import { Dataset, createPuppeteerRouter } from 'crawlee';

export const router = createPuppeteerRouter();

router.addDefaultHandler(async ({ enqueueLinks, log }) => {
    log.info(`enqueueing new URLs`);
    await enqueueLinks({
        globs: ['https://apify.com/*'],
        label: 'detail',
    });
});

router.addHandler('detail', async ({ request, page, log }) => {
    const title = await page.title();
    log.info(`${title}`, { url: request.loadedUrl });

    await Dataset.pushData({
        url: request.loadedUrl,
        title,
    });
});


src/routes.js

# configurations
.idea

# crawlee and apify storage folders
apify_storage
crawlee_storage
storage

# installed files
node_modules

# git folder
.git


.dockerignore

root = true

[*]
indent_style = space
indent_size = 4
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf


.editorconfig

{
    "extends": "@apify",
    "root": true
}


.eslintrc

# This file tells Git which files shouldn't be added to source control

.DS_Store
.idea
dist
node_modules
apify_storage
storage


.gitignore

crawlee-puppeteer-javascript

package.json

package1.json

Mass Shopee products crawling. Enter URLs having the following formats:
https://<shopee-domain>/product/<shopid>/<itemid>
or
https://<shopee-domain>/<product-name>-i.<shopid>.<itemid>
Examples:
https://shopee.vn/product/159804633/4638129333
or
https://shopee.vn/kinh-mat-V-i.34266970.4949864081

mass-shopee-product-crawler

Mass Shopee Product Crawler

Mass Shopee Product Crawler · Apify

Input · Mass Shopee Product Crawler · Apify

API · Mass Shopee Product Crawler · Apify

Related Actors · Mass Shopee Product Crawler · Apify

Mass Shopee products crawling. Enter URLs having the following formats:
https://<shopee-domain>/product/<shopid>/<itemid>
or
https://<shopee-domain>/<product-name>-i.<shopid>.<itemid>
Examples:
htt...

Where next?