Costco Product Scraper
Pricing
from $0.90 / 1,000 results
Scrape Costco product data from search URLs, category pages, and product URLs. Extract titles, prices, ratings, reviews, images, features, availability, and product URLs from Costco's search API.
Pricing
from $0.90 / 1,000 results
Rating
0.0
(0)
Developer
Crikit
Maintained by CommunityActor stats
0
Bookmarked
21
Total users
7
Monthly active users
12 days ago
Last modified
Categories
Share
node_modules/storage/dist/build/.DS_Store*.log{ "name": "costco-scraper", "version": "0.1.0", "type": "module", "description": "Apify actor that scrapes Costco product data from search, category, and product URLs.", "engines": { "node": ">=20" }, "dependencies": { "apify": "^3.4.2", "crawlee": "^3.13.0", "impit": "^0.14.0" }, "scripts": { "start": "node src/main.js", "validate-output": "node scripts/validate_output.js", "test:edge": "node scripts/test_edge_cases.js", "test:cloud": "node scripts/test_cloud_cases.js" }, "license": "MIT"}FROM apify/actor-node:22
COPY package*.json ./RUN npm ci --omit=dev
COPY . ./
CMD npm start{ "actorSpecification": 1, "name": "costco-product-scraper", "title": "Costco Product Scraper", "description": "Scrape Costco product data from search URLs, category pages, and product URLs. Extract titles, prices, ratings, reviews, images, features, availability, and product URLs from Costco's search API.", "version": "0.1", "buildTag": "latest", "meta": { "templateId": "js-api-wrapper", "generatedBy": "Codex with GPT-5" }, "input": "./input_schema.json", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }, "dockerfile": "./Dockerfile", "categories": ["ECOMMERCE"], "defaultRunOptions": { "build": "latest", "memoryMbytes": 1024, "timeoutSecs": 3600 }}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "productId": { "type": [ "string", "null" ] }, "sku": { "type": [ "string", "null" ] }, "productUrl": { "type": "string" }, "sourceUrl": { "type": [ "string", "null" ] }, "name": { "type": "string" }, "brand": { "type": [ "string", "null" ] }, "brands": { "type": "array", "items": { "type": "string" } }, "price": { "type": [ "number", "null" ] }, "listPrice": { "type": [ "number", "null" ] }, "minPrice": { "type": [ "number", "null" ] }, "maxPrice": { "type": [ "number", "null" ] }, "currency": { "type": [ "string", "null" ] }, "priceText": { "type": [ "string", "null" ] }, "unitPrice": { "type": [ "number", "string", "null" ] }, "rating": { "type": [ "number", "null" ] }, "ratingValues": { "type": "array", "items": { "type": "string" } }, "reviewCount": { "type": [ "integer", "null" ] }, "availability": { "type": [ "string", "null" ] }, "memberOnly": { "type": "boolean" }, "description": { "type": [ "string", "null" ] }, "features": { "type": "array", "items": { "type": "string" } }, "specifications": { "type": "object" }, "breadcrumbs": { "type": "array", "items": { "type": "string" } }, "mainImage": { "type": [ "string", "null" ] }, "images": { "type": "array", "items": { "type": "string" } }, "category": { "type": [ "string", "null" ] }, "categoryUrl": { "type": [ "string", "null" ] }, "id": { "type": [ "string", "null" ] }, "groupId": { "type": [ "string", "null" ] }, "itemNumber": { "type": [ "string", "null" ] }, "itemName": { "type": [ "string", "null" ] }, "manufacturingSkus": { "type": "array", "items": { "type": "string" } }, "onlineOnlyTags": { "type": "array", "items": { "type": "string" } }, "image": { "type": [ "string", "null" ] }, "currencyCode": { "type": [ "string", "null" ] }, "pricePerUnit": { "type": [ "number", "string", "null" ] }, "reviewsCount": { "type": [ "integer", "null" ] }, "isBuyable": { "type": [ "boolean", "null" ] }, "isPublished": { "type": "boolean" }, "isInStock": { "type": [ "boolean", "null" ] }, "availabilityStatus": { "type": [ "string", "null" ] }, "stockStatus": { "type": [ "string", "null" ] }, "deliveryStatus": { "type": [ "string", "null" ] }, "minQuantityRestriction": { "type": [ "integer", "null" ] }, "maxQuantityRestriction": { "type": [ "integer", "null" ] }, "startDate": { "type": [ "string", "null" ] }, "marketingFeatures": { "type": "array", "items": { "type": "string" } }, "marketingKeywords": { "type": "array", "items": { "type": "string" } }, "categoryPaths": { "type": "array", "items": { "type": "string" } }, "itemClassification": { "type": [ "string", "null" ] }, "as400Category": { "type": [ "string", "null" ] }, "isMemberOnly": { "type": "boolean" }, "isSingleSku": { "type": [ "boolean", "null" ] }, "scrapedAt": { "type": "string" }, "recordType": { "type": [ "string", "null" ], "title": "recordType", "description": "'diagnostic' for a row explaining why a run produced no products; absent on normal product rows." }, "diagnosticReason": { "type": [ "string", "null" ], "title": "diagnosticReason", "description": "Machine-readable cause on diagnostic rows: no_input or no_results." }, "message": { "type": [ "string", "null" ], "title": "message", "description": "Human-readable explanation on diagnostic rows." } }, "required": [ "productUrl", "name", "scrapedAt" ] }, "views": { "overview": { "title": "Overview", "transformation": { "fields": [ "productId", "name", "brand", "price", "currency", "rating", "reviewCount", "availability", "memberOnly", "mainImage", "productUrl" ] }, "display": { "component": "table", "properties": { "mainImage": { "label": "Image", "format": "image" }, "productUrl": { "label": "URL", "format": "link" } } } } }}{ "title": "Costco Product Scraper input", "description": "Paste Costco search URLs, category URLs, product URLs, or keywords. The scraper reads Costco's search API and extracts structured product data. Use maxItems to control run cost.", "type": "object", "schemaVersion": 1, "properties": { "startUrls": { "title": "Costco URLs", "description": "Costco search, category, or product URLs. Product URLs are resolved through Costco search, while search and category URLs are read through Costco's product API.", "type": "array", "editor": "requestListSources", "prefill": [ { "url": "https://www.costco.com/CatalogSearch?dept=All&keyword=tshirt" } ], "default": [ { "url": "https://www.costco.com/CatalogSearch?dept=All&keyword=tshirt" } ], "minItems": 1 }, "searchUrls": { "title": "Search URLs", "description": "Optional simple list of Costco search or category URLs. This is compatible with common Costco scraper inputs.", "type": "array", "editor": "stringList", "default": [] }, "keywords": { "title": "Keywords", "description": "Optional Costco search keywords. Each keyword becomes https://www.costco.com/s?keyword=<keyword>. Use concrete product terms such as tshirt, laptop, coffee, or vitamins.", "type": "array", "editor": "stringList", "default": [] }, "maxItems": { "title": "Max products", "description": "Hard cap on product records written to the dataset.", "type": "integer", "default": 50, "prefill": 25, "minimum": 1, "maximum": 10000 }, "maxItemsPerStartUrl": { "title": "Max products per start URL", "description": "Hard cap on product records discovered from each search or category start URL.", "type": "integer", "default": 50, "prefill": 25, "minimum": 1, "maximum": 10000 }, "maxSearchPagesPerStartUrl": { "title": "Max search pages per start URL", "description": "Maximum number of paginated Costco search/category API pages to request for each start URL.", "type": "integer", "default": 3, "prefill": 2, "minimum": 1, "maximum": 50 }, "scrapeProductDetails": { "title": "Scrape product detail pages", "description": "Compatibility option. Costco API results already include the core product details, so this does not require browser navigation.", "type": "boolean", "default": true }, "proxyConfiguration": { "title": "Proxy configuration", "description": "Costco blocks many data center IPs. The default uses Apify Residential routed through the US.", "type": "object", "editor": "proxy", "prefill": { "useApifyProxy": true, "apifyProxyGroups": [ "RESIDENTIAL" ], "apifyProxyCountry": "US" }, "default": { "useApifyProxy": true, "apifyProxyGroups": [ "RESIDENTIAL" ], "apifyProxyCountry": "US" } }, "maxConcurrency": { "title": "Max concurrency", "description": "Compatibility option retained for common scraper inputs. The API scraper runs paginated requests sequentially.", "type": "integer", "default": 2, "minimum": 1, "maximum": 10 }, "debug": { "title": "Debug logging", "description": "Enable verbose logs and save blocked-page evidence to the default key-value store.", "type": "boolean", "default": false } }, "required": [ "startUrls" ]}{ "actorOutputSchemaVersion": 1, "title": "Costco product data", "description": "Links to the dataset containing scraped Costco product records.", "properties": { "dataset": { "type": "string", "title": "Results dataset", "description": "Dataset containing all scraped Costco product records.", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1import fs from 'node:fs';2import path from 'node:path';3
4const actorId = process.env.APIFY_ACTOR_ID || 'Ep4B2PTXlOihgpeTO';5const token = process.env.APIFY_TOKEN;6const reportPath = path.resolve('build_results/costco-scraper-build-results/cloud-edge-test-report.json');7const defaultProxyConfiguration = {8 useApifyProxy: true,9 apifyProxyGroups: ['RESIDENTIAL'],10 apifyProxyCountry: 'US',11};12
13const testCases = [14 {15 name: 'cloud search returns capped products',16 input: {17 startUrls: [{ url: 'https://www.costco.com/CatalogSearch?dept=All&keyword=tshirt' }],18 maxItems: 3,19 maxItemsPerStartUrl: 3,20 maxSearchPagesPerStartUrl: 1,21 },22 expect: {23 status: 'SUCCEEDED',24 countExact: 3,25 everySourceUrl: 'https://www.costco.com/s?keyword=tshirt',26 },27 },28 {29 name: 'cloud product URL returns one product',30 input: {31 startUrls: [{ url: 'https://www.costco.com/kirkland-signature-mens-pima-tee.product.4000385619.html' }],32 maxItems: 3,33 maxItemsPerStartUrl: 3,34 maxSearchPagesPerStartUrl: 1,35 },36 expect: {37 status: 'SUCCEEDED',38 countExact: 1,39 includesGroupId: '4000385619',40 },41 },42 {43 name: 'cloud no-result category returns empty dataset',44 input: {45 startUrls: [{ url: 'https://www.costco.com/definitely-not-a-real-category-987654321.html' }],46 maxItems: 5,47 maxItemsPerStartUrl: 5,48 maxSearchPagesPerStartUrl: 1,49 },50 expect: {51 status: 'SUCCEEDED',52 countExact: 0,53 },54 },55 {56 name: 'cloud external URL fails',57 input: {58 startUrls: [{ url: 'https://example.com/not-costco' }],59 maxItems: 3,60 },61 expect: {62 status: 'FAILED',63 errorMessageIncludes: 'Input must contain at least one Costco',64 },65 },66];67
68const api = async (url, options = {}) => {69 const response = await fetch(`https://api.apify.com${url}`, {70 ...options,71 headers: {72 authorization: `Bearer ${token}`,73 'content-type': 'application/json',74 ...(options.headers || {}),75 },76 });77 const payload = await response.json().catch(() => null);78 if (!response.ok) {79 throw new Error(`Apify API ${response.status}: ${JSON.stringify(payload)}`);80 }81 return payload;82};83
84const runActor = async (input) => {85 const body = JSON.stringify({86 ...input,87 proxyConfiguration: input.proxyConfiguration || defaultProxyConfiguration,88 });89 const started = await api(`/v2/acts/${actorId}/runs?waitForFinish=120&memory=1024`, {90 method: 'POST',91 body,92 });93 let run = started.data;94 while (!['SUCCEEDED', 'FAILED', 'ABORTED', 'TIMED-OUT'].includes(run.status)) {95 await new Promise((resolve) => setTimeout(resolve, 5000));96 const polled = await api(`/v2/actor-runs/${run.id}?waitForFinish=60`);97 run = polled.data;98 }99 return run;100};101
102const fetchDatasetItems = async (datasetId) => {103 if (!datasetId) return [];104 const response = await fetch(`https://api.apify.com/v2/datasets/${datasetId}/items?clean=true`, {105 headers: { authorization: `Bearer ${token}` },106 });107 if (!response.ok) {108 throw new Error(`Dataset API ${response.status}: ${await response.text()}`);109 }110 return response.json();111};112
113const fetchRunLog = async (runId) => {114 const response = await fetch(`https://api.apify.com/v2/logs/${runId}`, {115 headers: { authorization: `Bearer ${token}` },116 });117 if (!response.ok) return '';118 return response.text();119};120
121const validateRecord = (item, index) => {122 const itemNumber = index + 1;123 const failures = [];124 for (const field of ['productUrl', 'name', 'id', 'groupId', 'itemNumber', 'scrapedAt', 'sourceUrl']) {125 if (typeof item[field] !== 'string' || item[field].trim() === '') {126 failures.push(`Item ${itemNumber} missing string field ${field}`);127 }128 }129 if (item.productUrl && !/^https:\/\/www\.costco\.com\/.+\.product\.[0-9]+\.html$/.test(item.productUrl)) {130 failures.push(`Item ${itemNumber} has unexpected productUrl ${item.productUrl}`);131 }132 for (const field of ['brands', 'images', 'features', 'marketingFeatures', 'marketingKeywords', 'onlineOnlyTags', 'categoryPaths']) {133 if (!Array.isArray(item[field])) failures.push(`Item ${itemNumber} field ${field} is not an array`);134 }135 return failures;136};137
138const validateItems = (items, testCase) => {139 const failures = [];140 const expected = testCase.expect;141 if (expected.countExact != null && items.length !== expected.countExact) {142 failures.push(`Expected exactly ${expected.countExact} items, got ${items.length}`);143 }144 if (expected.everySourceUrl && items.some((item) => item.sourceUrl !== expected.everySourceUrl)) {145 failures.push(`Expected every sourceUrl to be ${expected.everySourceUrl}`);146 }147 if (expected.includesGroupId && !items.some((item) => String(item.groupId) === expected.includesGroupId)) {148 failures.push(`Expected a record with groupId ${expected.includesGroupId}`);149 }150
151 const productUrls = new Set();152 for (const item of items) {153 if (productUrls.has(item.productUrl)) failures.push(`Duplicate productUrl ${item.productUrl}`);154 productUrls.add(item.productUrl);155 }156 items.forEach((item, index) => failures.push(...validateRecord(item, index)));157 return failures;158};159
160if (!token) {161 console.error('APIFY_TOKEN is required to run deployed cloud tests.');162 process.exit(1);163}164
165const startedAt = new Date().toISOString();166const results = [];167
168for (const testCase of testCases) {169 process.stdout.write(`Running cloud case: ${testCase.name} ... `);170 const run = await runActor(testCase.input);171 const failures = [];172 let items = [];173 let logTail = '';174
175 if (run.status !== testCase.expect.status) {176 failures.push(`Expected status ${testCase.expect.status}, got ${run.status}`);177 }178
179 if (run.status === 'SUCCEEDED') {180 items = await fetchDatasetItems(run.defaultDatasetId);181 failures.push(...validateItems(items, testCase));182 } else if (testCase.expect.errorMessageIncludes) {183 logTail = (await fetchRunLog(run.id)).slice(-4000);184 if (!logTail.includes(testCase.expect.errorMessageIncludes)) {185 failures.push(`Expected log to include "${testCase.expect.errorMessageIncludes}"`);186 }187 }188
189 const result = {190 name: testCase.name,191 passed: failures.length === 0,192 runId: run.id,193 datasetId: run.defaultDatasetId,194 status: run.status,195 usageTotalUsd: run.usageTotalUsd,196 memoryMbytes: run.options?.memoryMbytes,197 itemCount: items.length,198 failures,199 logTail,200 };201 results.push(result);202 process.stdout.write(result.passed ? 'PASS\n' : 'FAIL\n');203 if (!result.passed) {204 for (const failure of failures) process.stdout.write(` - ${failure}\n`);205 }206}207
208const report = {209 actorId,210 startedAt,211 finishedAt: new Date().toISOString(),212 total: results.length,213 passed: results.filter((result) => result.passed).length,214 failed: results.filter((result) => !result.passed).length,215 results,216};217
218fs.mkdirSync(path.dirname(reportPath), { recursive: true });219fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);220
221console.log(JSON.stringify({222 total: report.total,223 passed: report.passed,224 failed: report.failed,225 reportPath,226}, null, 2));227
228if (report.failed > 0) {229 process.exitCode = 1;230}1import { spawnSync } from 'node:child_process';2import fs from 'node:fs';3import path from 'node:path';4
5const datasetDir = path.resolve('storage/datasets/default');6const reportPath = path.resolve('build_results/costco-scraper-build-results/edge-test-report.json');7const defaultProxyConfiguration = {8 useApifyProxy: true,9 apifyProxyGroups: ['RESIDENTIAL'],10 apifyProxyCountry: 'US',11};12
13const testCases = [14 {15 name: 'legacy CatalogSearch URL normalizes to Costco search',16 input: {17 startUrls: [{ url: 'https://www.costco.com/CatalogSearch?dept=All&keyword=tshirt' }],18 maxItems: 3,19 maxItemsPerStartUrl: 3,20 maxSearchPagesPerStartUrl: 1,21 },22 expect: {23 countExact: 3,24 everySourceUrl: 'https://www.costco.com/s?keyword=tshirt',25 },26 },27 {28 name: 'keyword-only search returns products',29 input: {30 startUrls: [],31 keywords: ['vitamins'],32 maxItems: 3,33 maxItemsPerStartUrl: 3,34 maxSearchPagesPerStartUrl: 1,35 },36 expect: {37 countExact: 3,38 everySourceUrl: 'https://www.costco.com/s?keyword=vitamins',39 },40 },41 {42 name: 'searchUrls input returns products',43 input: {44 startUrls: [],45 searchUrls: ['https://www.costco.com/s?keyword=coffee'],46 maxItems: 3,47 maxItemsPerStartUrl: 3,48 maxSearchPagesPerStartUrl: 1,49 },50 expect: {51 countExact: 3,52 everySourceUrl: 'https://www.costco.com/s?keyword=coffee',53 },54 },55 {56 name: 'category URL returns category records',57 input: {58 startUrls: [{ url: 'https://www.costco.com/mens-clothing.html' }],59 maxItems: 4,60 maxItemsPerStartUrl: 4,61 maxSearchPagesPerStartUrl: 1,62 },63 expect: {64 countExact: 4,65 everySourceUrl: 'https://www.costco.com/mens-clothing.html',66 },67 },68 {69 name: 'product URL returns only the requested product group',70 input: {71 startUrls: [{ url: 'https://www.costco.com/kirkland-signature-mens-pima-tee.product.4000385619.html' }],72 maxItems: 3,73 maxItemsPerStartUrl: 3,74 maxSearchPagesPerStartUrl: 1,75 },76 expect: {77 countExact: 1,78 includesGroupId: '4000385619',79 },80 },81 {82 name: 'duplicate source inputs do not create duplicate output records',83 input: {84 startUrls: [85 { url: 'https://www.costco.com/CatalogSearch?dept=All&keyword=tshirt' },86 { url: 'https://www.costco.com/s?keyword=tshirt' },87 ],88 keywords: ['tshirt'],89 maxItems: 5,90 maxItemsPerStartUrl: 5,91 maxSearchPagesPerStartUrl: 1,92 },93 expect: {94 countExact: 5,95 everySourceUrl: 'https://www.costco.com/s?keyword=tshirt',96 },97 },98 {99 name: 'global maxItems cap stops mixed sources',100 input: {101 startUrls: [{ url: 'https://www.costco.com/CatalogSearch?dept=All&keyword=tshirt' }],102 keywords: ['coffee'],103 maxItems: 2,104 maxItemsPerStartUrl: 10,105 maxSearchPagesPerStartUrl: 1,106 },107 expect: {108 countExact: 2,109 },110 },111 {112 name: 'maxItemsPerStartUrl caps each source independently',113 input: {114 startUrls: [],115 keywords: ['tshirt', 'coffee'],116 maxItems: 10,117 maxItemsPerStartUrl: 1,118 maxSearchPagesPerStartUrl: 1,119 },120 expect: {121 countExact: 2,122 sourceCountsAtMost: 1,123 },124 },125 {126 name: 'pagination can collect beyond the first search page',127 input: {128 startUrls: [],129 keywords: ['tshirt'],130 maxItems: 30,131 maxItemsPerStartUrl: 30,132 maxSearchPagesPerStartUrl: 2,133 },134 expect: {135 countExact: 30,136 everySourceUrl: 'https://www.costco.com/s?keyword=tshirt',137 },138 },139 {140 name: 'no-result category exits cleanly with empty dataset',141 input: {142 startUrls: [{ url: 'https://www.costco.com/definitely-not-a-real-category-987654321.html' }],143 maxItems: 5,144 maxItemsPerStartUrl: 5,145 maxSearchPagesPerStartUrl: 1,146 },147 expect: {148 countExact: 0,149 },150 },151 {152 name: 'external URL input fails clearly',153 input: {154 startUrls: [{ url: 'https://example.com/not-costco' }],155 maxItems: 3,156 },157 expectFailure: 'Input must contain at least one Costco',158 },159 {160 name: 'malformed URL input fails clearly',161 input: {162 startUrls: [{ url: 'not a url' }],163 maxItems: 3,164 },165 expectFailure: 'do not contain valid URLs',166 },167];168
169const stripAnsi = (value) => String(value || '').replace(/\u001b\[[0-9;]*m/g, '');170const tail = (value, maxLength = 4000) => {171 const cleaned = stripAnsi(value);172 return cleaned.length > maxLength ? cleaned.slice(cleaned.length - maxLength) : cleaned;173};174
175const readDataset = () => {176 if (!fs.existsSync(datasetDir)) return [];177 return fs.readdirSync(datasetDir)178 .filter((file) => file.endsWith('.json'))179 .sort()180 .map((file) => JSON.parse(fs.readFileSync(path.join(datasetDir, file), 'utf8')));181};182
183const validateRecord = (item, index) => {184 const itemNumber = index + 1;185 const failures = [];186 const requiredStringFields = ['productUrl', 'name', 'id', 'groupId', 'itemNumber', 'scrapedAt', 'sourceUrl'];187 for (const field of requiredStringFields) {188 if (typeof item[field] !== 'string' || item[field].trim() === '') {189 failures.push(`Item ${itemNumber} missing string field ${field}`);190 }191 }192 if (item.productUrl && !/^https:\/\/www\.costco\.com\/.+\.product\.[0-9]+\.html$/.test(item.productUrl)) {193 failures.push(`Item ${itemNumber} has unexpected productUrl ${item.productUrl}`);194 }195 if (item.sourceUrl && !/^https:\/\/www\.costco\.com\//.test(item.sourceUrl)) {196 failures.push(`Item ${itemNumber} has unexpected sourceUrl ${item.sourceUrl}`);197 }198 if (Number.isNaN(Date.parse(item.scrapedAt))) {199 failures.push(`Item ${itemNumber} has invalid scrapedAt ${item.scrapedAt}`);200 }201 for (const field of ['brands', 'images', 'features', 'marketingFeatures', 'marketingKeywords', 'onlineOnlyTags', 'categoryPaths']) {202 if (!Array.isArray(item[field])) failures.push(`Item ${itemNumber} field ${field} is not an array`);203 }204 for (const field of ['price', 'listPrice', 'minPrice', 'maxPrice', 'unitPrice', 'pricePerUnit', 'rating']) {205 if (item[field] != null && typeof item[field] !== 'number') {206 failures.push(`Item ${itemNumber} field ${field} is not a number or null`);207 }208 }209 for (const field of ['memberOnly', 'isPublished', 'isMemberOnly']) {210 if (typeof item[field] !== 'boolean') failures.push(`Item ${itemNumber} field ${field} is not boolean`);211 }212 return failures;213};214
215const validateItems = (items, testCase) => {216 const failures = [];217 const expected = testCase.expect || {};218 if (expected.countExact != null && items.length !== expected.countExact) {219 failures.push(`Expected exactly ${expected.countExact} items, got ${items.length}`);220 }221 if (expected.countMin != null && items.length < expected.countMin) {222 failures.push(`Expected at least ${expected.countMin} items, got ${items.length}`);223 }224 if (expected.countMax != null && items.length > expected.countMax) {225 failures.push(`Expected at most ${expected.countMax} items, got ${items.length}`);226 }227 if (testCase.input.maxItems != null && items.length > testCase.input.maxItems) {228 failures.push(`Returned ${items.length} items, above maxItems ${testCase.input.maxItems}`);229 }230
231 const productUrlCounts = new Map();232 for (const item of items) {233 productUrlCounts.set(item.productUrl, (productUrlCounts.get(item.productUrl) || 0) + 1);234 }235 for (const [url, count] of productUrlCounts) {236 if (url && count > 1) failures.push(`Duplicate productUrl ${url}`);237 }238
239 items.forEach((item, index) => {240 failures.push(...validateRecord(item, index));241 });242
243 if (expected.everySourceUrl) {244 const wrong = items.filter((item) => item.sourceUrl !== expected.everySourceUrl);245 if (wrong.length > 0) {246 failures.push(`Expected every sourceUrl to be ${expected.everySourceUrl}, got ${[...new Set(wrong.map((item) => item.sourceUrl))].join(', ')}`);247 }248 }249 if (expected.includesGroupId && !items.some((item) => String(item.groupId) === expected.includesGroupId)) {250 failures.push(`Expected a record with groupId ${expected.includesGroupId}`);251 }252 if (expected.sourceCountsAtMost != null) {253 const sourceCounts = new Map();254 for (const item of items) {255 sourceCounts.set(item.sourceUrl, (sourceCounts.get(item.sourceUrl) || 0) + 1);256 }257 for (const [sourceUrl, count] of sourceCounts) {258 if (count > expected.sourceCountsAtMost) {259 failures.push(`Source ${sourceUrl} returned ${count} items, above cap ${expected.sourceCountsAtMost}`);260 }261 }262 }263 return failures;264};265
266const runActor = (input) => {267 const result = spawnSync('apify', ['run', '--purge', '--input', JSON.stringify({268 ...input,269 proxyConfiguration: input.proxyConfiguration || defaultProxyConfiguration,270 })], {271 cwd: path.resolve('.'),272 env: {273 ...process.env,274 APIFY_TOKEN: process.env.APIFY_TOKEN,275 },276 encoding: 'utf8',277 maxBuffer: 20 * 1024 * 1024,278 });279 return {280 status: result.status,281 signal: result.signal,282 stdout: tail(result.stdout),283 stderr: tail(result.stderr),284 };285};286
287if (!process.env.APIFY_TOKEN) {288 console.error('APIFY_TOKEN is required so local runs can use Apify proxy.');289 process.exit(1);290}291
292const startedAt = new Date().toISOString();293const results = [];294
295for (const testCase of testCases) {296 process.stdout.write(`Running: ${testCase.name} ... `);297 const run = runActor(testCase.input);298 const combinedOutput = `${run.stdout}\n${run.stderr}`;299 const failures = [];300 let itemCount = 0;301 let fields = [];302
303 if (testCase.expectFailure) {304 if (!combinedOutput.includes(testCase.expectFailure)) {305 failures.push(`Expected failure output to include "${testCase.expectFailure}"`);306 }307 if (run.status === 0 && !/\bERROR\b|Error:/.test(combinedOutput)) {308 failures.push('Expected actor or Apify CLI to report a failure');309 }310 } else {311 if (run.status !== 0) {312 failures.push(`Actor exited with status ${run.status}`);313 } else {314 const items = readDataset();315 itemCount = items.length;316 fields = items[0] ? Object.keys(items[0]).sort() : [];317 failures.push(...validateItems(items, testCase));318 }319 }320
321 const result = {322 name: testCase.name,323 passed: failures.length === 0,324 itemCount,325 fields,326 failures,327 exitStatus: run.status,328 stdoutTail: run.stdout,329 stderrTail: run.stderr,330 };331 results.push(result);332 process.stdout.write(result.passed ? 'PASS\n' : 'FAIL\n');333 if (!result.passed) {334 for (const failure of failures) process.stdout.write(` - ${failure}\n`);335 }336}337
338const report = {339 startedAt,340 finishedAt: new Date().toISOString(),341 total: results.length,342 passed: results.filter((result) => result.passed).length,343 failed: results.filter((result) => !result.passed).length,344 results,345};346
347fs.mkdirSync(path.dirname(reportPath), { recursive: true });348fs.writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);349
350console.log(JSON.stringify({351 total: report.total,352 passed: report.passed,353 failed: report.failed,354 reportPath,355}, null, 2));356
357if (report.failed > 0) {358 process.exitCode = 1;359}1import fs from 'node:fs';2import path from 'node:path';3
4const datasetDir = path.resolve('storage/datasets/default');5if (!fs.existsSync(datasetDir)) {6 throw new Error(`Dataset directory not found: ${datasetDir}`);7}8
9const files = fs.readdirSync(datasetDir).filter((file) => file.endsWith('.json')).sort();10const items = files.map((file) => JSON.parse(fs.readFileSync(path.join(datasetDir, file), 'utf8')));11
12const required = ['productUrl', 'name', 'scrapedAt'];13const failures = [];14items.forEach((item, index) => {15 for (const field of required) {16 if (!item[field]) failures.push(`Item ${index + 1} missing ${field}`);17 }18 if (item.productUrl && !/^https:\/\/www\.costco\.com\//.test(item.productUrl)) {19 failures.push(`Item ${index + 1} has non-Costco URL: ${item.productUrl}`);20 }21});22
23console.log(JSON.stringify({24 itemCount: items.length,25 fields: items[0] ? Object.keys(items[0]).sort() : [],26 failures,27}, null, 2));28
29if (items.length === 0 || failures.length > 0) {30 process.exitCode = 1;31}1import {2 cleanText,3 compactArray,4 normalizeCostcoUrl,5 parseInteger,6 parsePrice,7 productIdFromUrl,8} from './helpers.js';9
10const PRODUCT_LINK_RE = /\.product\.\d+\.html/i;11
12const addCompetitorCompatibleFields = (record) => {13 const availabilityText = cleanText(record.availability);14 const isInStock = availabilityText == null ? null : !/out|unavailable|sold out/i.test(availabilityText);15 const marketingKeywords = compactArray([16 ...(record.name || '').split(/[^a-z0-9]+/i),17 ...(record.brand || '').split(/[^a-z0-9]+/i),18 ...(record.features || []).flatMap((feature) => feature.split(/[^a-z0-9]+/i)),19 ])20 .map((word) => word.toLowerCase())21 .filter((word) => word.length >= 3 && word.length <= 40)22 .slice(0, 80);23
24 return {25 ...record,26 categoryUrl: record.sourceUrl || null,27 id: record.sku ? `${record.sku}!item.en-US` : null,28 groupId: record.productId || null,29 itemNumber: record.sku || record.productId || null,30 itemName: record.name,31 image: record.mainImage || null,32 currencyCode: record.currency || null,33 pricePerUnit: record.unitPrice || record.price || null,34 reviewsCount: record.reviewCount ?? null,35 isBuyable: isInStock,36 isPublished: true,37 isInStock,38 availabilityStatus: availabilityText,39 stockStatus: availabilityText,40 deliveryStatus: availabilityText,41 minQuantityRestriction: null,42 maxQuantityRestriction: null,43 startDate: null,44 marketingFeatures: record.features || [],45 marketingKeywords,46 categoryPaths: record.breadcrumbs || [],47 itemClassification: null,48 as400Category: null,49 isMemberOnly: record.memberOnly,50 isSingleSku: null,51 };52};53
54export const isBlockedPage = async (page) => {55 const title = cleanText(await page.title().catch(() => '')) || '';56 const body = cleanText(await page.locator('body').innerText({ timeout: 5000 }).catch(() => '')) || '';57 const html = await page.content().catch(() => '');58 const combined = `${title} ${body} ${html.slice(0, 5000)}`.toLowerCase();59 return combined.includes('access denied')60 || combined.includes('errors.edgesuite.net')61 || combined.includes('powered and protected by')62 || combined.includes('sec-if-cpt-container')63 || combined.includes('pardon our interruption')64 || combined.includes('verify you are human')65 || combined.includes('captcha')66 || combined.includes('bot detection');67};68
69export const waitForCostcoContent = async (page) => {70 const hasAkamaiChallenge = async () => {71 const html = await page.content().catch(() => '');72 return /sec-if-cpt-container|powered and protected by/i.test(html);73 };74
75 if (await hasAkamaiChallenge()) {76 await page.waitForFunction(() => {77 const html = document.documentElement.outerHTML;78 return !/sec-if-cpt-container|powered and protected by/i.test(html);79 }, { timeout: 35000 }).catch(() => {});80 }81
82 await page.waitForLoadState('networkidle', { timeout: 15000 }).catch(() => {});83 await page.waitForFunction(() => {84 const text = document.body?.innerText || '';85 return document.querySelector('a[href*=".product."]')86 || document.querySelector('h1')87 || /no results|did not match|out of stock|add to cart/i.test(text)88 || text.length > 5000;89 }, { timeout: 20000 }).catch(() => {});90};91
92export const maybeAcceptCookies = async (page) => {93 const candidates = [94 'button:has-text("Accept All")',95 'button:has-text("Accept all")',96 'button:has-text("I Accept")',97 'button:has-text("Agree")',98 '#onetrust-accept-btn-handler',99 ];100 for (const selector of candidates) {101 const button = page.locator(selector).first();102 if (await button.isVisible({ timeout: 750 }).catch(() => false)) {103 await button.click({ timeout: 2000 }).catch(() => {});104 return;105 }106 }107};108
109export const autoScroll = async (page) => {110 await page.evaluate(async () => {111 const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));112 const steps = Math.min(8, Math.ceil(document.body.scrollHeight / Math.max(1, window.innerHeight)));113 for (let i = 0; i < steps; i += 1) {114 window.scrollBy(0, Math.max(600, window.innerHeight * 0.85));115 await delay(250);116 }117 window.scrollTo(0, 0);118 }).catch(() => {});119};120
121export const extractListingProducts = async (page, sourceUrl) => {122 const products = await page.evaluate(() => {123 const clean = (value) => (value || '').replace(/\s+/g, ' ').trim();124 const parsePriceText = (value) => {125 const match = String(value || '').replace(/,/g, '').match(/\$\s*\d+(?:\.\d{1,2})?/);126 return match ? match[0].replace(/\s+/g, '') : null;127 };128 const anchors = Array.from(document.querySelectorAll('a[href*=".product."]'));129 const byUrl = new Map();130
131 for (const anchor of anchors) {132 const href = anchor.getAttribute('href');133 if (!href || !/\.product\.\d+\.html/i.test(href)) continue;134 let productUrl;135 try {136 const parsed = new URL(href, location.href);137 parsed.hash = '';138 parsed.search = '';139 productUrl = parsed.href;140 } catch {141 continue;142 }143
144 const card = anchor.closest('[data-testid], [class*="product"], li, article, .col, .row') || anchor;145 const cardText = clean(card.innerText);146 const name = clean(147 anchor.getAttribute('aria-label')148 || anchor.getAttribute('title')149 || anchor.innerText150 || card.querySelector('[class*="description"], [class*="title"], [class*="name"], h2, h3')?.innerText,151 );152 const image = card.querySelector('img')?.currentSrc || card.querySelector('img')?.src || null;153 const priceText = parsePriceText(cardText);154 const ratingText = clean(card.querySelector('[aria-label*="star" i], [class*="rating"]')?.getAttribute('aria-label')155 || card.querySelector('[class*="rating"]')?.innerText);156 const reviewText = clean(card.querySelector('[class*="review"], [aria-label*="review" i]')?.innerText);157
158 const existing = byUrl.get(productUrl) || {};159 byUrl.set(productUrl, {160 ...existing,161 productUrl,162 name: existing.name || name || null,163 priceText: existing.priceText || priceText,164 ratingText: existing.ratingText || ratingText || null,165 reviewText: existing.reviewText || reviewText || null,166 mainImage: existing.mainImage || image,167 });168 }169
170 for (const script of Array.from(document.scripts)) {171 const text = script.textContent || '';172 if (!text.includes('.product.')) continue;173 const matches = text.match(/https?:\\?\/\\?\/www\.costco\.com\\?\/[^"' <>\s]+?\.product\.\d+\.html/g) || [];174 for (const raw of matches) {175 const href = raw.replace(/\\\//g, '/');176 try {177 const parsed = new URL(href);178 parsed.hash = '';179 parsed.search = '';180 if (!byUrl.has(parsed.href)) {181 byUrl.set(parsed.href, { productUrl: parsed.href, name: null, priceText: null, mainImage: null });182 }183 } catch {}184 }185 }186
187 return Array.from(byUrl.values());188 });189
190 return products191 .map((item) => {192 const productUrl = normalizeCostcoUrl(item.productUrl, sourceUrl);193 if (!productUrl) return null;194 const price = parsePrice(item.priceText);195 return addCompetitorCompatibleFields({196 productId: productIdFromUrl(productUrl),197 sku: null,198 productUrl,199 sourceUrl,200 name: cleanText(item.name) || `Costco product ${productIdFromUrl(productUrl) || ''}`.trim(),201 brand: null,202 price,203 listPrice: null,204 minPrice: null,205 maxPrice: null,206 currency: price == null ? null : 'USD',207 priceText: cleanText(item.priceText),208 unitPrice: null,209 rating: parsePrice(item.ratingText),210 reviewCount: parseInteger(item.reviewText),211 availability: null,212 memberOnly: false,213 description: null,214 features: [],215 specifications: {},216 breadcrumbs: [],217 mainImage: item.mainImage || null,218 images: compactArray([item.mainImage]),219 category: null,220 scrapedAt: new Date().toISOString(),221 });222 })223 .filter(Boolean);224};225
226export const extractPaginationUrls = async (page, baseUrl) => {227 const urls = await page.evaluate(() => {228 return Array.from(document.querySelectorAll('a[href]'))229 .map((anchor) => ({230 href: anchor.getAttribute('href'),231 rel: anchor.getAttribute('rel') || '',232 text: (anchor.innerText || anchor.getAttribute('aria-label') || '').trim(),233 }))234 .filter(({ href, rel, text }) => {235 if (!href) return false;236 const marker = `${rel} ${text}`.toLowerCase();237 return marker.includes('next') || /[?&](page|currentPage|p)=\d+/i.test(href);238 })239 .map(({ href }) => href);240 });241 return compactArray(urls.map((url) => normalizeCostcoUrl(url, baseUrl)));242};243
244export const extractProductDetails = async (page, sourceUrl) => {245 const raw = await page.evaluate(() => {246 const clean = (value) => (value || '').replace(/\s+/g, ' ').trim();247 const textOf = (selectors) => {248 for (const selector of selectors) {249 const element = document.querySelector(selector);250 const text = clean(element?.textContent || element?.getAttribute?.('content') || '');251 if (text) return text;252 }253 return null;254 };255 const attrOf = (selectors, attr) => {256 for (const selector of selectors) {257 const element = document.querySelector(selector);258 const value = element?.getAttribute?.(attr);259 if (value) return value;260 }261 return null;262 };263 const parseJson = (text) => {264 try { return JSON.parse(text); } catch { return null; }265 };266 const flatten = (value) => {267 if (!value) return [];268 if (Array.isArray(value)) return value.flatMap(flatten);269 if (typeof value === 'object' && Array.isArray(value['@graph'])) return [value, ...value['@graph'].flatMap(flatten)];270 return [value];271 };272 const jsonObjects = Array.from(document.querySelectorAll('script[type="application/ld+json"]'))273 .flatMap((script) => flatten(parseJson(script.textContent || '')));274 const product = jsonObjects.find((item) => {275 const type = item?.['@type'];276 return type === 'Product' || (Array.isArray(type) && type.includes('Product'));277 }) || {};278 const breadcrumbList = jsonObjects.find((item) => item?.['@type'] === 'BreadcrumbList');279 const breadcrumbs = (breadcrumbList?.itemListElement || [])280 .map((entry) => clean(entry?.item?.name || entry?.name))281 .filter(Boolean);282 const offers = Array.isArray(product.offers) ? product.offers[0] : product.offers || {};283 const aggregateRating = product.aggregateRating || {};284 const imageValues = Array.isArray(product.image) ? product.image : [product.image];285 const domImages = Array.from(document.querySelectorAll('img'))286 .map((img) => img.currentSrc || img.src)287 .filter((src) => src && /costco|scene7|images/i.test(src));288 const tableSpecs = {};289 for (const row of Array.from(document.querySelectorAll('table tr'))) {290 const cells = Array.from(row.querySelectorAll('th, td')).map((cell) => clean(cell.textContent)).filter(Boolean);291 if (cells.length >= 2 && cells[0].length < 80 && cells[1].length < 300) {292 tableSpecs[cells[0].replace(/:$/, '')] = cells.slice(1).join(' ');293 }294 }295 for (const item of Array.from(document.querySelectorAll('dl'))) {296 const terms = Array.from(item.querySelectorAll('dt'));297 for (const term of terms) {298 const key = clean(term.textContent)?.replace(/:$/, '');299 const value = clean(term.nextElementSibling?.textContent);300 if (key && value && key.length < 80 && value.length < 300) tableSpecs[key] = value;301 }302 }303 const bodyText = clean(document.body.innerText);304 const featureSelectors = [305 '[class*="feature"] li',306 '[class*="Feature"] li',307 '[class*="product"] li',308 '#productDetails li',309 '[data-testid*="feature"] li',310 ];311 const features = Array.from(document.querySelectorAll(featureSelectors.join(',')))312 .map((item) => clean(item.textContent))313 .filter((text) => text && text.length >= 5 && text.length <= 240)314 .slice(0, 80);315 const skuText = bodyText?.match(/(?:Item|Item #|Item Number|SKU)\s*#?:?\s*(\d{4,})/i)?.[1] || null;316 const availability = clean(offers.availability || '')317 .replace(/^https?:\/\/schema\.org\//i, '')318 || (bodyText && /out of stock/i.test(bodyText) ? 'OutOfStock' : null)319 || (bodyText && /in stock/i.test(bodyText) ? 'InStock' : null);320
321 return {322 url: location.href,323 canonical: attrOf(['link[rel="canonical"]'], 'href'),324 name: clean(product.name) || textOf(['h1', '[itemprop="name"]', 'meta[property="og:title"]']),325 brand: clean(product.brand?.name || product.brand) || textOf(['[itemprop="brand"]', '[class*="brand"]']),326 description: clean(product.description) || textOf(['meta[name="description"]', 'meta[property="og:description"]', '[itemprop="description"]', '[class*="description"]']),327 price: offers.price || attrOf(['meta[property="product:price:amount"]', '[itemprop="price"]'], 'content') || textOf(['[class*="price"]']),328 priceCurrency: offers.priceCurrency || attrOf(['meta[property="product:price:currency"]'], 'content'),329 listPrice: textOf(['[class*="list-price"]', '[class*="was-price"]', '[class*="strike"]']),330 rating: aggregateRating.ratingValue || textOf(['[itemprop="ratingValue"]', '[class*="rating"]']),331 reviewCount: aggregateRating.reviewCount || aggregateRating.ratingCount || textOf(['[itemprop="reviewCount"]', '[class*="review"]']),332 availability,333 sku: product.sku || product.mpn || skuText,334 breadcrumbs,335 features,336 specifications: tableSpecs,337 images: [...imageValues, ...domImages].filter(Boolean),338 mainImage: attrOf(['meta[property="og:image"]'], 'content') || imageValues.find(Boolean) || domImages[0] || null,339 bodyText,340 };341 });342
343 const productUrl = normalizeCostcoUrl(raw.canonical || raw.url, sourceUrl) || normalizeCostcoUrl(page.url(), sourceUrl);344 const images = compactArray((raw.images || []).map((url) => normalizeCostcoUrl(url, productUrl) || url));345 const price = parsePrice(raw.price);346 const listPrice = parsePrice(raw.listPrice);347 const productId = productIdFromUrl(productUrl);348 const name = cleanText(raw.name) || `Costco product ${productId || raw.sku || ''}`.trim();349 const breadcrumbs = compactArray(raw.breadcrumbs || []);350
351 return addCompetitorCompatibleFields({352 productId,353 sku: cleanText(raw.sku) || productId,354 productUrl,355 sourceUrl,356 name,357 brand: cleanText(raw.brand),358 price,359 listPrice,360 minPrice: null,361 maxPrice: null,362 currency: cleanText(raw.priceCurrency) || (price == null ? null : 'USD'),363 priceText: cleanText(raw.price),364 unitPrice: null,365 rating: parsePrice(raw.rating),366 reviewCount: parseInteger(raw.reviewCount),367 availability: cleanText(raw.availability),368 memberOnly: /member only|members only|sign in to see price|member-only/i.test(raw.bodyText || ''),369 description: cleanText(raw.description),370 features: compactArray(raw.features || []),371 specifications: raw.specifications || {},372 breadcrumbs,373 mainImage: raw.mainImage || images[0] || null,374 images,375 category: breadcrumbs.length ? breadcrumbs[breadcrumbs.length - 1] : null,376 scrapedAt: new Date().toISOString(),377 });378};379
380export const isLikelyProductLink = (url) => PRODUCT_LINK_RE.test(url || '');1const COSTCO_HOST_RE = /(^|\.)costco\.com$/i;2
3export const cleanText = (value) => {4 if (value == null) return null;5 const cleaned = String(value)6 .replace(/\s+/g, ' ')7 .replace(/\u00a0/g, ' ')8 .trim();9 return cleaned || null;10};11
12export const compactArray = (values) => {13 const seen = new Set();14 const output = [];15 for (const value of values || []) {16 const cleaned = typeof value === 'string' ? cleanText(value) : value;17 if (!cleaned) continue;18 const key = typeof cleaned === 'string' ? cleaned.toLowerCase() : JSON.stringify(cleaned);19 if (seen.has(key)) continue;20 seen.add(key);21 output.push(cleaned);22 }23 return output;24};25
26export const parsePrice = (value) => {27 if (value == null) return null;28 if (typeof value === 'number' && Number.isFinite(value)) return value;29 const text = String(value).replace(/,/g, '');30 const match = text.match(/(?:USD|\$)?\s*(\d+(?:\.\d{1,2})?)/i);31 return match ? Number(match[1]) : null;32};33
34export const parseInteger = (value) => {35 if (value == null) return null;36 if (typeof value === 'number' && Number.isFinite(value)) return Math.trunc(value);37 const match = String(value).replace(/,/g, '').match(/(\d+)/);38 return match ? Number(match[1]) : null;39};40
41export const toAbsoluteUrl = (url, base = 'https://www.costco.com/') => {42 if (!url) return null;43 try {44 const absolute = new URL(url, base);45 absolute.hash = '';46 return absolute.href;47 } catch {48 return null;49 }50};51
52export const normalizeCostcoUrl = (url, base) => {53 const absolute = toAbsoluteUrl(url, base);54 if (!absolute) return null;55 try {56 const parsed = new URL(absolute);57 if (!COSTCO_HOST_RE.test(parsed.hostname)) return null;58 parsed.hash = '';59 if (isProductUrl(parsed.href)) {60 parsed.search = '';61 }62 return parsed.href;63 } catch {64 return null;65 }66};67
68export const isCostcoUrl = (url) => {69 try {70 return COSTCO_HOST_RE.test(new URL(url).hostname);71 } catch {72 return false;73 }74};75
76export const isProductUrl = (url) => /\.product\.\d+\.html(?:$|[?#])/i.test(url || '');77
78export const classifyCostcoUrl = (url) => {79 if (isProductUrl(url)) return 'PRODUCT';80 return 'SEARCH';81};82
83export const productIdFromUrl = (url) => {84 const match = String(url || '').match(/\.product\.(\d+)\.html/i);85 return match ? match[1] : null;86};87
88export const keywordToSearchUrl = (keyword) => {89 const cleaned = cleanText(keyword);90 if (!cleaned) return null;91 return `https://www.costco.com/s?keyword=${encodeURIComponent(cleaned)}`;92};93
94export const slugForKey = (value) => {95 return String(value || 'page')96 .replace(/^https?:\/\//i, '')97 .replace(/[^a-z0-9]+/gi, '-')98 .replace(/^-+|-+$/g, '')99 .slice(0, 80) || 'page';100};1import { Actor } from 'apify';2import { log } from 'crawlee';3import { Impit } from 'impit';4import {5 cleanText,6 compactArray,7 isCostcoUrl,8 isProductUrl,9 keywordToSearchUrl,10 normalizeCostcoUrl,11 parseInteger,12 parsePrice,13 productIdFromUrl,14} from './helpers.js';15
16const COSTCO_SEARCH_BASE_URL = 'https://search.costco.com/api/apps';17const COSTCO_SEARCH_APP = 'www_costco_com';18const COSTCO_SEARCH_API_KEY = '273db6be-f015-4de7-b0d6-dd4746ccd5c3';19const COSTCO_LOCALE = 'en-US';20const DEFAULT_ROWS = 24;21
22const buildProductSlug = (name, fallbackId) => {23 const slug = String(name || '')24 .toLowerCase()25 .replace(/['’]/g, '')26 .replace(/[^a-z0-9-]+/g, '-')27 .replace(/^-+|-+$/g, '');28 return slug || `costco-product-${fallbackId || 'item'}`;29};30
31const numberOrNull = (value) => {32 if (typeof value === 'number' && Number.isFinite(value)) return value;33 return parsePrice(value);34};35
36const boolOrNull = (value) => {37 if (typeof value === 'boolean') return value;38 if (value == null || value === '') return null;39 if (String(value).toLowerCase() === 'true') return true;40 if (String(value).toLowerCase() === 'false') return false;41 return null;42};43
44const getUrlFromEntry = (entry) => {45 if (typeof entry === 'string') return entry;46 return entry?.url || entry?.request?.url || null;47};48
49const getKeywordFromUrl = (url) => {50 try {51 const parsed = new URL(url);52 return cleanText(parsed.searchParams.get('keyword') || parsed.searchParams.get('q'));53 } catch {54 return null;55 }56};57
58const getCategoryPath = (url) => {59 try {60 const parsed = new URL(url);61 const pathname = parsed.pathname || '';62 if (!pathname || pathname === '/' || pathname.includes('/consumer-web/')) return null;63 if (/CatalogSearch|\/s$/i.test(pathname)) return null;64 return pathname.endsWith('.html') ? pathname : `${pathname.replace(/\/$/, '')}.html`;65 } catch {66 return null;67 }68};69
70const normalizeSourceUrl = (url) => {71 const normalized = normalizeCostcoUrl(url);72 if (!normalized) return null;73 const keyword = getKeywordFromUrl(normalized);74 if (keyword && !isProductUrl(normalized)) return keywordToSearchUrl(keyword);75 return normalized;76};77
78const createSources = ({ startUrls, searchUrls, keywords }) => {79 const rawUrls = [80 ...(startUrls || []).map(getUrlFromEntry),81 ...(searchUrls || []),82 ].filter(Boolean);83
84 const sources = [];85 for (const rawUrl of rawUrls) {86 const normalized = normalizeSourceUrl(rawUrl);87 if (!normalized || !isCostcoUrl(normalized)) continue;88
89 if (isProductUrl(normalized)) {90 const productId = productIdFromUrl(normalized);91 if (productId) {92 sources.push({93 type: 'product',94 query: productId,95 sourceUrl: normalized,96 productId,97 });98 }99 continue;100 }101
102 const keyword = getKeywordFromUrl(normalized);103 if (keyword) {104 sources.push({105 type: 'search',106 query: keyword,107 sourceUrl: keywordToSearchUrl(keyword),108 });109 continue;110 }111
112 const categoryPath = getCategoryPath(normalized);113 if (categoryPath) {114 sources.push({115 type: 'category',116 query: '*:*',117 categoryPath,118 sourceUrl: normalized,119 });120 }121 }122
123 for (const keyword of keywords || []) {124 const cleaned = cleanText(keyword);125 if (!cleaned) continue;126 sources.push({127 type: 'search',128 query: cleaned,129 sourceUrl: keywordToSearchUrl(cleaned),130 });131 }132
133 const seen = new Set();134 return sources.filter((source) => {135 const key = `${source.type}:${source.query}:${source.categoryPath || ''}:${source.sourceUrl}`;136 if (seen.has(key)) return false;137 seen.add(key);138 return true;139 });140};141
142const buildSearchUrl = ({ source, start, rows }) => {143 const route = source.type === 'category' ? 'navigation' : 'search';144 const url = new URL(`${COSTCO_SEARCH_BASE_URL}/${COSTCO_SEARCH_APP}/query/${COSTCO_SEARCH_APP}_${route}`);145 url.searchParams.set('expoption', 'def');146 url.searchParams.set('q', source.query || '*:*');147 url.searchParams.set('locale', COSTCO_LOCALE);148 url.searchParams.set('start', String(start));149 url.searchParams.set('expand', 'false');150 url.searchParams.set('loc', '*');151 url.searchParams.set('rows', String(rows));152 if (source.type === 'category' && source.categoryPath) {153 url.searchParams.set('url', source.categoryPath);154 }155 return url.href;156};157
158const createHttpClient = (proxyUrl) => new Impit({159 browser: 'chrome',160 proxyUrl,161 timeout: 60000,162 headers: {163 accept: 'application/json,text/plain,*/*',164 'accept-language': 'en-US,en;q=0.9',165 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',166 },167});168
169const fetchJson = async ({ url, sourceUrl, proxyConfiguration, debug, maxRetries = 5 }) => {170 let lastError;171 for (let attempt = 1; attempt <= maxRetries; attempt += 1) {172 const proxyInfo = proxyConfiguration ? await proxyConfiguration.newProxyInfo() : null;173 const client = createHttpClient(proxyInfo?.url);174 try {175 const response = await client.fetch(url, {176 headers: {177 accept: 'application/json',178 'x-api-key': COSTCO_SEARCH_API_KEY,179 origin: 'https://www.costco.com',180 referer: sourceUrl || 'https://www.costco.com/',181 },182 });183 const text = await response.text();184 if (!response.ok) {185 if (debug) {186 await Actor.setValue(`debug-response-${Date.now()}.txt`, text, { contentType: 'text/plain' });187 }188 throw new Error(`Costco API returned HTTP ${response.status}`);189 }190 return JSON.parse(text);191 } catch (error) {192 lastError = error;193 log.warning(`Costco API attempt ${attempt}/${maxRetries} failed: ${error.message}`);194 await new Promise((resolve) => setTimeout(resolve, 300 * attempt));195 }196 }197 throw lastError;198};199
200const extractImageUrls = (doc) => {201 const fromImages = Array.isArray(doc.images)202 ? doc.images.flatMap((image) => {203 if (typeof image === 'string') return [image];204 if (!image || typeof image !== 'object') return [];205 return Object.values(image).filter((value) => typeof value === 'string');206 })207 : [];208 return compactArray([209 doc.item_collateral_primaryimage,210 doc.item_product_primary_image,211 doc.image,212 ...fromImages,213 ]);214};215
216const extractFeatures = (doc) => compactArray(217 Array.isArray(doc.item_product_marketing_features) ? doc.item_product_marketing_features : [],218);219
220const extractOnlineOnlyTags = (doc) => {221 const rawTags = Array.isArray(doc.item_pill_attributes) ? doc.item_pill_attributes : [];222 return compactArray(rawTags.map((tag) => String(tag).split('|').at(-1)));223};224
225const mapDocToRecord = ({ doc, source, breadcrumb }) => {226 const groupId = cleanText(doc.group_id) || cleanText(doc.groupId);227 const itemNumber = cleanText(doc.item_location_itemNumber)228 || cleanText(doc.item_number)229 || cleanText(doc.itemNumber)230 || cleanText(doc.id)?.split('!')[0]231 || null;232 const name = cleanText(doc.item_product_name)233 || cleanText(doc.name)234 || cleanText(doc.item_name)235 || `Costco product ${groupId || itemNumber || ''}`.trim();236 const itemName = cleanText(doc.item_name)237 || cleanText(doc.item_short_description)238 || name;239 const productId = groupId || itemNumber;240 const productUrl = `https://www.costco.com/${buildProductSlug(name, productId)}.product.${productId}.html`;241 const images = extractImageUrls(doc);242 const price = numberOrNull(doc.item_location_pricing_salePrice ?? doc.minSalePrice);243 const listPrice = numberOrNull(doc.item_location_pricing_listPrice ?? doc.maxSalePrice);244 const pricePerUnit = numberOrNull(doc.item_location_pricing_pricePerUnit_price ?? price);245 const rating = numberOrNull(doc.item_ratings) || numberOrNull(doc.item_rating_average_value);246 const reviewsCount = parseInteger(doc.item_product_review_count) || parseInteger(doc.item_review_count);247 const availability = cleanText(doc.item_location_availability || doc.deliveryStatus || doc.item_location_stockStatus);248 const isInStock = boolOrNull(doc.isItemInStock)249 ?? (availability ? !/out|unavailable|sold out/i.test(availability) : null);250 const categoryPaths = compactArray(doc.categoryPath_ss || []);251 const breadcrumbNames = compactArray((breadcrumb || []).map((item) => item?.name));252 const marketingFeatures = extractFeatures(doc);253 const marketingKeywords = (Array.isArray(doc.item_product_marketingcontent_keywords) ? doc.item_product_marketingcontent_keywords : [])254 .map(cleanText)255 .filter(Boolean);256 const currency = cleanText(doc.item_location_currencyCode) || (price == null ? null : 'USD');257 const brands = compactArray(Array.isArray(doc.Brand_attr) ? doc.Brand_attr : [doc.Brand_attr]);258 const onlineOnlyTags = extractOnlineOnlyTags(doc);259
260 return {261 productId,262 sku: itemNumber,263 productUrl,264 sourceUrl: source.sourceUrl,265 name,266 brand: brands[0] || null,267 brands,268 price,269 listPrice,270 minPrice: numberOrNull(doc.minSalePrice ?? price),271 maxPrice: numberOrNull(doc.maxSalePrice ?? price),272 currency,273 priceText: price == null ? null : `$${price.toFixed(2)}`,274 unitPrice: pricePerUnit,275 rating,276 ratingValues: Array.isArray(doc.item_rating_value) ? doc.item_rating_value : undefined,277 reviewCount: reviewsCount,278 availability,279 memberOnly: Boolean(doc.item_member_only),280 description: cleanText(doc.description || doc.item_short_description || doc.item_product_short_description),281 features: marketingFeatures,282 specifications: {},283 breadcrumbs: breadcrumbNames,284 mainImage: images[0] || null,285 images,286 category: breadcrumbNames.at(-1) || categoryPaths.at(-1) || null,287 categoryUrl: source.sourceUrl,288 id: cleanText(doc.id) || (itemNumber ? `${itemNumber}!item.en-US` : null),289 groupId,290 itemNumber,291 itemName,292 manufacturingSkus: Array.isArray(doc.item_manufacturing_skus) ? doc.item_manufacturing_skus : undefined,293 onlineOnlyTags,294 image: images[0] || null,295 currencyCode: currency,296 pricePerUnit,297 reviewsCount,298 isBuyable: boolOrNull(doc.item_buyable ?? doc.item_product_buyable),299 isPublished: boolOrNull(doc.item_status_published ?? doc.item_product_status_published) ?? true,300 isInStock,301 availabilityStatus: availability,302 stockStatus: cleanText(doc.item_location_stockStatus || availability),303 deliveryStatus: cleanText(doc.deliveryStatus || availability),304 minQuantityRestriction: parseInteger(doc.item_location_fulfillment_restrictions_minQty),305 maxQuantityRestriction: parseInteger(doc.item_location_fulfillment_restrictions_maxQty),306 startDate: cleanText(doc.item_startDate),307 marketingFeatures,308 marketingKeywords,309 categoryPaths,310 itemClassification: cleanText(doc.item_classification_itemclass),311 as400Category: cleanText(doc.item_as400_category),312 isMemberOnly: Boolean(doc.item_member_only),313 isSingleSku: boolOrNull(doc.hasSingleSku),314 scrapedAt: new Date().toISOString(),315 };316};317
318await Actor.main(async () => {319 const input = (await Actor.getInput()) || {};320 const {321 startUrls = [],322 searchUrls = [],323 keywords = [],324 maxItems = 50,325 maxItemsPerStartUrl = 50,326 maxSearchPagesPerStartUrl = 3,327 proxyConfiguration: proxyInput,328 debug = false,329 } = input;330
331 const sources = createSources({ startUrls, searchUrls, keywords });332 if (sources.length === 0) {333 // Report bad input as a row, not as a failed run. This is the highest-334 // volume actor in the fleet, so its public success rate is also the most335 // expensive to spend on a customer's empty input.336 const message = 'Input must contain at least one Costco start URL, search URL, product URL, category URL, or keyword.';337 log.error(message);338 await Actor.pushData({339 recordType: 'diagnostic',340 diagnosticReason: 'no_input',341 message,342 productUrl: '',343 name: '[no_input]',344 scrapedAt: new Date().toISOString(),345 });346 await Actor.setStatusMessage(message);347 return;348 }349
350 const totalLimit = Math.max(1, Number(maxItems) || 50);351 const perStartLimit = Math.max(1, Number(maxItemsPerStartUrl) || totalLimit);352 const pageLimit = Math.max(1, Number(maxSearchPagesPerStartUrl) || 1);353 const proxyConfiguration = await Actor.createProxyConfiguration(proxyInput || {354 useApifyProxy: true,355 apifyProxyGroups: ['RESIDENTIAL'],356 apifyProxyCountry: 'US',357 });358
359 const seenProducts = new Set();360 let pushedCount = 0;361
362 log.info(`Starting Costco API scrape with ${sources.length} source(s), maxItems=${totalLimit}`);363 for (const source of sources) {364 if (pushedCount >= totalLimit) break;365
366 let pushedForSource = 0;367 const sourcePageLimit = source.type === 'product' ? 1 : pageLimit;368 for (let pageIndex = 0; pageIndex < sourcePageLimit; pageIndex += 1) {369 if (pushedCount >= totalLimit || pushedForSource >= perStartLimit) break;370
371 const remaining = Math.min(totalLimit - pushedCount, perStartLimit - pushedForSource);372 const rows = source.type === 'product' ? Math.min(DEFAULT_ROWS, Math.max(5, remaining)) : Math.min(DEFAULT_ROWS, remaining);373 const start = pageIndex * DEFAULT_ROWS;374 const url = buildSearchUrl({ source, start, rows });375 const data = await fetchJson({376 url,377 sourceUrl: source.sourceUrl,378 proxyConfiguration,379 debug,380 });381
382 const docs = Array.isArray(data?.response?.docs) ? data.response.docs : [];383 const breadcrumb = Array.isArray(data?.fusion?.breadcrumb) ? data.fusion.breadcrumb : [];384 log.info(`Costco ${source.type} source returned ${docs.length} item(s) at offset ${start}: ${source.sourceUrl}`);385 if (docs.length === 0) break;386
387 for (const doc of docs) {388 if (pushedCount >= totalLimit || pushedForSource >= perStartLimit) break;389 if (source.type === 'product') {390 const wanted = String(source.productId);391 const docIds = [doc.group_id, doc.item_number, doc.item_location_itemNumber, doc.id].filter(Boolean).map(String);392 if (!docIds.some((value) => value.includes(wanted))) continue;393 }394 const record = mapDocToRecord({ doc, source, breadcrumb });395 if (!record.productUrl || seenProducts.has(record.productUrl)) continue;396 seenProducts.add(record.productUrl);397 await Actor.pushData(record);398 pushedCount += 1;399 pushedForSource += 1;400 }401
402 const totalFound = Number(data?.response?.numFound || 0);403 if (docs.length < rows || (totalFound && start + docs.length >= totalFound)) break;404 }405 }406
407 if (pushedCount === 0) {408 // An empty dataset reads to Apify's daily auto-test as a broken actor.409 // With 1,500+ runs a month this is the actor least able to afford a410 // silent zero-item run, so say why it was empty.411 const message = 'No Costco products matched. The search returned nothing, or the '412 + 'category/product URL is no longer live. Try a broader keyword or check the URL.';413 log.warning(message);414 await Actor.pushData({415 recordType: 'diagnostic',416 diagnosticReason: 'no_results',417 message,418 productUrl: '',419 name: '[no_results]',420 scrapedAt: new Date().toISOString(),421 });422 await Actor.setStatusMessage(message);423 pushedCount = 1;424 }425
426 log.info(`Finished Costco scrape with ${pushedCount} product record(s).`);427});