1import { Actor, log } from 'apify';
2import { request, mapLimit, HttpError } from './http.js';
3import { htmlToText } from './normalize.js';
4
5await Actor.init();
6
7const input = (await Actor.getInput()) ?? {};
8const {
9 storeUrls = [],
10 outputMode = 'products',
11 keywords = [],
12 onlyAvailable = false,
13 includeDescription = true,
14 monitorMode = 'off',
15 monitorStateName = 'shopify-monitor',
16 maxProductsPerStore = 0,
17 maxConcurrency = 5,
18} = input;
19
20const kw = (Array.isArray(keywords) ? keywords : String(keywords).split(',')).map((s) => s.trim().toLowerCase()).filter(Boolean);
21const perStore = maxProductsPerStore > 0 ? maxProductsPerStore : Infinity;
22const isPpe = Actor.getChargingManager().getPricingInfo().isPayPerEvent;
23const state = monitorMode !== 'off' ? await Actor.openKeyValueStore(monitorStateName) : null;
24let stop = false;
25let saved = 0;
26const report = [];
27
28
29function parseTarget(raw) {
30 const s = String(raw).trim();
31 const u = new URL(/^https?:\/\//.test(s) ? s : `https://${s}`);
32 const col = u.pathname.match(/\/collections\/([^/?#]+)/)?.[1];
33 const prod = u.pathname.match(/\/products\/([^/?#]+)/)?.[1];
34 return { origin: u.origin, collection: col && col !== 'all' ? col : null, product: prod || null };
35}
36
37async function storeMeta(origin) {
38 try {
39 const m = await request(`${origin}/meta.json`, { retries: 1 });
40 return { storeName: m.name ?? null, storeCurrency: m.currency ?? null, storeCountry: m.country ?? null, myshopifyDomain: m.myshopify_domain ?? null };
41 } catch {
42 return { storeName: null, storeCurrency: null, storeCountry: null, myshopifyDomain: null };
43 }
44}
45
46async function* products(t) {
47 if (t.product) {
48 const d = await request(`${t.origin}/products/${t.product}.json`);
49 if (d.product) yield d.product;
50 return;
51 }
52 const base = t.collection ? `${t.origin}/collections/${t.collection}/products.json` : `${t.origin}/products.json`;
53 for (let page = 1; page <= 400; page++) {
54 const d = await request(`${base}?limit=250&page=${page}`);
55 const list = d.products || [];
56 for (const p of list) yield p;
57 if (list.length < 250) return;
58 }
59}
60
61const num = (v) => (v === null || v === undefined || v === '' ? null : Number(v));
62
63function productItem(p, t, meta) {
64 const variants = p.variants || [];
65 const prices = variants.map((v) => num(v.price)).filter((x) => x !== null);
66 const compare = variants.map((v) => num(v.compare_at_price)).filter((x) => x);
67 const url = `${t.origin}/products/${p.handle}`;
68 return {
69 type: 'product',
70 productId: String(p.id),
71 title: p.title,
72 handle: p.handle,
73 url,
74 vendor: p.vendor || null,
75 productType: p.product_type || null,
76 tags: Array.isArray(p.tags) ? p.tags : String(p.tags || '').split(',').map((s) => s.trim()).filter(Boolean),
77 price: prices.length ? Math.min(...prices) : null,
78 priceMax: prices.length ? Math.max(...prices) : null,
79 compareAtPrice: compare.length ? Math.max(...compare) : null,
80 onSale: variants.some((v) => num(v.compare_at_price) > num(v.price)),
81 currency: meta.storeCurrency,
82 available: variants.some((v) => v.available),
83 variantsCount: variants.length,
84 variantsAvailable: variants.filter((v) => v.available).length,
85 options: (p.options || []).map((o) => ({ name: o.name, values: o.values })),
86 variants: variants.map((v) => ({
87 variantId: String(v.id), title: v.title, sku: v.sku || null, price: num(v.price), compareAtPrice: num(v.compare_at_price),
88 available: Boolean(v.available), option1: v.option1, option2: v.option2, option3: v.option3, grams: v.grams ?? null,
89 })),
90 images: (p.images || []).map((i) => i.src),
91 imageUrl: p.images?.[0]?.src ?? null,
92 descriptionText: includeDescription ? htmlToText(p.body_html) : null,
93 descriptionHtml: includeDescription ? p.body_html || null : null,
94 createdAt: p.created_at ?? null,
95 publishedAt: p.published_at ?? null,
96 updatedAt: p.updated_at ?? null,
97 store: t.origin,
98 ...meta,
99 };
100}
101
102function variantRows(item) {
103 return item.variants.map((v) => ({
104 type: 'variant',
105 productId: item.productId,
106 variantId: v.variantId,
107 title: item.title,
108 variantTitle: v.title,
109 sku: v.sku,
110 price: v.price,
111 compareAtPrice: v.compareAtPrice,
112 onSale: v.compareAtPrice > v.price,
113 available: v.available,
114 option1: v.option1, option2: v.option2, option3: v.option3,
115 currency: item.currency,
116 vendor: item.vendor,
117 productType: item.productType,
118 url: `${item.url}?variant=${v.variantId}`,
119 imageUrl: item.imageUrl,
120 store: item.store,
121 storeName: item.storeName,
122 }));
123}
124
125const matches = (it) => {
126 if (onlyAvailable && !it.available) return false;
127 if (!kw.length) return true;
128 const hay = `${it.title} ${it.vendor} ${it.productType} ${it.tags.join(' ')}`.toLowerCase();
129 return kw.some((k) => hay.includes(k));
130};
131
132
133function diff(prev, it) {
134 if (!prev) return { changeType: 'new' };
135 const changes = [];
136 if (prev.price !== it.price) changes.push('price');
137 if (prev.available !== it.available || prev.variantsAvailable !== it.variantsAvailable) changes.push('stock');
138 if (!changes.length) return null;
139 return {
140 changeType: changes.join('+') + '_changed',
141 previousPrice: prev.price,
142 previousAvailable: prev.available,
143 previousVariantsAvailable: prev.variantsAvailable,
144 priceChange: it.price !== null && prev.price !== null ? +(it.price - prev.price).toFixed(2) : null,
145 };
146}
147
148async function push(items) {
149 if (!items.length || stop) return 0;
150 const res = await Actor.pushData(items, 'product');
151 const n = isPpe ? res.chargedCount : items.length;
152 saved += n;
153 if (res.eventChargeLimitReached || (isPpe && n < items.length)) {
154 stop = true;
155 log.warning('The maximum cost per run you set was reached. Stopping.');
156 }
157 return n;
158}
159
160await mapLimit(storeUrls.map((s) => (typeof s === 'string' ? s : s?.url)).filter(Boolean), Math.max(1, Math.min(maxConcurrency, 20)), async (raw) => {
161 if (stop) return;
162 const rep = { input: raw, products: 0, saved: 0 };
163 report.push(rep);
164 let t;
165 try {
166 t = parseTarget(raw);
167 } catch {
168 rep.error = 'Invalid URL';
169 return;
170 }
171 const meta = await storeMeta(t.origin);
172 const stateKey = `store-${t.origin.replace(/^https?:\/\//, '')}${t.collection ? `-${t.collection}` : ''}`.replace(/[^a-zA-Z0-9!\-_.'()]/g, '_').slice(0, 250);
173 const prevState = state ? (await state.getValue(stateKey)) || null : null;
174 const nextState = {};
175 let batch = [];
176 try {
177 for await (const p of products(t)) {
178 if (stop) break;
179 const it = productItem(p, t, meta);
180 nextState[it.productId] = { price: it.price, available: it.available, variantsAvailable: it.variantsAvailable, title: it.title, url: it.url };
181 if (!matches(it)) continue;
182 rep.products++;
183 if (rep.products > perStore) {
184 if (!state) break;
185 continue;
186 }
187 let out = it;
188 if (state && prevState) {
189 const change = diff(prevState[it.productId], it);
190 if (!change) continue;
191 if (monitorMode === 'new' && change.changeType !== 'new') continue;
192 out = { ...it, ...change };
193 } else if (state) {
194 out = { ...it, changeType: 'baseline' };
195 }
196 batch.push(...(outputMode === 'variants'
197 ? variantRows(out).filter((v) => !onlyAvailable || v.available).map((v) => ({ ...v, changeType: out.changeType }))
198 : [out]));
199 if (batch.length >= 200) { rep.saved += await push(batch); batch = []; }
200 }
201
202 if (state && prevState && monitorMode === 'changes' && !t.product) {
203 for (const [id, prev] of Object.entries(prevState)) {
204 if (!nextState[id]) batch.push({ type: 'product', productId: id, title: prev.title, url: prev.url, store: t.origin, ...meta, changeType: 'removed', previousPrice: prev.price });
205 }
206 }
207 rep.saved += await push(batch);
208 if (state && !stop) await state.setValue(stateKey, nextState);
209 log.info(`${meta.storeName || t.origin}${t.collection ? ` / ${t.collection}` : ''}: ${rep.products} matching products, ${rep.saved} items saved`);
210 } catch (err) {
211 rep.error = err instanceof HttpError && [401, 403, 404].includes(err.status)
212 ? `Not a public Shopify store or products are hidden (HTTP ${err.status}).`
213 : err.message;
214 log.warning(`${raw}: ${rep.error}`);
215 }
216});
217
218await Actor.setValue('STORES_REPORT', report);
219log.info(`Done. ${saved} items saved from ${report.filter((r) => !r.error).length}/${report.length} stores.`);
220await Actor.setStatusMessage(`${saved} items from ${report.filter((r) => !r.error).length} stores`, { isStatusMessageTerminal: true });
221await Actor.exit();
