1import { Actor } from 'apify';
2import { gotScraping } from 'got-scraping';
3import { delay } from './utils.js';
4
5const BASE_URL = 'https://console-backend.apify.com/public/store/memoized-search';
6
7
8export class ScraperState {
9 constructor(persistenceKey) {
10 this.persistenceKey = persistenceKey;
11 this.processedItems = [];
12 this.currentOffset = 0;
13 this.totalResults = 0;
14 this.stats = { success: 0, failures: 0 };
15 }
16
17 async load() {
18 const data = await Actor.getValue(this.persistenceKey) || {};
19 Object.assign(this, data);
20 }
21
22 async save() {
23 await Actor.setValue(this.persistenceKey, {
24 processedItems: this.processedItems,
25 currentOffset: this.currentOffset,
26 totalResults: this.totalResults,
27 stats: this.stats
28 });
29 }
30}
31
32export async function scrapeStore(input) {
33
34 const PERSISTENCE_KEY = `SEARCH_STATE`;
35 const state = new ScraperState(PERSISTENCE_KEY);
36 await state.load();
37
38 Actor.on('migrating', async () => {
39 console.log(`Migration detected - saving state for "${input.search}"`);
40 await state.save();
41 });
42
43 let hasMore = true;
44 const itemsPerRequest = input.limit;
45 const maxResults = input.maxResults;
46 while (hasMore) {
47 const remainingItems = maxResults > 0
48 ? Math.max(0, maxResults - state.stats.success)
49 : itemsPerRequest;
50
51 const currentLimit = maxResults > 0
52 ? Math.min(itemsPerRequest, remainingItems)
53 : itemsPerRequest;
54
55 if (maxResults > 0 && remainingItems <= 0) {
56 console.log('Reached maximum requested results');
57 break;
58 }
59
60 const params = {
61 search: input.search || '',
62 sortBy: input.sortBy || 'RELEVANCE',
63 category: input.category || '',
64 pricingModel: input.pricingModel || '',
65 limit: currentLimit.toString(),
66 offset: state.currentOffset.toString()
67 };
68
69
70 if (input.managedBy) {
71 params.managedBy = input.managedBy;
72 }
73
74 const url = `${BASE_URL}?${new URLSearchParams(params)}`;
75 console.log(`Fetching ${currentLimit} items from offset ${state.currentOffset}...`);
76
77
78
79 try {
80 const response = await gotScraping.get(url, {
81 headers: {
82 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',
83 'Accept': 'application/json',
84 'Referer': 'https://console.apify.com/',
85 'Origin': 'https://console.apify.com'
86 },
87 responseType: 'json'
88 });
89
90 const data = response.body;
91
92 if (!data.items || !Array.isArray(data.items)) {
93 const errorMsg = `Invalid response format - missing items array at offset ${state.currentOffset}`;
94 console.error(errorMsg);
95 throw new Error(errorMsg);
96 }
97
98 for (const item of data.items) {
99 if (maxResults > 0 && state.stats.success >= maxResults) {
100 hasMore = false;
101 break;
102 }
103 try {
104 const itemData = {
105 id: item.id,
106 title: item.title,
107 name: item.name,
108 username: item.username,
109 userFullName: item.userFullName,
110 userPictureUrl: item.userPictureUrl,
111 description: item.description,
112 pictureUrl: item.pictureUrl,
113 notice: item.notice,
114 actorReviewRating: item.actorReviewRating,
115 bookmarkCount: item.bookmarkCount,
116 url: `https://console.apify.com/actors/${item.id}`,
117 price: item.currentPricingInfo
118 ? `${item.currentPricingInfo.pricingModel || 'UNKNOWN'} - $${item.currentPricingInfo.pricePerUnitUsd || 0}`
119 : 'Free or Unknown',
120 totalUsers7Days: item.stats?.totalUsers7Days,
121 totalUsers30Days: item.stats?.totalUsers30Days,
122 stats: {
123 totalBuilds: item.stats?.totalBuilds,
124 totalRuns: item.stats?.totalRuns,
125 totalUsers: item.stats?.totalUsers,
126 totalUsers90Days: item.stats?.totalUsers90Days,
127 lastRunStartedAt: item.stats?.lastRunStartedAt,
128 publicActorRunStats30Days: item.stats?.publicActorRunStats30Days
129 },
130 currentPricingInfo: {
131 pricingModel: item.currentPricingInfo?.pricingModel,
132 pricePerUnitUsd: item.currentPricingInfo?.pricePerUnitUsd,
133 trialMinutes: item.currentPricingInfo?.trialMinutes,
134 startedAt: item.currentPricingInfo?.startedAt,
135 createdAt: item.currentPricingInfo?.createdAt,
136 apifyMarginPercentage: item.currentPricingInfo?.apifyMarginPercentage,
137 notifiedAboutChangeAt: item.currentPricingInfo?.notifiedAboutChangeAt,
138 notifiedAboutFutureChangeAt: item.currentPricingInfo?.notifiedAboutFutureChangeAt
139 }
140 };
141 await Actor.pushData(itemData);
142 state.stats.success++;
143 } catch (error) {
144 console.error(`Failed to process item ${item.id}:`, error.message);
145 state.stats.failures++;
146 }
147 }
148
149 state.currentOffset += data.items.length;
150 await state.save();
151 console.log(`Progress: ${state.stats.success} items fetched (${maxResults > 0 ? `${state.stats.success}/${maxResults}` : 'unlimited'})`);
152
153 if (data.items.length < currentLimit ||
154 (maxResults > 0 && state.stats.success >= maxResults)) {
155 hasMore = false;
156 console.log('Reached end condition');
157 }
158
159 if (input.batchDelay > 0 && hasMore) {
160 await delay(input.batchDelay);
161 }
162
163 } catch (error) {
164 state.stats.failures++;
165 await state.save();
166 console.error(`Failed to fetch batch at offset ${state.currentOffset}:`, error.message);
167 throw error;
168 }
169 }
170
171 await Actor.setValue(PERSISTENCE_KEY, null);
172 return {
173 stats: state.stats
174 };
175}
