1import { Actor } from 'apify';
2import axios from 'axios';
3import * as cheerio from 'cheerio';
4
5const API_BASE = 'https://disdex.io/servers';
6const SITE_BASE = 'https://disdex.io';
7
8const DEFAULT_HEADERS = {
9 accept: '*/*',
10 'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',
11 'cache-control': 'no-cache',
12 'hx-request': 'true',
13 pragma: 'no-cache',
14 'sec-ch-ua': '"Chromium";v="148", "Google Chrome";v="148", "Not/A)Brand";v="99"',
15 'sec-ch-ua-mobile': '?0',
16 'sec-ch-ua-platform': '"macOS"',
17 'sec-fetch-dest': 'empty',
18 'sec-fetch-mode': 'cors',
19 'sec-fetch-site': 'same-origin',
20 'user-agent':
21 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36',
22};
23
24const FILTER_FIELDS = [
25 'tag',
26 'badge',
27 'from_members',
28 'to_members',
29 'from_online',
30 'to_online',
31 'from_created',
32 'to_created',
33 'from_added',
34 'to_added',
35 'from_boosts',
36 'to_boosts',
37 'nsfw',
38 'vanity',
39];
40
41class DiscordServerSearchScraper {
42 normalizeFilterValue(value) {
43 if (value === undefined || value === null) return '';
44 return String(value).trim();
45 }
46
47 buildFilters(input) {
48 const filters = {};
49
50 for (const field of FILTER_FIELDS) {
51 filters[field] = this.normalizeFilterValue(input[field]);
52 }
53
54 return filters;
55 }
56
57 buildQueryParams(keyword, page) {
58 const params = new URLSearchParams({
59 q: keyword,
60 ...this.filters,
61 });
62
63 if (page) {
64 params.set('page', String(page));
65 }
66
67 return params;
68 }
69
70 buildSearchUrl(keyword) {
71 return `${SITE_BASE}/servers?${this.buildQueryParams(keyword).toString()}`;
72 }
73
74 buildApiUrl(keyword, page) {
75 return `${API_BASE}?${this.buildQueryParams(keyword, page).toString()}`;
76 }
77
78 getAxiosConfig(proxyUrl) {
79 if (!proxyUrl) return {};
80
81 const parsed = new URL(proxyUrl);
82 return {
83 proxy: {
84 protocol: parsed.protocol.replace(':', ''),
85 host: parsed.hostname,
86 port: Number(parsed.port),
87 ...(parsed.username
88 ? {
89 auth: {
90 username: decodeURIComponent(parsed.username),
91 password: decodeURIComponent(parsed.password),
92 },
93 }
94 : {}),
95 },
96 };
97 }
98
99 parseNumber(text) {
100 if (!text) return null;
101 const value = Number(String(text).replace(/,/g, '').trim());
102 return Number.isFinite(value) ? value : null;
103 }
104
105 parseServers(html, keyword) {
106 const $ = cheerio.load(`<table>${html}</table>`);
107 const results = [];
108
109 $('tr').each((_, element) => {
110 const row = $(element);
111 const href = row.find('a.row-link').attr('href') || '';
112 const serverId = href.split('/').pop() || null;
113 const traits = [];
114
115 row.find('[data-col="traits"] .trait-chip').each((__, chip) => {
116 traits.push($(chip).text().trim());
117 });
118
119 results.push({
120 searchKeyword: keyword,
121 serverId,
122 name: row.find('a.row-link').text().trim() || null,
123 icon: row.find('img.icon-xs').attr('src') || null,
124 clanTag: row.find('a.user-tag span').text().trim() || null,
125 clanBadge: row.find('a.user-tag img').attr('src') || null,
126 vanityUrl: row.find('.row-sub').text().trim().replace(/^\//, '') || null,
127 members: this.parseNumber(row.find('[data-col="members"]').text()),
128 online: this.parseNumber(row.find('[data-col="online"]').text()),
129 createdAt: row.find('[data-col="created"] .date-full').text().trim() || null,
130 boosts: this.parseNumber(row.find('[data-col="boosts"]').text()),
131 tagUsers: this.parseNumber(row.find('[data-col="tag_users"]').text()),
132 addedAt: row.find('[data-col="added"] .date-full').text().trim() || null,
133 checkedAt: row.find('[data-col="checked"] .date-full').text().trim() || null,
134 traits,
135 inviteUrl: row.find('a.invite-chip').attr('href') || null,
136 inviteCode: row.find('a.invite-chip').text().trim() || null,
137 scrapedAt: new Date().toISOString(),
138 });
139 });
140
141 return results;
142 }
143
144 async run(input) {
145 const { keywords, maxItems = Infinity, maxPages = Infinity, proxyConfiguration } = input;
146
147 if (!Array.isArray(keywords) || keywords.length === 0) {
148 throw new Error('Input must include a non-empty keywords array');
149 }
150
151 this.maxItems = maxItems;
152 this.maxPages = maxPages;
153 this.filters = this.buildFilters(input);
154
155 const proxyConfig = proxyConfiguration
156 ? await Actor.createProxyConfiguration(proxyConfiguration)
157 : undefined;
158
159 for (const rawKeyword of keywords) {
160 const keyword = String(rawKeyword || '').trim();
161 if (!keyword) continue;
162
163 const proxyUrl = proxyConfig ? await proxyConfig.newUrl() : undefined;
164 const axiosConfig = this.getAxiosConfig(proxyUrl);
165
166 await this.scrapeKeyword(keyword, axiosConfig);
167 await this.randomDelay(500, 1500);
168 }
169 }
170
171 async scrapeKeyword(keyword, axiosConfig) {
172 const searchUrl = this.buildSearchUrl(keyword);
173 const seenIds = new Set();
174 let page = 1;
175 let totalSaved = 0;
176
177 console.log(`Searching Discord servers for "${keyword}"...`);
178
179 while (page <= this.maxPages && totalSaved < this.maxItems) {
180 const apiUrl = this.buildApiUrl(keyword, page);
181 console.log(`Fetching page ${page} for "${keyword}"...`);
182
183 let html;
184 try {
185 const response = await axios.get(apiUrl, {
186 ...axiosConfig,
187 headers: {
188 ...DEFAULT_HEADERS,
189 'hx-current-url': searchUrl,
190 Referer: searchUrl,
191 },
192 timeout: 60000,
193 });
194 html = response.data;
195 } catch (error) {
196 const message = error.response?.data?.message || error.message;
197 console.error(`API request failed for "${keyword}" page ${page}:`, message);
198 if (totalSaved === 0) {
199 await Actor.pushData([
200 {
201 searchKeyword: keyword,
202 error: message,
203 scrapedAt: new Date().toISOString(),
204 },
205 ]);
206 }
207 break;
208 }
209
210 const servers = this.parseServers(html, keyword);
211 if (servers.length === 0) {
212 console.log(`No more servers on page ${page}`);
213 break;
214 }
215
216 const currentData = [];
217 for (const server of servers) {
218 if (totalSaved >= this.maxItems) break;
219
220 const dedupeKey = server.serverId || `${server.name}-${server.inviteUrl}`;
221 if (seenIds.has(dedupeKey)) continue;
222 seenIds.add(dedupeKey);
223
224 currentData.push(server);
225 totalSaved++;
226 }
227
228 if (currentData.length > 0) {
229 console.log(`Saved ${currentData.length} servers from page ${page}`);
230 await Actor.pushData(currentData);
231 }
232
233 if (totalSaved >= this.maxItems) break;
234
235 page++;
236 await this.randomDelay(500, 1200);
237 }
238
239 console.log(`Finished "${keyword}" with ${totalSaved} servers`);
240 }
241
242 async randomDelay(min = 1000, max = 3000) {
243 const delay = Math.floor(Math.random() * (max - min + 1) + min);
244 await new Promise((resolve) => setTimeout(resolve, delay));
245 }
246}
247
248await Actor.init();
249
250Actor.main(async () => {
251 const input = await Actor.getInput();
252
253
254
255
256
257
258 const scraper = new DiscordServerSearchScraper();
259 await scraper.run(input);
260});
