1import { Actor } from 'apify';
2import { connect as connectRealBrowser } from 'puppeteer-real-browser';
3
4const SITE_BASE = 'https://telegramchannels.me';
5
6const SCRAPER_CONFIG = {
7 turnstile: true,
8 headless: false,
9};
10
11class TelegramChannelsSearchScraper {
12 async getPageStats(page) {
13 try {
14 return await page.evaluate(() => {
15 const body = document.body;
16 const bodyText = body?.innerText || '';
17
18 return {
19 title: document.title || '',
20 bodyText,
21 cardCount: document.querySelectorAll('a.index-card').length,
22 hasPagination: Boolean(document.querySelector('.index-pagination-current')),
23 ready: Boolean(body),
24 error: null,
25 };
26 });
27 } catch (error) {
28 return {
29 title: '',
30 bodyText: '',
31 cardCount: 0,
32 hasPagination: false,
33 ready: false,
34 error: error.message,
35 };
36 }
37 }
38
39 isCloudflareChallenge(title = '', bodyText = '') {
40 return (
41 title.includes('Just a moment') ||
42 title.includes('Attention Required') ||
43 title.includes('Cloudflare') ||
44 /security verification/i.test(bodyText)
45 );
46 }
47
48 isPageContentReady(stats) {
49 return stats.cardCount > 0 || stats.hasPagination;
50 }
51
52 async waitForPageReady(page, timeoutMs = 30000) {
53 const start = Date.now();
54
55 while (Date.now() - start < timeoutMs) {
56 const stats = await this.getPageStats(page);
57 if (stats.ready && !stats.error) {
58 return stats;
59 }
60
61 await this.randomDelay(500, 1000);
62 }
63
64 return this.getPageStats(page);
65 }
66
67 buildProxyOptions(proxyUrl) {
68 if (!proxyUrl) return undefined;
69
70 try {
71 const parsed = new URL(proxyUrl);
72 if (!parsed.hostname || !parsed.port) return undefined;
73
74 return {
75 host: parsed.hostname,
76 port: Number(parsed.port),
77 username: parsed.username || undefined,
78 password: parsed.password || undefined,
79 };
80 } catch (error) {
81 console.warn('Invalid proxy configuration');
82 return undefined;
83 }
84 }
85
86 buildSearchUrl(keyword, page = 1) {
87 const params = new URLSearchParams({ q: keyword });
88 if (page > 1) {
89 params.set('page', String(page));
90 }
91 return `${SITE_BASE}/index?${params.toString()}`;
92 }
93
94 async run(input) {
95 const { keyword, maxItems = 100, proxyConfiguration } = input;
96 const searchKeyword = String(keyword || '').trim();
97 const itemLimit = Number(maxItems);
98
99 if (!searchKeyword) {
100 throw new Error('Input must include a keyword');
101 }
102
103 if (!Number.isFinite(itemLimit) || itemLimit < 1) {
104 throw new Error('maxItems must be a positive number');
105 }
106
107 const proxyConfig = proxyConfiguration
108 ? await Actor.createProxyConfiguration(proxyConfiguration)
109 : undefined;
110
111 const maxAttempts = 2;
112
113 for (let attempt = 1; attempt <= maxAttempts; attempt++) {
114 const proxyUrl = proxyConfig ? await proxyConfig.newUrl() : undefined;
115 const proxyOptions = this.buildProxyOptions(proxyUrl);
116
117 const realBrowserOption = {
118 args: ['--start-maximized', '--no-sandbox'],
119 turnstile: SCRAPER_CONFIG.turnstile,
120 headless: SCRAPER_CONFIG.headless,
121 customConfig: {},
122 connectOption: {
123 defaultViewport: { width: 1400, height: 900 },
124 },
125 ...(proxyOptions ? { proxy: proxyOptions } : {}),
126 plugins: [],
127 };
128
129 let browser;
130 try {
131 const { page, browser: connectedBrowser } = await connectRealBrowser(realBrowserOption);
132 browser = connectedBrowser;
133
134 await page.setDefaultNavigationTimeout(180000);
135 await page.setDefaultTimeout(180000);
136
137 await this.scrapeKeyword(page, searchKeyword, itemLimit);
138 return;
139 } catch (error) {
140 const message = error?.message || String(error);
141 const retryable = message.includes('Verification timed out');
142
143 if (!retryable || attempt === maxAttempts) {
144 throw error;
145 }
146
147 console.log(`Attempt ${attempt} failed (${message}), retrying with a fresh browser session...`);
148 } finally {
149 if (browser) {
150 await browser.close();
151 }
152 }
153
154 await this.randomDelay(2000, 4000);
155 }
156 }
157
158 async safeGoto(page, url, options = {}) {
159 const opts = { waitUntil: 'domcontentloaded', timeout: 180000, ...options };
160
161 for (let attempt = 1; attempt <= 3; attempt++) {
162 try {
163 await page.goto(url, opts);
164 return;
165 } catch (error) {
166 const message = error?.message || String(error);
167 const abortable = message.includes('ERR_ABORTED') || message.includes('net::ERR_');
168 if (!abortable || attempt === 3) {
169 throw error;
170 }
171
172 await this.randomDelay(1500, 2500);
173 }
174 }
175 }
176
177 async scrapeKeyword(page, keyword, maxItems) {
178 console.log(`Searching channels for "${keyword}"...`);
179
180 await this.safeGoto(page, SITE_BASE).catch(() => {});
181 await this.waitForCloudflare(page);
182 await this.randomDelay(1500, 2500);
183
184 let pageNumber = 1;
185 let totalSaved = 0;
186 const seen = new Set();
187
188 while (totalSaved < maxItems) {
189 const searchUrl = this.buildSearchUrl(keyword, pageNumber);
190
191 await this.safeGoto(page, searchUrl);
192 await this.waitForCloudflare(page);
193 await this.waitForResults(page);
194 await this.scrollResults(page);
195
196 const pageMeta = await this.getPageMeta(page);
197 const items = await this.extractChannels(page, keyword);
198
199 if (items.length === 0) {
200 break;
201 }
202
203 const results = [];
204 for (const item of items) {
205 if (totalSaved >= maxItems) break;
206
207 const key = `${item.title}::${item.openUrl || ''}`;
208 if (seen.has(key)) continue;
209 seen.add(key);
210
211 results.push({
212 ...item,
213 scrapedAt: new Date().toISOString(),
214 });
215 totalSaved++;
216 }
217
218 if (results.length > 0) {
219 await Actor.pushData(results);
220 console.log(`Saved ${totalSaved} channels for "${keyword}"`);
221 }
222
223 if (totalSaved >= maxItems) {
224 break;
225 }
226
227 const hasNextPage = pageMeta.currentPage < pageMeta.totalPages;
228 if (!hasNextPage) {
229 break;
230 }
231
232 pageNumber++;
233 await this.randomDelay(1500, 2500);
234 }
235
236 if (totalSaved === 0) {
237 console.log(`No channels found for "${keyword}"`);
238 await Actor.pushData([
239 {
240 searchKeyword: keyword,
241 error: 'No channels found',
242 scrapedAt: new Date().toISOString(),
243 },
244 ]);
245 return;
246 }
247 }
248
249 async waitForCloudflare(page, timeoutMs = 180000) {
250 const start = Date.now();
251 let sawChallenge = false;
252 let lastReloadAt = 0;
253
254 while (Date.now() - start < timeoutMs) {
255 try {
256 const stats = await this.getPageStats(page);
257
258 if (stats.error) {
259 await this.randomDelay(1000, 2000);
260 continue;
261 }
262
263 if (this.isPageContentReady(stats)) {
264 if (sawChallenge) {
265 const seconds = Math.round((Date.now() - start) / 1000);
266 console.log(`Security check — passed (${seconds}s)`);
267 }
268 await this.waitForPageReady(page);
269 return;
270 }
271
272 if (!this.isCloudflareChallenge(stats.title, stats.bodyText)) {
273 if (sawChallenge) {
274 const seconds = Math.round((Date.now() - start) / 1000);
275 console.log(`Security check — passed (${seconds}s)`);
276 }
277 await this.waitForPageReady(page);
278 return;
279 }
280
281 if (!sawChallenge) {
282 sawChallenge = true;
283 console.log('Security check — waiting...');
284 }
285
286 const elapsed = Date.now() - start;
287 if (elapsed - lastReloadAt > 45000) {
288 console.log('Security check — reloading page...');
289 try {
290 await page.reload({ waitUntil: 'domcontentloaded', timeout: 180000 });
291 } catch {
292
293 }
294 lastReloadAt = elapsed;
295 }
296 } catch {
297
298 }
299
300 await this.randomDelay(1000, 2000);
301 }
302
303 throw new Error('Verification timed out');
304 }
305
306 async waitForResults(page, timeoutMs = 90000) {
307 const start = Date.now();
308 let previousCount = -1;
309 let stableRounds = 0;
310
311 while (Date.now() - start < timeoutMs) {
312 const stats = await this.getPageStats(page);
313
314 if (stats.error) {
315 await this.randomDelay(500, 800);
316 continue;
317 }
318
319 if (stats.cardCount > 0) {
320 if (stats.cardCount === previousCount) {
321 stableRounds++;
322 if (stableRounds >= 2) {
323 await this.randomDelay(800, 1200);
324 return;
325 }
326 } else {
327 stableRounds = 0;
328 previousCount = stats.cardCount;
329 }
330 }
331
332 await this.randomDelay(500, 800);
333 }
334
335 throw new Error('Search results did not load');
336 }
337
338 async scrollResults(page) {
339 let previousCount = 0;
340 let stableRounds = 0;
341
342 for (let i = 0; i < 10; i++) {
343 const count = await page.evaluate(() => {
344 window.scrollTo(0, document.body.scrollHeight);
345 return document.querySelectorAll('a.index-card').length;
346 });
347
348 if (count === previousCount) {
349 stableRounds++;
350 if (stableRounds >= 2) {
351 break;
352 }
353 } else {
354 stableRounds = 0;
355 previousCount = count;
356 }
357
358 await this.randomDelay(800, 1200);
359 }
360 }
361
362 async getPageMeta(page) {
363 return page.evaluate(() => {
364 const pageText = document.querySelector('.index-pagination-current')?.textContent || '';
365 const match = pageText.match(/Page\s+(\d+)\s*\/\s*(\d+)/i);
366
367 return {
368 currentPage: match ? Number(match[1]) : 1,
369 totalPages: match ? Number(match[2]) : 1,
370 };
371 });
372 }
373
374 async extractChannels(page, keyword) {
375 return page.evaluate((searchKeyword) => {
376 const parseOpenUrl = (onclick) => {
377 if (!onclick) return null;
378
379 const indexOpenMatch = onclick.match(/indexOpen\(event,'([^']+)'\)/);
380 if (indexOpenMatch) return indexOpenMatch[1];
381
382 const gowMatch = onclick.match(/gow\(this,'([^']+)'\)/);
383 if (gowMatch) return gowMatch[1];
384
385 return null;
386 };
387
388 const parseTags = (card) => {
389 const tags = [...card.querySelectorAll('.index-card-tag')].map((tag) =>
390 tag.textContent?.replace(/\s+/g, ' ').trim(),
391 );
392
393 let type = null;
394 let subscribers = null;
395 let language = null;
396
397 for (const tag of tags) {
398 const lower = tag.toLowerCase();
399 if (['channel', 'group', 'bot'].includes(lower)) {
400 type = lower;
401 } else if (/^[a-z]{2}$/i.test(tag)) {
402 language = tag.toLowerCase();
403 } else if (tag) {
404 subscribers = tag;
405 }
406 }
407
408 return { type, subscribers, language };
409 };
410
411 return [...document.querySelectorAll('a.index-card')].map((card) => {
412 const { type, subscribers, language } = parseTags(card);
413 const avatarImg = card.querySelector('.index-card-avatar img');
414
415 return {
416 searchKeyword,
417 title: card.querySelector('.index-card-title')?.textContent?.trim() || null,
418 description: card.querySelector('.index-card-desc')?.textContent?.trim() || null,
419 type,
420 subscribers,
421 language,
422 verified: Boolean(card.querySelector('.index-card-verified')),
423 sponsored: card.classList.contains('index-card-sponsored'),
424 avatarUrl: avatarImg?.src || null,
425 openUrl: parseOpenUrl(card.getAttribute('onclick')),
426 };
427 });
428 }, keyword);
429 }
430
431 async randomDelay(min = 500, max = 1500) {
432 const delay = Math.floor(Math.random() * (max - min + 1) + min);
433 await new Promise((resolve) => setTimeout(resolve, delay));
434 }
435}
436
437await Actor.init();
438
439Actor.main(async () => {
440 const input = await Actor.getInput();
441 const scraper = new TelegramChannelsSearchScraper();
442 await scraper.run(input);
443});