1
2
3
4
5
6
7import { Actor } from 'apify';
8import { PlaywrightCrawler, createPlaywrightRouter } from 'crawlee';
9
10
11const EMAIL_REGEX = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
12
13
14const IMPORTANT_SELECTORS = [
15 'footer', '.footer', '#footer', '[class*="footer"]', '[id*="footer"]',
16 '.contact', '#contact', '[class*="contact"]', '[id*="contact"]',
17 '.legal', '.mentions', '.copyright', '.about', '.team', '.staff',
18 '[class*="legal"]', '[class*="mention"]', '[class*="copyright"]',
19 '[class*="about"]', '[class*="team"]', '[class*="staff"]',
20 '.qui-sommes-nous', '.equipe', '.équipe', '.coordonnees', '.coordonnées'
21];
22
23
24const IMPORTANT_LINK_PATTERNS = [
25 /contact/i, /mention/i, /legal/i, /about/i, /team/i, /staff/i,
26 /qui.*sommes/i, /équipe/i, /equipe/i, /cgv/i, /cgu/i,
27 /politique/i, /confidentialité/i, /confidentialite/i, /privacy/i,
28 /terms/i, /conditions/i, /coordonnees/i, /coordonnées/i
29];
30
31
32interface CrawlerInput {
33 startUrls: any[];
34 maxRequestsPerCrawl: number;
35 maxDepth?: number;
36 maxPagesPerDomain?: number;
37 followImportantLinks?: boolean;
38 excludePatterns?: string[];
39}
40
41
42interface ImportantLink {
43 url: string;
44 text: string;
45}
46
47
48function validateAndCleanEmails(emails: string[], excludePatterns: string[]): string[] {
49 return emails
50 .map(email => email.trim().toLowerCase())
51 .filter(email => {
52 if (!email) return false;
53
54
55 const hasImageExtension = excludePatterns.some(pattern =>
56 email.includes(pattern.toLowerCase())
57 );
58
59
60 const isValidFormat = /^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$/.test(email);
61
62
63 const isGeneric = [
64 'example.com', 'test.com', 'domain.com', 'yoursite.com',
65 'website.com', 'noreply@', 'no-reply@', 'placeholder',
66 'dummy', 'fake', 'sample'
67 ].some(generic => email.includes(generic));
68
69 return !hasImageExtension && isValidFormat && !isGeneric;
70 });
71}
72
73
74function getDomainFromUrl(url: string): string {
75 try {
76 return new URL(url).hostname;
77 } catch {
78 return 'unknown';
79 }
80}
81
82
83const router = createPlaywrightRouter();
84
85router.addDefaultHandler(async ({ page, request, log, enqueueLinks }) => {
86 const url = request.url;
87 const depth = request.userData?.depth || 0;
88 const maxDepth = request.userData?.maxDepth || 1;
89 const domain = getDomainFromUrl(url);
90
91 log.info(`🔍 Traitement de: ${url} (profondeur: ${depth}, domaine: ${domain})`);
92
93 try {
94
95 await page.waitForLoadState('networkidle', { timeout: 10000 });
96
97
98 const foundEmails = await Actor.getValue('foundEmails') as string[] || [];
99 const visitedUrls = await Actor.getValue('visitedUrls') as string[] || [];
100 const domainEmails = await Actor.getValue('domainEmails') as Record<string, string[]> || {};
101
102
103 if (!visitedUrls.includes(url)) {
104 visitedUrls.push(url);
105 await Actor.setValue('visitedUrls', visitedUrls);
106 }
107
108
109 const emailsFromZones = await page.evaluate((selectors: string[]) => {
110 const emails: string[] = [];
111 const emailRegex = /[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g;
112
113 selectors.forEach(selector => {
114 try {
115 const elements = document.querySelectorAll(selector);
116 elements.forEach(element => {
117 const htmlElement = element as HTMLElement;
118 const text = element.textContent || htmlElement.innerText || '';
119 const html = element.innerHTML || '';
120
121
122 const textEmails = text.match(emailRegex) || [];
123
124 const htmlEmails = html.match(emailRegex) || [];
125
126 emails.push(...textEmails, ...htmlEmails);
127 });
128 } catch (e) {
129
130 }
131 });
132
133 return emails;
134 }, IMPORTANT_SELECTORS);
135
136
137 const mailtoEmails = await page.evaluate(() => {
138 const mailtoLinks = document.querySelectorAll('a[href^="mailto:"]');
139 return Array.from(mailtoLinks).map(link => {
140 const href = link.getAttribute('href');
141 return href ? href.replace('mailto:', '').split('?')[0] : '';
142 }).filter(email => email);
143 });
144
145
146 const allPageText = await page.evaluate(() => {
147 return document.body.innerText || document.body.textContent || '';
148 });
149 const allPageEmails = allPageText.match(EMAIL_REGEX) || [];
150
151
152 const allEmails = [...new Set([...emailsFromZones, ...mailtoEmails, ...allPageEmails])];
153
154
155 const validEmails = validateAndCleanEmails(allEmails, [
156 'jpg', 'png', 'pdf', 'gif', 'jpeg', 'svg', 'ico', 'webp'
157 ]);
158
159
160 const newEmails = validEmails.filter(email => !foundEmails.includes(email));
161
162 if (newEmails.length > 0) {
163 log.info(`✅ Trouvé ${newEmails.length} nouveaux emails sur ${url}: ${newEmails.join(', ')}`);
164
165
166 foundEmails.push(...newEmails);
167 await Actor.setValue('foundEmails', foundEmails);
168
169
170 if (!domainEmails[domain]) {
171 domainEmails[domain] = [];
172 }
173 domainEmails[domain].push(...newEmails);
174 await Actor.setValue('domainEmails', domainEmails);
175
176
177 await Actor.pushData({
178 url,
179 domain,
180 emails: newEmails,
181 emailCount: newEmails.length,
182 depth,
183 timestamp: new Date().toISOString(),
184 source: 'targeted_extraction'
185 });
186 } else {
187 log.info(`❌ Aucun nouvel email trouvé sur ${url}`);
188 }
189
190
191 if (depth < maxDepth) {
192 const importantLinks = await page.evaluate((patterns: RegExp[]) => {
193 const links = Array.from(document.querySelectorAll('a[href]'));
194 const importantLinks: ImportantLink[] = [];
195
196 links.forEach(link => {
197 const href = link.getAttribute('href');
198 const htmlElement = link as HTMLElement;
199 const text = link.textContent || htmlElement.innerText || '';
200 const title = link.getAttribute('title') || '';
201
202 if (href && (href.startsWith('http') || href.startsWith('/'))) {
203 const fullText = `${text} ${title} ${href}`.toLowerCase();
204
205
206 const isImportant = patterns.some((pattern: RegExp) => pattern.test(fullText));
207
208 if (isImportant) {
209 const fullUrl = href.startsWith('/') ?
210 new URL(href, window.location.origin).href : href;
211 importantLinks.push({
212 url: fullUrl,
213 text: text.trim()
214 });
215 }
216 }
217 });
218
219 return importantLinks;
220 }, IMPORTANT_LINK_PATTERNS);
221
222
223 for (const link of importantLinks.slice(0, 5)) {
224 if (!visitedUrls.includes(link.url)) {
225 log.info(`🔗 Ajout du lien important: ${link.url} (${link.text})`);
226
227 await enqueueLinks({
228 urls: [link.url],
229 userData: {
230 depth: depth + 1,
231 maxDepth,
232 source: url,
233 linkText: link.text
234 }
235 });
236 }
237 }
238 }
239
240 } catch (error) {
241 const errorMessage = error instanceof Error ? error.message : String(error);
242 log.error(`❌ Erreur lors du traitement de ${url}: ${errorMessage}`);
243 }
244});
245
246
247await Actor.main(async () => {
248 const input = await Actor.getInput<CrawlerInput>();
249
250 if (!input || !Array.isArray(input.startUrls) || input.startUrls.length === 0) {
251 throw new Error('La configuration "startUrls" est requise et doit être un tableau non-vide d\'URLs');
252 }
253
254
255 await Actor.setValue('foundEmails', [] as string[]);
256 await Actor.setValue('visitedUrls', [] as string[]);
257 await Actor.setValue('domainEmails', {} as Record<string, string[]>);
258
259 console.log('🚀 Démarrage du crawler avec les URLs suivantes:', input.startUrls);
260
261
262 const maxDepth = input.maxDepth || 2;
263 const maxPagesPerDomain = input.maxPagesPerDomain || 30;
264 const followImportantLinks = input.followImportantLinks !== false;
265
266
267 const crawler = new PlaywrightCrawler({
268 requestHandler: router,
269
270 launchContext: {
271 launchOptions: {
272 headless: true,
273 args: [
274 '--no-sandbox',
275 '--disable-setuid-sandbox',
276 '--disable-dev-shm-usage',
277 '--disable-web-security'
278 ]
279 },
280 },
281
282 maxRequestsPerCrawl: input.maxRequestsPerCrawl || maxPagesPerDomain,
283 navigationTimeoutSecs: 20,
284 maxConcurrency: 2,
285
286
287 failedRequestHandler: async ({ request, error }) => {
288 const errorMessage = error instanceof Error ? error.message : String(error);
289 console.error(`❌ Échec de traitement pour ${request.url}:`, errorMessage);
290 },
291 });
292
293
294 const initialRequests = input.startUrls.map((urlData) => {
295 const baseRequest = typeof urlData === 'string' ? { url: urlData } : urlData;
296 return {
297 ...baseRequest,
298 userData: {
299 depth: 0,
300 maxDepth,
301 followImportantLinks
302 }
303 };
304 });
305
306 try {
307
308 await crawler.run(initialRequests);
309
310
311 const foundEmails = await Actor.getValue('foundEmails') as string[] || [];
312 const visitedUrls = await Actor.getValue('visitedUrls') as string[] || [];
313 const domainEmails = await Actor.getValue('domainEmails') as Record<string, string[]> || {};
314
315
316 const finalResult = {
317 success: true,
318 totalEmailsFound: foundEmails.length,
319 emails: foundEmails.sort(),
320 emailsByDomain: domainEmails,
321 urlsVisited: visitedUrls.length,
322 visitedUrls: visitedUrls,
323 timestamp: new Date().toISOString()
324 };
325
326
327 await Actor.setValue('OUTPUT', finalResult);
328 await Actor.pushData(finalResult);
329
330 console.log(`🎉 Crawling terminé avec succès !`);
331 console.log(`📧 ${foundEmails.length} emails uniques trouvés`);
332 console.log(`📄 ${visitedUrls.length} pages visitées`);
333 console.log(`🌐 ${Object.keys(domainEmails).length} domaines analysés`);
334
335 } catch (error) {
336 console.error('❌ Erreur lors du crawling:', error);
337
338
339 const foundEmails = await Actor.getValue('foundEmails') as string[] || [];
340 const visitedUrls = await Actor.getValue('visitedUrls') as string[] || [];
341 const domainEmails = await Actor.getValue('domainEmails') as Record<string, string[]> || {};
342
343 const errorMessage = error instanceof Error ? error.message : String(error);
344
345 const finalResult = {
346 success: false,
347 totalEmailsFound: foundEmails.length,
348 emails: foundEmails.sort(),
349 emailsByDomain: domainEmails,
350 urlsVisited: visitedUrls.length,
351 visitedUrls: visitedUrls,
352 error: errorMessage,
353 timestamp: new Date().toISOString()
354 };
355
356 await Actor.setValue('OUTPUT', finalResult);
357 await Actor.pushData(finalResult);
358
359 console.log('⚠️ Traitement terminé avec erreur, résultats partiels sauvegardés.');
360 }
361});