1"""
2This module defines the `main()` coroutine for the Apify Actor, executed from the `__main__.py` file.
3
4Feel free to modify this file to suit your specific needs.
5
6To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:
7https://docs.apify.com/sdk/python
8"""
9
10from urllib.parse import urljoin
11
12from apify import Actor
13import re
14from urllib.parse import urljoin
15from bs4 import BeautifulSoup
16import requests
17import requests.exceptions
18import time
19import random
20from urllib.parse import quote_plus
21import urllib
22import os
23import sys
24import asyncio
25from urllib.parse import urlparse, urlsplit, urlunsplit, parse_qs, urlencode
26import datetime
27from .datascrapify import StartProcess,on_aborting_event,parse_proxy_url,Insertapifylog
28from .playwritescraper import callpython
29import threading
30import requests
31from concurrent.futures import ThreadPoolExecutor, as_completed
32all_users = []
33MAX_WORKERS = 4
34REQUEST_TIMEOUT = 30
35
36camp_trial="user-trials-linkedinemail"
37
38async def check_free_trial(user_id: str, is_paid_user: bool = False):
39 """
40 Returns True if the user is allowed to use the Actor.
41
42 Paid users: always allowed.
43 Free users: allowed for 24 hours from first use.
44 """
45
46
47 if is_paid_user:
48 return True
49
50 store = await Actor.open_key_value_store(name=camp_trial)
51
52 trial_key = f"trial_start_{user_id}"
53
54 now = datetime.datetime.now(datetime.timezone.utc)
55
56 first_use = await store.get_value(trial_key)
57
58
59 if not first_use:
60 await store.set_value(
61 trial_key,
62 now.isoformat()
63 )
64
65 Actor.log.info(
66 f"Free trial started for user {user_id}. "
67 f"Trial valid for 24 hours."
68 )
69
70 return True
71
72
73 first_use_date = datetime.datetime.fromisoformat(first_use)
74
75
76 if first_use_date.tzinfo is None:
77 first_use_date = first_use_date.replace(
78 tzinfo=datetime.timezone.utc
79 )
80
81 elapsed = now - first_use_date
82
83 if elapsed < datetime.timedelta(hours=24):
84 remaining = datetime.timedelta(hours=24) - elapsed
85
86 hours = int(remaining.total_seconds() // 3600)
87 minutes = int((remaining.total_seconds() % 3600) // 60)
88
89 Actor.log.info(
90 f"Free trial active. "
91 f"Remaining: {hours}h {minutes}m"
92 )
93
94 return True
95
96 Actor.log.warning(
97 f"Free trial expired for user {user_id}"
98 )
99
100 return False
101
102
103email_pattern = re.compile(
104 r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}'
105)
106
107
108email_lock = threading.Lock()
109stop_event = threading.Event()
110
111existing_emails = {
112 item['Email'].lower()
113 for item in all_users
114 if item.get('Email')
115}
116
117result_count = 0
118
119
120def get_email_queries(Email_Type_val,Other_Email_Type_val):
121
122 queries = [
123 "email @gmail.com",
124 "@gmail.com",
125 "@hotmail.com",
126 "email @hotmail.com",
127 "@icloud.com",
128 "email @icloud.com",
129 "@yahoo.com",
130 "email @yahoo.com"
131 ]
132
133 if Email_Type_val == "1":
134
135 if "@" in Other_Email_Type_val:
136 queries = [Other_Email_Type_val + " "]
137 else:
138 queries = ["@" + Other_Email_Type_val]
139
140 return queries
141
142
143def save_email(email, title, alltext, detail_url,Limit_val):
144
145 global result_count
146
147 email = email.lower().strip()
148
149 with email_lock:
150
151 if stop_event.is_set():
152 return False
153
154 if email in existing_emails:
155 return False
156
157 existing_emails.add(email)
158
159 all_users.append({
160 'Email': email
161 })
162
163 result_count += 1
164
165 if Limit_val and Limit_val != "0":
166
167 if result_count >= int(Limit_val):
168 stop_event.set()
169
170
171
172
173
174 return True
175
176
177def process_query(
178 query_item,
179 query,
180 proxyurl,
181 max_pages,
182 social_network_val,
183 Limit_val
184):
185
186 newquery = query.replace(
187 '@gmail.com',
188 query_item
189 )
190
191
192
193 page = 0
194 try_count = 0
195
196 session = requests.Session()
197
198 headers = {
199 "User-Agent": (
200 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
201 "AppleWebKit/537.36 (KHTML, like Gecko) "
202 "Chrome/120.0.0.0 Safari/537.36"
203 )
204 }
205
206 while page < max_pages:
207
208 if stop_event.is_set():
209 break
210 Actor.log.info(
211 f"Starting query: {newquery} : Page : {page}"
212 )
213 start = page * 10
214
215 search_url = (
216 "http://www.google.com/search?q="
217 + quote_plus(newquery)
218 + f"&num=10&hl=en&start={start}"
219 )
220
221 try:
222
223 proxies = None
224
225 if proxyurl:
226 proxies = {
227 'http': proxyurl,
228 'https': proxyurl
229 }
230
231 response = session.get(
232 search_url,
233 proxies=proxies,
234 headers=headers
235 )
236
237 if response.status_code != 200:
238
239 Actor.log.warning(
240 f"Request failed: {response.status_code}"
241 )
242
243
244 break
245
246 soup = BeautifulSoup(
247 response.text,
248 "html.parser"
249 )
250
251 if social_network_val in (
252 "youtube.com/",
253 "tiktok.com/",
254 "instagram.com/"
255 ):
256
257 result_blocks = soup.select(
258 "div.MjjYud:has(h3)"
259 )
260
261 else:
262
263 result_blocks = soup.select(
264 "div.tF2Cxc:has(h3)"
265 )
266
267 if not result_blocks:
268
269 try_count += 1
270
271 Actor.log.warning(
272 f"No results: {newquery}, page {page}"
273 )
274
275 if try_count >= 3:
276 break
277
278 time.sleep(1)
279 continue
280
281 try_count = 0
282
283 page += 1
284
285 for block in result_blocks:
286
287 if stop_event.is_set():
288 break
289
290 title_el = block.select_one("h3")
291 link_el = block.select_one("a")
292
293 snippet_el = block.select_one(
294 ".VwiC3b, .IsZvec, .aCOpRe"
295 )
296
297 if not title_el or not link_el:
298 continue
299
300 title = title_el.get_text(strip=True)
301
302 detail_url = link_el.get("href", "")
303
304 snippet = (
305 snippet_el.get_text(strip=True)
306 if snippet_el else ""
307 )
308
309 alltext = title + " " + snippet
310
311 matches = email_pattern.findall(alltext)
312
313 if not matches:
314 continue
315
316 for email in matches:
317
318 if stop_event.is_set():
319 break
320
321 saved = save_email(
322 email,
323 title,
324 alltext,
325 detail_url,
326 Limit_val
327 )
328
329 if saved:
330
331 Actor.log.info(
332 f"Email found: {email}"
333 )
334
335
336
337
338 result_queue.put({
339 'Email': email.lower(),
340 'title': title,
341 'Description': alltext,
342 'Detail_Link': detail_url
343 })
344
345 if not stop_event.is_set():
346
347 time.sleep(
348 random.uniform(1.5, 3.0)
349 )
350
351 except requests.RequestException as e:
352
353 Actor.log.warning(
354 f"Request error [{newquery}]: {e}"
355 )
356
357 break
358
359 except Exception as e:
360
361 Actor.log.error(
362 f"Query error [{newquery}]: {e}"
363 )
364
365 break
366
367 session.close()
368
369 Actor.log.info(
370 f"Completed query: {newquery}"
371 )
372
373
374
375import queue
376
377result_queue = queue.Queue()
378
379
380async def run_multiple_queries(
381 query,
382 Email_Type_val,
383 Other_Email_Type_val,
384 proxyurl,
385 max_pages,
386 social_network_val,
387 proxy_configuration,
388 Limit_val
389):
390
391 queries = get_email_queries(
392 Email_Type_val,
393 Other_Email_Type_val
394 )
395
396 with ThreadPoolExecutor(max_workers=4) as executor:
397 proxyurl =await proxy_configuration.new_url()
398 if proxy_configuration:
399 proxyurl =await proxy_configuration.new_url()
400 print(proxyurl)
401 futures = [
402 executor.submit(
403 process_query,
404 item,
405 query,
406 proxyurl,
407 max_pages,
408 social_network_val,
409 Limit_val
410 )
411 for item in queries
412 ]
413
414 while any(not f.done() for f in futures) or not result_queue.empty():
415
416 try:
417 data = result_queue.get(timeout=0.2)
418 await Actor.push_data(data)
419
420 except queue.Empty:
421 pass
422
423 for future in as_completed(futures):
424 future.result()
425
426 while not result_queue.empty():
427 await Actor.push_data(result_queue.get())
428
429 Actor.log.info(f"Total new emails: {result_count}")
430
431
432
433
434
435
436
437
438
439
440def scrape_contact_emails(link):
441 res = requests.get(link)
442 domain = link.split(".")
443 mailaddr = link
444 soup = BeautifulSoup(res.text,"lxml")
445 links = soup.find_all("a")
446 contact_link = ''
447 final_result = ""
448 try:
449
450 emails = soup.find_all(text=re.compile('.*@'+domain[1]+'.'+domain[2].replace("/","")))
451 emails.sort(key=len)
452 print(emails[0].replace("\n",""))
453 final_result = emails[0]
454 except:
455
456 try:
457 flag = 0
458 for link in links:
459 if "contact" in link.get("href") or "Contact" in link.get("href") or "CONTACT" in link.get("href") or 'contact' in link.text or 'Contact' in link.text or 'CONTACT' in link.text:
460 if len(link.get("href"))>2 and flag<2:
461 flag = flag + 1
462 contact_link = link.get("href")
463
464 except:
465 pass
466
467 domain = domain[0]+"."+domain[1]+"."+domain[2]
468 if(len(contact_link)<len(domain)):
469 domain = domain+contact_link.replace("/","")
470 else:
471 domain = contact_link
472
473 try:
474
475 res = requests.get(domain)
476 soup = BeautifulSoup(res.text,"lxml")
477 emails = soup.find_all(text=re.compile('.*@'+mailaddr[7:].replace("/","")))
478 emails.sort(key=len)
479 try:
480 print(emails[0].replace("\n",""))
481 final_result = emails[0]
482 return final_result
483 except:
484 pass
485 except Exception as e:
486 pass
487
488 return ""
489
490
491
492async def main() -> None:
493 """
494 The main coroutine is being executed using `asyncio.run()`, so do not attempt to make a normal function
495 out of it, it will not work. Asynchronous execution is required for communication with Apify platform,
496 and it also enhances performance in the field of web scraping significantly.
497 """
498 async with Actor:
499
500 actor_input = await Actor.get_input() or {}
501 Keyword_val = actor_input.get('Keyword')
502 location_val = actor_input.get('location')
503 social_network_val = actor_input.get('social_network')
504 Country_val = actor_input.get('Country')
505 print(Country_val)
506 Email_Type_val = actor_input.get('Email_Type')
507 Other_Email_Type_val = actor_input.get('Other_Email_Type')
508 Limit_val =actor_input.get('Limit')
509 proxy_settings = actor_input.get('proxySettings')
510 print(proxy_settings)
511 proxy_configuration_selected = await Actor.create_proxy_configuration(actor_proxy_input=proxy_settings)
512 proxyurl_selected =None
513 if proxy_configuration_selected and proxy_settings:
514 proxyurl_selected =await proxy_configuration_selected.new_url()
515 print(proxyurl_selected)
516 if Limit_val:
517 print('ok')
518 else:
519 Limit_val=0
520 proxy_configuration = await Actor.create_proxy_configuration(groups=['GOOGLE_SERP'])
521
522 proxyurl =await proxy_configuration.new_url()
523 if proxy_configuration and proxy_settings:
524 proxyurl =await proxy_configuration.new_url()
525 print(proxyurl)
526
527
528
529 if not Keyword_val:
530 Actor.log.info('Please insert keyword')
531 await Actor.push_data({'Email': 'Please insert keyword'})
532 await Actor.exit()
533 return
534
535 if Keyword_val=='TestKeyword':
536 await Actor.push_data({'Email': 'Please insert Your Keyword'})
537 await Actor.exit()
538
539
540
541
542
543
544
545
546 is_paying = os.getenv("APIFY_USER_IS_PAYING")
547 print(is_paying)
548 if is_paying=="1":
549 print('Paid User')
550 else:
551 Limit_val=0
552 is_paid_user=False
553 APIFY_USER_ID = os.getenv("APIFY_USER_ID")
554 allowed = await check_free_trial(user_id=APIFY_USER_ID, is_paid_user=is_paid_user)
555 if not allowed:
556 await Actor.push_data({'Email': 'Free User can run for One Day Only. Please Subcribe'})
557 return
558
559
560
561
562
563
564
565
566
567
568
569 l1 = ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"]
570 l2= ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"]
571 select_index=1
572 select_country='United States'
573 for count, ele in enumerate(l1):
574 if(ele==Country_val):
575 select_index=count
576 break
577
578
579
580 for count, ele in enumerate(l2):
581 if(count==select_index):
582 select_country=ele
583 break
584
585 print(select_country)
586
587 concatstring = ""
588 concatstring = concatstring + Keyword_val
589 option = " @gmail.com ";
590 if Email_Type_val=="1":
591 if not Other_Email_Type_val:
592 Actor.log.info('Please insert Email Type Domain')
593 await Actor.push_data({'Email': 'Please insert Email Type Domain'})
594 await Actor.exit()
595 return
596 if Other_Email_Type_val.find("@") > -1:
597 option = " ( " + Other_Email_Type_val + " )"
598 elif Email_Type=="0":
599 option = " ( @" + Other_Email_Type_val + " )"
600 if location_val:
601 concatstring = concatstring+ " in "+ location_val
602 concatstring = concatstring + option
603 if social_network_val:
604 concatstring = concatstring + " site:"
605 if social_network_val == "linkedin.com/" or social_network_val == "pinterest.com/" :
606 concatstring = concatstring + Country_val + ".";
607
608 if social_network_val == "amazon.com/" :
609 if Country_val=='gm':
610 Country_val='de'
611 elif Country_val=='sp':
612 Country_val='es'
613 elif Country_val=='fr':
614 Country_val='fr'
615 elif Country_val=='uk':
616 Country_val='co.uk'
617 elif Country_val=='as':
618 Country_val='com.au'
619 elif Country_val=='www':
620 Country_val='com'
621 elif Country_val=='in':
622 Country_val='in'
623 elif Country_val=='be':
624 Country_val='com.be'
625 elif Country_val=='br':
626 Country_val='com.br'
627 elif Country_val=='ca':
628 Country_val='ca'
629 elif Country_val=='ch':
630 Country_val='cn'
631 elif Country_val=='eg':
632 Country_val='eg'
633 elif Country_val=='it':
634 Country_val='it'
635 elif Country_val=='ja':
636 Country_val='co.jp'
637 elif Country_val=='mx':
638 Country_val='com.mx'
639 elif Country_val=='nl':
640 Country_val='nl'
641 elif Country_val=='pl':
642 Country_val='pl'
643 elif Country_val=='sa':
644 Country_val='sa'
645 elif Country_val=='sn':
646 Country_val='sg'
647 elif Country_val=='sw':
648 Country_val='se'
649 elif Country_val=='tu':
650 Country_val='com.tr'
651 elif Country_val=='ae':
652 Country_val='ae'
653 elif Country_val=='ae':
654 Country_val='ae'
655 else :
656 Country_val=='com'
657
658 social_network_val=social_network_val.replace('.com','.'+Country_val)
659
660 concatstring = concatstring + "" + social_network_val + "";
661 SearchEngine='Google'
662
663
664 gl_map = {
665 "it": "it",
666 "www": "us",
667 "gm": "de",
668 "fr": "fr",
669 "sp": "es",
670 "uk": "uk",
671 "al": "al",
672 "ag": "dz",
673 "ar": "ar",
674 "am": "am",
675 "as": "au",
676 "au": "at",
677 "aj": "az",
678 "bg": "bd",
679 "bo": "by",
680 "be": "be",
681 "bh": "bz",
682 "bk": "ba",
683 "br": "br",
684 "bu": "bg",
685 "ca": "ca",
686 "ci": "cl",
687 "ch": "cn",
688 "co": "co",
689 "cs": "cr",
690 "hr": "hr",
691 "cu": "cu",
692 "ez": "ez",
693 "dk": "dk",
694 "ec": "ec",
695 "eg": "eg",
696 "en": "ee",
697 "fi": "fi",
698 "gg": "ge",
699 "gr": "gr",
700 "hk": "hk",
701 "hu": "hu",
702 "ic": "is",
703 "in": "in",
704 "id": "id",
705 "ir": "ir",
706 "iz": "iq",
707 "ei": "ie",
708 "is": "il",
709 "jm": "jm",
710 "ja": "jp",
711
712 "ke": "ke",
713 "kn": "kp",
714 "ks": "kr",
715 "ku": "kw",
716 "lg": "lv",
717 "ly": "ly",
718 "ls": "li",
719 "lh": "lt",
720 "lu": "lu",
721 "mc": "mo",
722 "mk": "mk",
723 "my": "my",
724 "mt": "mt",
725 "mx": "mx",
726 "md": "md",
727 "mn": "mc",
728 "mj": "me",
729 "mo": "ma",
730 "np": "np",
731 "nl": "nl",
732 "nz": "nz",
733 "ni": "ng",
734 "no": "no",
735 "pk": "pk",
736 "we": "ps",
737 "pm": "pa",
738 "pa": "py",
739 "pe": "pe",
740 "rp": "ph",
741 "pl": "pl",
742 "po": "pt",
743 "rq": "pr",
744 "qa": "qa",
745 "ro": "ro",
746 "rs": "ru",
747 "sm": "sm",
748 "sa": "sa",
749 "sg": "sn",
750 "ri": "rs",
751 "sn": "sg",
752 "lo": "sk",
753 "si": "si",
754 "sf": "za",
755 "sw": "se",
756 "sz": "ch",
757 "sy": "sy",
758 "tw": "tw",
759 "th": "th",
760 "ts": "tn",
761 "tu": "tu",
762 "ua": "ua",
763 "ae": "ae",
764 "uy": "uy",
765 "uz": "uz",
766 "ve": "ve",
767 "vn": "vn",
768 "lk": "lk",
769 }
770
771 gl = gl_map.get(Country_val, Country_val)
772
773
774
775
776
777
778
779
780
781
782 qry="https://google.com/search?q="+ concatstring
783 Actor.log.info(concatstring)
784
785
786
787 query =concatstring
788 max_pages = 100
789 results = []
790 count_result=0
791 has_next=True
792 try_count=0
793 page=0
794
795 '''
796 queries = generate_search_queries(
797 keyword=Keyword_val,
798 social_network_val=social_network_val,
799 location_val=location_val,
800 country_val=Country_val,
801 )
802 print(queries)
803 '''
804 APIFY_USER_ID = os.getenv("APIFY_USER_ID")
805 ACTOR_RUN_ID= os.getenv("ACTOR_RUN_ID")
806 camp_name="Apify_camp_linkedin_"+APIFY_USER_ID+"_"+ACTOR_RUN_ID
807
808 if is_paying=="1":
809 await Insertapifylog(camp_name,0)
810 try:
811 await run_multiple_queries(
812 query=query,
813 Email_Type_val=Email_Type_val,
814 Other_Email_Type_val=Other_Email_Type_val,
815 proxyurl=proxyurl,
816 max_pages=max_pages,
817 social_network_val=social_network_val,
818 proxy_configuration=proxy_configuration,
819 Limit_val=Limit_val)
820
821 '''
822 queries=["email @gmail.com","@gmail.com","@hotmail.com","email @hotmail.com","@icloud.com","email @icloud.com","@yahoo.com","email @yahoo.com"]
823 if Email_Type_val=="1":
824 if Other_Email_Type_val.find("@") > -1:
825 option = "" + Other_Email_Type_val + " "
826 else :
827 option = " @" + Other_Email_Type_val + ""
828 queries=[option]
829
830 for item in queries:
831 page=0
832 try_count=0
833 if has_next==False:
834 break
835 print('try next varient')
836 newquery=query.replace('@gmail.com',item)
837 print(newquery)
838 APIFY_USER_ID = os.getenv("APIFY_USER_ID")
839 while page<max_pages:
840 try :
841 if has_next==False:
842 break
843 page=page+1
844 Actor.log.info(' method Check Next Page'+str(page))
845 start = page * 10
846 url = f"http://www.google.com/search?q={newquery}&num=10&hl=en&start={start}"
847 #print(url)
848 # Get fresh proxy URL for every request
849 #proxy_url = await proxy_configuration.new_url()
850 proxies=None
851 if proxyurl:
852 proxies = {'http': proxyurl, 'https': proxyurl}
853
854 response = requests.get(
855 url,
856 proxies=proxies,
857 headers={
858 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
859 "AppleWebKit/537.36 (KHTML, like Gecko) "
860 "Chrome/120.0.0.0 Safari/537.36"
861 }
862 )
863 if response.status_code != 200:
864 Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")
865 continue
866
867 soup = BeautifulSoup(response.text, "html.parser")
868 #result_blocks = soup.select("div.g, div.tF2Cxc")
869 if social_network_val == "youtube.com/" or social_network_val == "tiktok.com/" or social_network_val == "instagram.com/" :
870 result_blocks = soup.select("div.MjjYud:has(h3)")
871 else:
872 result_blocks = soup.select("div.tF2Cxc:has(h3)")
873 if(len(result_blocks)==0):
874 print('No Result')
875 page=page-1
876 try_count=try_count+1
877 await asyncio.sleep(1)
878 proxyurl =await proxy_configuration.new_url()
879 if proxy_configuration and proxy_settings:
880 proxyurl =await proxy_configuration.new_url()
881 print(proxyurl)
882 if try_count>3:
883 break;
884 continue
885 if(len(result_blocks)>0):
886 try_count=0
887 print(len(result_blocks))
888 for j, block in enumerate(result_blocks):
889 Email=''
890 title_el = block.select_one("h3")
891 link_el = block.select_one("a")
892 snippet_el = block.select_one(".VwiC3b, .IsZvec, .aCOpRe")
893
894
895
896 title= title_el.get_text(strip=True)
897 url =link_el.get("href")
898
899 snippet= snippet_el.get_text(strip=True)
900 print(title)
901 alltext = title+snippet
902 #match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', alltext)
903 match =re.findall(r'[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.(?:com|net|org|in|co|io|edu)',alltext)
904 Actor.log.info('match email '+str(len(match)))
905 if len(match)>0:
906 for i in match:
907 Email=i
908 Actor.log.info('email '+Email)
909 else:
910 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', alltext)
911 for i in match_website:
912 Website = "http://www."+i;
913 Actor.log.info('Website '+Website)
914 Email=scrape_contact_emails(Website)
915 if Email :
916 existindb=False
917 #Actor.log.info('Email Store '+Email)
918 if len(all_users)>0:
919 for item in all_users:
920 if item['Email'] == Email :
921 existindb=True
922 break
923
924 if existindb==False:
925 all_users.append({'Email': Email});
926 await Actor.push_data({'Email': Email, 'title': title,'Description':alltext,'Detail_Link':url})
927 count_result=count_result+1
928 if(Limit_val):
929 if Limit_val!='0':
930 if(count_result>=int(Limit_val)):
931 has_next = False
932 print('Limit Exceed')
933 break
934 # Random polite delay between pages
935 if has_next==False:
936 break
937 await asyncio.sleep(random.uniform(1.5, 3.0))
938 except Exception as e:
939 print(e)
940 continue
941 #await Actor.exit()
942
943
944 '''
945 count_result=result_count
946 print('End Limit_val'+ str(Limit_val))
947 if(count_result>=int(Limit_val)) and int(Limit_val)>0:
948 has_next = False
949 print('Limit Exceed')
950 return
951
952 print('Check in Yahoo Search Engine')
953 query=concatstring.replace("OR", "or")
954
955
956 for page in range(max_pages):
957 try:
958 start = page * 10
959 Actor.log.info('Check Page '+str(page))
960
961 url = "https://search.yahoo.com/search"
962 params = {
963 "p": query,
964 "b": start,
965 "pz": 10,
966 "bct": 0,
967 "xargs": 0
968 }
969
970
971 split_url = urlsplit(url)
972 query_str = urlencode(params)
973 full_url = urlunsplit((split_url.scheme, split_url.netloc, split_url.path, query_str, ""))
974
975
976
977
978 proxies=None
979 if proxyurl_selected:
980 proxies = {'http': proxyurl_selected, 'https': proxyurl_selected}
981
982 response = requests.get(
983 full_url,
984 proxies=proxies,
985 headers={
986 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
987 "AppleWebKit/537.36 (KHTML, like Gecko) "
988 "Chrome/120.0.0.0 Safari/537.36"
989 }
990 )
991
992 if response.status_code != 200:
993 Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")
994 break
995
996 soup = BeautifulSoup(response.text, "html.parser")
997 search_results = soup.select("div.dd.algo.algo-sr")
998 print(search_results)
999 if not search_results:
1000 log("⚠️ No more results found, ending.")
1001 break
1002 if(len(search_results)==0):
1003 print('No Result')
1004 break;
1005
1006 for j, r in enumerate(search_results):
1007 try:
1008 Email=''
1009 Website=''
1010 Address=''
1011 BusinessName=''
1012 DetailsLink=''
1013 Category=''
1014 link_tag = r.select_one("h3.title")
1015
1016 link_tag1 = r.select_one("a")
1017 if link_tag:
1018 BusinessName = link_tag.get_text(strip=True)
1019 print(BusinessName)
1020 DetailsLink = link_tag1.get("href")
1021 Category = urlparse(DetailsLink).netloc
1022 addr_tag = r.select_one("div.compText.aAbs")
1023 if addr_tag:
1024 Address = addr_tag.get_text(separator=" ").strip()
1025 combined = r.get_text(strip=True)
1026 print(combined)
1027 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', combined)
1028 Actor.log.info('match email '+str(len(match)))
1029 if len(match)>0:
1030 for i in match:
1031 Email=i
1032 Actor.log.info('email '+Email)
1033 else:
1034 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', combined)
1035 for i in match_website:
1036 Website = "http://www."+i;
1037 Actor.log.info('Website '+Website)
1038 Email=scrape_contact_emails(Website)
1039 if Email :
1040 existindb=False
1041
1042 if len(all_users)>0:
1043 for item in all_users:
1044 if item['Email'] == Email :
1045 existindb=True
1046 break
1047
1048 if existindb==False:
1049 all_users.append({'Email': Email});
1050 await Actor.push_data({'Email': Email, 'title': BusinessName,'Description':combined,'Detail_Link':DetailsLink})
1051 count_result=count_result+1
1052 if(Limit_val):
1053 if Limit_val!='0':
1054 if(count_result>=int(Limit_val)):
1055 has_next = False
1056 print('Limit Exceed')
1057 break
1058
1059 except Exception as ex:
1060 print(f"⚠️ Parse error: {ex}")
1061 continue
1062
1063 if has_next==False:
1064 break
1065 await asyncio.sleep(random.uniform(1.5, 3.0))
1066 except:
1067
1068 break
1069
1070 if is_paying=="1":
1071 print('Total Result '+str(count_result))
1072 print('Start from Method 3')
1073 try:
1074 APIFY_USER_ID = os.getenv("APIFY_USER_ID")
1075 ACTOR_RUN_ID= os.getenv("ACTOR_RUN_ID")
1076 proxyurl=''
1077 USE_Proxy=False
1078 my_proxy_settings=None
1079 if APIFY_USER_ID=='6zwMZqjoXwUIMRY2V':
1080
1081
1082 await StartProcess(
1083 camp_name,
1084 "ALLINONE",
1085 Keyword_val,
1086 location_val,
1087 social_network_val,
1088 Country_val,
1089 "Google",
1090 USE_Proxy,
1091 my_proxy_settings,all_users,count_result,Limit_val
1092 )
1093 else:
1094
1095 try:
1096 qry="https://google.com/search?q="+ concatstring+"&gl="+gl
1097 Actor.log.info(concatstring)
1098 await callpython(qry,all_users,social_network_val,count_result,Limit_val)
1099 except:
1100 print("")
1101
1102 await StartProcess(
1103 camp_name,
1104 "ALLINONE",
1105 Keyword_val,
1106 location_val,
1107 social_network_val,
1108 Country_val,
1109 "Google",
1110 USE_Proxy,
1111 my_proxy_settings,all_users,count_result,Limit_val
1112 )
1113
1114
1115
1116 await Insertapifylog(camp_name,count_result)
1117
1118 finally:
1119 await on_aborting_event()
1120
1121 except Exception as e:
1122
1123 print(f"An error occurred: {e}")
1124 APIFY_USER_ID = os.getenv("APIFY_USER_ID")
1125 ACTOR_RUN_ID= os.getenv("ACTOR_RUN_ID")
1126 proxyurl=''
1127 USE_Proxy=False
1128 my_proxy_settings=None
1129 await StartProcess(
1130 camp_name,
1131 "ALLINONE",
1132 Keyword_val,
1133 location_val,
1134 social_network_val,
1135 Country_val,
1136 "Google",
1137 USE_Proxy,
1138 my_proxy_settings,all_users,count_result,Limit_val
1139 )
1140
1141 return