Linkedin Email Scraper
Pricing
from $7.00 / 1,000 results
Linkedin Email Scraper- Scrap Emails from linkedin specific profile using google search engine
Pricing
from $7.00 / 1,000 results
Rating
3.0
(4)
Developer
bhansalisoft
Maintained by CommunityActor stats
75
Bookmarked
3.7K
Total users
22
Monthly active users
a day ago
Last modified
Categories
Share
# First, specify the base Docker image.# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.# You can also use any other image from Docker Hub.#FROM apify/actor-python-selenium:3.11FROM apify/actor-python-playwright:3.14-1.63.0# Second, copy just requirements.txt into the Actor image,# since it should be the only file that affects the dependency install in the next step,# in order to speed up the buildCOPY requirements.txt ./
# Install the packages specified in requirements.txt,# Print the installed Python version, pip version# and all installed packages with their versions for debuggingRUN echo "Python version:" \ && python --version \ && echo "Pip version:" \ && pip --version \ && echo "Installing dependencies:" \ && pip install -r requirements.txt \ && echo "All installed Python packages:" \ && pip freeze
# Next, copy the remaining files and directories with the source code.# Since we do this after installing the dependencies, quick build will be really fast# for most source file changes.COPY . ./
# Use compileall to ensure the runnability of the Actor Python code.RUN python3 -m compileall -q .
# Specify how to launch the source code of your Actor.# By default, the "python3 -m src" command is runCMD ["python3", "-m", "src"]{ "actorSpecification": 1, "name": "Linkedin Email Scraper", "title": "Linkedin Email Scraper", "description": "Scrap Emails from Linkedin social media profile using Search Engine", "version": "0.0", "meta": { "templateId": "python-selenium" }, "input": "./input_schema.json", "output": "./output_schema.json", "dockerfile": "./Dockerfile", "storages": { "dataset": { "actorSpecification": 1, "title": "Linkedin Email Scraper", "views": { "titles": { "title": "Linkedin Email Scraper", "transformation": { "fields": [ "Email", "title", "Description", "Detail_Link" ] }, "display": { "component": "table", "properties": { "Email": { "label": "Email", "format": "text" }, "title": { "label": "Title", "format": "text" } , "Description": { "label": "Description", "format": "text" } , "Detail_Link": { "label": "Detail_Link", "format": "text" } } } } } } }}{ "title": "Linkedin Email Scraper", "description": "Linkedin Email Scraper : Scrap Emails from Linkedin Profiles.", "type": "object", "schemaVersion": 1, "properties": { "Keyword": { "title": "Keyword", "description": "Keyword", "type": "string", "editor": "textfield", "prefill":"TestKeyword" }, "location": { "title": "Location(optional)", "description": "Location", "type": "string", "editor": "textfield" }, "social_network": { "title": "Social Network", "description": "Social Network", "type": "string", "editor": "select", "default": "linkedin.com/", "enum": ["linkedin.com/"], "enumTitles": ["Linkedin"] }, "Country": { "title": "Country", "description": "Country", "type": "string", "editor": "select", "default": "www", "enum": ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"], "enumTitles": ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"] }, "Email_Type": { "title": "Email Type", "description": "Email Type", "type": "string", "editor": "select", "default": "0", "enum": ["0", "1"], "enumTitles": ["Popular Emails only(@gmail.com or @yahoo.com or @hotmail.com)", "Other Private Email"] }, "Other_Email_Type": { "title": "Other Email Type(Example @domain.com)", "description": "Other Email Type(Example @domain.com)", "type": "string", "editor": "textfield" }, "Limit": { "title": "Max Number of Result", "description": "Max Number of Result : if this option is blank or 0 then it will scrap unlimited", "type": "string", "editor": "textfield", "prefill":"0" }, "proxySettings": { "title": "Proxy configuration", "type": "object", "description": "Select proxies to be used by your crawler.", "editor": "proxy", "prefill": { "useApifyProxy": false } }
}, "required": ["Keyword","social_network","Email_Type","Country"]}{ "actorOutputSchemaVersion": 1, "title": "Output schema", "properties": { "overview": { "type": "string", "title": "Overview", "template": "{{links.apiDefaultDatasetUrl}}/items?view=overview" } }}1"""2This module serves as the entry point for executing the Apify Actor. It handles the configuration of logging3settings. The `main()` coroutine is then executed using `asyncio.run()`.4
5Feel free to modify this file to suit your specific needs.6"""7
8import asyncio9import logging10
11from apify.log import ActorLogFormatter12
13from .main import main14
15# Configure loggers16handler = logging.StreamHandler()17handler.setFormatter(ActorLogFormatter())18
19apify_client_logger = logging.getLogger('apify_client')20apify_client_logger.setLevel(logging.INFO)21apify_client_logger.addHandler(handler)22
23apify_logger = logging.getLogger('apify')24apify_logger.setLevel(logging.DEBUG)25apify_logger.addHandler(handler)26
27# Execute the Actor main coroutine28asyncio.run(main())1# datascrapify.py2import json3from urllib.parse import urlparse4import requests5from apify import Actor6import time7processId='0'8api_key = 'nTYunJBzJ1Nc0R8n6I1'9async def StartProcess(camp_name, camp_type, keyword, Location,SocialMedia,Country,search_engine,Use_Proxy,proxy_settings,all_users,count_result,Limit_val):10 try:11
12 url = 'https://www.datascrapify.com/api/Tool/EmailScraper'13 headers = {14 'Content-Type': 'application/json',15 'Accept': 'application/json',16 'x_api_key': api_key17 }18 19 payload_data = {20 "camp_name": camp_name,21 "camp_type": camp_type,22 "keyword": keyword,23 "Location": Location, # Note: This key is duplicated. Consider using "location" instead.24 "SocialMedia": SocialMedia,25 "Country": Country,26 "search_engine": search_engine,27 "Use_Proxy": Use_Proxy,28 "run_process":True29 }30 31 # Conditionally add the proxy settings only if a proxy is used32 if Use_Proxy:33 payload_data["_ProxySettings"] = proxy_settings34 35 payload = json.dumps(payload_data)36 print(payload)37 # Make the POST request38 response = requests.request("POST", url, headers=headers, data=payload)39 result = response.json()40 print(result)41 result = json.loads(result) 42 print(result)43 processId=result['processId']44 print(processId)45 if processId:46 print('Start Getting Result from Campaign')47 Fetch_True=True48 while Fetch_True :49 url='https://www.datascrapify.com/api/Tool/GetCampResult?processId='+processId50 response = requests.request("GET", url, headers=headers)51 result = response.json()52 result = json.loads(result)53 new_items = result['data']54 print(new_items)55 if new_items:56 #await Actor.push_data(new_items)57 for user in new_items:58 print(user)59 exists=False60 if len(all_users)>0: 61 for item in all_users:62 if item['Email'] == user['Email'] :63 exists=True64 break65 66 if exists==False and user['Email']!='':67 print('insert')68 all_users.append({'Email': user['Email']});69 #all_users.append({'uniqueid': user['uniqueid'], 'Name': user['Name'],'Email':user['Email'],'Description':user['Description'],'Link':user['Link'] ,'Category':user['Category'], 'Website':user['Website'] });70 #await Actor.push_data({'uniqueid': user['uniqueid'], 'title': user['Name'],'Email':user['Email'],'Description':user['Description'],'Detail_Link':user['Link'] ,'Category':user['Category'], 'Website':user['Website'] }) 71 await Actor.push_data({'Email': user['Email'], 'title': user['Name'],'Description':user['Description'],'Detail_Link':user['Link']}) 72 count_result=count_result+173 if(Limit_val):74 if Limit_val!='0':75 if(count_result>=int(Limit_val)):76 Fetch_True = False77 print('Limit Exceed')78 Actor.log.info('Aborting process')79 await on_aborting_event() 80 break81
82 if result['status']=='Complete':83 Fetch_True=False84 time.sleep(20) 85 86 else:87 print('We cannot process request') 88
89 except Exception as e:90 Actor.log.info(e)91 await on_aborting_event() 92 finally:93 Actor.log.info('Aborting process')94 await on_aborting_event() 95 96 97 98async def on_aborting_event():99 headers = {100 'Content-Type': 'application/json',101 'Accept': 'application/json',102 'x_api_key': api_key103 }104 """105 This function will be called when the ABORTING event is emitted.106 Perform any necessary cleanup or state persistence here.107 """108 Actor.log.info('Actor is gracefully aborting. Performing cleanup...')109 # Example: Save current state to Key-Value store110 url='https://www.datascrapify.com/api/Tool/StopProcess?processId='+processId111 response = requests.request("GET", url, headers=headers)112 result = response.json()113 print(result)114 Actor.log.info('Cleanup complete.')115
116def parse_proxy_url(proxy_url_string):117 """118 Parses a proxy URL string and returns a dictionary of settings.119 """120 parsed = urlparse(proxy_url_string)121
122 if parsed.username and parsed.password:123 return {124 "ProxyServer": parsed.hostname,125 "ProxyPort": parsed.port,126 "ProxyAuthentification": True,127 "ProxyAuthLogin": parsed.username,128 "ProxyAuthPassword": parsed.password129 }130 elif parsed.hostname and parsed.port:131 return {132 "ProxyServer": parsed.hostname,133 "ProxyPort": parsed.port,134 "ProxyAuthentification": False,135 "ProxyAuthLogin": None,136 "ProxyAuthPassword": None137 }138 else:139 return None1"""2This module defines the `main()` coroutine for the Apify Actor, executed from the `__main__.py` file.3
4Feel free to modify this file to suit your specific needs.5
6To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:7https://docs.apify.com/sdk/python8"""9
10from urllib.parse import urljoin11
12from apify import Actor13import re14from urllib.parse import urljoin15from bs4 import BeautifulSoup16import requests17import requests.exceptions18import time19import random20from .datascrapify import StartProcess, parse_proxy_url,on_aborting_event21import asyncio22from urllib.parse import urlparse, urlsplit, urlunsplit, parse_qs, urlencode23import os24from .playwritescraper import callpython25# To run this Actor locally, you need to have the Selenium Chromedriver installed.26# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/27# When running on the Apify platform, it is already included in the Actor's Docker image.28
29import datetime30
31async def check_free_trial(user_id: str, is_paid_user: bool = False):32 """33 Returns True if the user is allowed to use the Actor.34 35 Paid users: always allowed.36 Free users: allowed for 24 hours from first use.37 """38
39 # Paid users don't need the free-trial restriction40 if is_paid_user:41 return True42
43 store = await Actor.open_key_value_store(name="user-trials-linkedinemailscrapers")44
45 trial_key = f"trial_start_{user_id}"46
47 now = datetime.datetime.now(datetime.timezone.utc)48
49 first_use = await store.get_value(trial_key)50
51 # First time this user has ever used the tool52 if not first_use:53 await store.set_value(54 trial_key,55 now.isoformat()56 )57
58 Actor.log.info(59 f"Free trial started for user {user_id}. "60 f"Trial valid for 24 hours."61 )62
63 return True64
65 # Existing user66 first_use_date = datetime.datetime.fromisoformat(first_use)67
68 # Make sure both timestamps are timezone-aware69 if first_use_date.tzinfo is None:70 first_use_date = first_use_date.replace(71 tzinfo=datetime.timezone.utc72 )73
74 elapsed = now - first_use_date75
76 if elapsed < datetime.timedelta(hours=24):77 remaining = datetime.timedelta(hours=24) - elapsed78
79 hours = int(remaining.total_seconds() // 3600)80 minutes = int((remaining.total_seconds() % 3600) // 60)81
82 Actor.log.info(83 f"Free trial active. "84 f"Remaining: {hours}h {minutes}m"85 )86
87 return True88
89 Actor.log.warning(90 f"Free trial expired for user {user_id}"91 )92
93 return False94
95def scrape_contact_emails(link):96 res = requests.get(link)97 domain = link.split(".")98 mailaddr = link99 soup = BeautifulSoup(res.text,"lxml")100 links = soup.find_all("a")101 contact_link = ''102 final_result = ""103 try:104 # Check if there is any email address in the homepage. 105 emails = soup.find_all(text=re.compile('.*@'+domain[1]+'.'+domain[2].replace("/","")))106 emails.sort(key=len)107 print(emails[0].replace("\n",""))108 final_result = emails[0]109 except:110 # Searching for Contact Us Page's url.111 try:112 flag = 0113 for link in links:114 if "contact" in link.get("href") or "Contact" in link.get("href") or "CONTACT" in link.get("href") or 'contact' in link.text or 'Contact' in link.text or 'CONTACT' in link.text:115 if len(link.get("href"))>2 and flag<2:116 flag = flag + 1117 contact_link = link.get("href")118
119 except:120 pass121
122 domain = domain[0]+"."+domain[1]+"."+domain[2]123 if(len(contact_link)<len(domain)):124 domain = domain+contact_link.replace("/","")125 else:126 domain = contact_link127
128 try:129 # Check if there is any email address in the Contact Us Page. 130 res = requests.get(domain)131 soup = BeautifulSoup(res.text,"lxml")132 emails = soup.find_all(text=re.compile('.*@'+mailaddr[7:].replace("/","")))133 emails.sort(key=len)134 try:135 print(emails[0].replace("\n",""))136 final_result = emails[0]137 return final_result138 except:139 pass140 except Exception as e:141 pass142
143 return ""144
145async def main() -> None:146 """147 The main coroutine is being executed using `asyncio.run()`, so do not attempt to make a normal function148 out of it, it will not work. Asynchronous execution is required for communication with Apify platform,149 and it also enhances performance in the field of web scraping significantly.150 """151 async with Actor:152 # Read the Actor input153 actor_input = await Actor.get_input() or {}154 Keyword_val = actor_input.get('Keyword')155 location_val = actor_input.get('location')156 social_network_val = actor_input.get('social_network')157 Country_val = actor_input.get('Country')158 Email_Type_val = actor_input.get('Email_Type')159 Other_Email_Type_val = actor_input.get('Other_Email_Type')160 proxy_settings = actor_input.get('proxySettings')161 Limit_val =actor_input.get('Limit')162 163 proxy_configuration_selected = await Actor.create_proxy_configuration(actor_proxy_input=proxy_settings)164 proxyurl_selected =None165 if proxy_configuration_selected and proxy_settings:166 proxyurl_selected =await proxy_configuration_selected.new_url()167 print(proxyurl_selected)168 #proxy_configuration = await Actor.create_proxy_configuration(actor_proxy_input=proxy_settings)169 proxy_configuration = await Actor.create_proxy_configuration(groups=['GOOGLE_SERP'])170 #Actor.log.info(proxy_configuration)171 proxyurl =await proxy_configuration.new_url()172 if proxy_configuration and proxy_settings:173 proxyurl =await proxy_configuration.new_url()174 #Actor.log.info(proxyurl)175 176
177 if not Keyword_val:178 Actor.log.info('Please insert keyword')179 await Actor.push_data({'Email': 'Please insert keyword'})180 await Actor.exit()181 return182 183 if Keyword_val=='TestKeyword':184 Actor.log.info('Please insert keyword')185 await Actor.push_data({'Email': 'Please insert Your Keyword'})186 await Actor.exit()187 return188 189 is_paying = os.getenv("APIFY_USER_IS_PAYING")190 print(is_paying) 191 if is_paying=="1":192 print('Paid User')193 else:194 Limit_val=0195 is_paid_user=False196 APIFY_USER_ID = os.getenv("APIFY_USER_ID")197 allowed = await check_free_trial(user_id=APIFY_USER_ID, is_paid_user=is_paid_user)198 if not allowed:199 await Actor.push_data({'Email': 'Free User can run for One Day Only. Please Subcribe'})200 return201 #await Actor.push_data({'Email': 'Free User can run tool with 1 day with 10 record'})202 203 '''204 try:205 proxyurl=''206 USE_Proxy=False207 if proxyurl:208 USE_Proxy=True209 my_proxy_settings = parse_proxy_url(proxyurl) 210 # Call the function from the imported module211 await StartProcess(212 "Apify_camp_LinkedinEmail_"+username+"_"+isPaying, 213 "ALLINONE", 214 Keyword_val, 215 location_val, 216 social_network_val, 217 Country_val, 218 "Google",219 USE_Proxy, 220 my_proxy_settings221 )222 223 finally:224 # Code that always executes (e.g., cleanup)225 print("This block always runs, regardless of exceptions.")226
227 await Actor.exit();228 ''' 229 230 l1 = ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"]231 l2= ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"]232 select_index=1233 select_country='United States'234 for count, ele in enumerate(l1):235 if(ele==Country_val):236 select_index=count237 break238 #print(count)239 #print(ele)240
241 for count, ele in enumerate(l2):242 if(count==select_index):243 select_country=ele244 break245
246 print(select_country) 247
248 concatstring = ""249 concatstring = concatstring + Keyword_val250 option = "( @gmail.com OR @hotmail.com OR @yahoo.com)";251 if Email_Type_val=="1":252 if not Other_Email_Type_val:253 Actor.log.info('Please insert Email Type Domain')254 await Actor.push_data({'Email': 'Please insert Email Type Domain'})255 await Actor.exit()256 return257 if Other_Email_Type_val.find("@") > -1:258 option = " ( " + Other_Email_Type_val + " )"259 else: 260 option = " ( @" + Other_Email_Type_val + " )" 261 concatstring = concatstring + option262 if location_val:263 concatstring = concatstring+ " in "+ location_val264 265 266 267 if social_network_val:268 concatstring = concatstring + " site:"269
270 if social_network_val == "linkedin.com/" or social_network_val == "pinterest.com/" :271 concatstring = concatstring + Country_val + ".";272 273
274 275
276 if social_network_val == "amazon.com/" :277 if Country_val=='gm':278 Country_val='de'279 elif Country_val=='sp':280 Country_val='es'281 elif Country_val=='fr':282 Country_val='fr' 283 elif Country_val=='uk':284 Country_val='co.uk'285 elif Country_val=='as':286 Country_val='com.au'287 elif Country_val=='www':288 Country_val='com' 289 elif Country_val=='in':290 Country_val='in'291 elif Country_val=='be':292 Country_val='com.be'293 elif Country_val=='br':294 Country_val='com.br'295 elif Country_val=='ca':296 Country_val='ca'297 elif Country_val=='ch':298 Country_val='cn' 299 elif Country_val=='eg': 300 Country_val='eg'301 elif Country_val=='it':302 Country_val='it' 303 elif Country_val=='ja':304 Country_val='co.jp'305 elif Country_val=='mx':306 Country_val='com.mx'307 elif Country_val=='nl':308 Country_val='nl'309 elif Country_val=='pl':310 Country_val='pl' 311 elif Country_val=='sa':312 Country_val='sa'313 elif Country_val=='sn':314 Country_val='sg' 315 elif Country_val=='sw':316 Country_val='se'317 elif Country_val=='tu':318 Country_val='com.tr' 319 elif Country_val=='ae':320 Country_val='ae' 321 elif Country_val=='ae':322 Country_val='ae'323 else : 324 Country_val=='com'325 326 social_network_val=social_network_val.replace('.com','.'+Country_val)327 328
329 concatstring = concatstring + "" + social_network_val + "";330 331 SearchEngine='Google'332 desktop_user_agents = ["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",333 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",334 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.139 Safari/537.36",335 "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",336 # Firefox - Windows337 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",338 "Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0",339 "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0",340 # Edge - Windows341 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.86 Safari/537.36 Edg/123.0.2420.65",342 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.112 Safari/537.36 Edg/122.0.2365.66",343 # Opera - Windows344 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 OPR/96.0.0.0",345 # Chrome - macOS346 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.129 Safari/537.36",347 "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",348 # Firefox - macOS349 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13.5; rv:124.0) Gecko/20100101 Firefox/124.0",350 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0",351 # Safari - macOS352 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",353 "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",354 # Chrome - Linux355 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.184 Safari/537.36",356 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",357 # Firefox - Linux358 "Mozilla/5.0 (X11; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",359 "Mozilla/5.0 (X11; Linux x86_64; rv:124.0) Gecko/20100101 Firefox/124.0",360 "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0",361 # Safari - iMac (intel)362 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15",363 # Edge - macOS364 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.86 Safari/537.36 Edg/123.0.2420.65",365 # Opera - macOS366 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/96.0.0.0"367 ]368
369 mobile_user_agents = [370 # Chrome - Android371 "Mozilla/5.0 (Linux; Android 13; Pixel 7 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.105 Mobile Safari/537.36",372 "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36",373 "Mozilla/5.0 (Linux; Android 11; Pixel 4a) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.140 Mobile Safari/537.36",374
375 # Samsung Internet - Android376 "Mozilla/5.0 (Linux; Android 13; SAMSUNG SM-S911B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/24.0 Chrome/123.0.0.0 Mobile Safari/537.36",377 "Mozilla/5.0 (Linux; Android 12; SAMSUNG SM-A525F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/23.0 Chrome/121.0.0.0 Mobile Safari/537.36",378
379 # Chrome - Xiaomi380 "Mozilla/5.0 (Linux; Android 13; M2101K6G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.105 Mobile Safari/537.36",381
382 # Safari - iPhone383 "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",384 "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",385 "Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/604.1",386
387 # Safari - iPad388 "Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",389 "Mozilla/5.0 (iPad; CPU OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",390
391 # Chrome - iPhone392 "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) CriOS/123.0.0.0 Mobile/15E148 Safari/604.1",393
394 # Opera - Android395 "Mozilla/5.0 (Linux; Android 13; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36 OPR/76.0.4017.123",396
397 # Edge - Android398 "Mozilla/5.0 (Linux; Android 13; Pixel 6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Mobile Safari/537.36 EdgA/123.0.2420.64",399
400 # Firefox - Android401 "Mozilla/5.0 (Android 13; Mobile; rv:124.0) Gecko/124.0 Firefox/124.0",402 "Mozilla/5.0 (Android 12; Mobile; rv:122.0) Gecko/122.0 Firefox/122.0"403] 404
405 406 all_user_agents = desktop_user_agents + mobile_user_agents407 random_user_agent = random.choice(all_user_agents)408
409
410 qry="https://google.com/search?q="+ concatstring411 Actor.log.info(concatstring)412
413 gl_map = {414 "it": "it",415 "www": "us",416 "gm": "de",417 "fr": "fr",418 "sp": "es",419 "uk": "uk",420 "al": "al",421 "ag": "dz",422 "ar": "ar",423 "am": "am",424 "as": "au",425 "au": "at",426 "aj": "az",427 "bg": "bd",428 "bo": "by",429 "be": "be",430 "bh": "bz",431 "bk": "ba",432 "br": "br",433 "bu": "bg",434 "ca": "ca",435 "ci": "cl",436 "ch": "cn",437 "co": "co",438 "cs": "cr",439 "hr": "hr",440 "cu": "cu",441 "ez": "ez",442 "dk": "dk",443 "ec": "ec",444 "eg": "eg",445 "en": "ee",446 "fi": "fi",447 "gg": "ge",448 "gr": "gr",449 "hk": "hk",450 "hu": "hu",451 "ic": "is",452 "in": "in",453 "id": "id",454 "ir": "ir",455 "iz": "iq",456 "ei": "ie",457 "is": "il",458 "jm": "jm",459 "ja": "jp",460
461 "ke": "ke",462 "kn": "kp",463 "ks": "kr",464 "ku": "kw",465 "lg": "lv",466 "ly": "ly",467 "ls": "li",468 "lh": "lt",469 "lu": "lu",470 "mc": "mo",471 "mk": "mk",472 "my": "my",473 "mt": "mt",474 "mx": "mx",475 "md": "md",476 "mn": "mc",477 "mj": "me",478 "mo": "ma",479 "np": "np",480 "nl": "nl",481 "nz": "nz",482 "ni": "ng",483 "no": "no",484 "pk": "pk",485 "we": "ps",486 "pm": "pa",487 "pa": "py",488 "pe": "pe",489 "rp": "ph",490 "pl": "pl",491 "po": "pt",492 "rq": "pr",493 "qa": "qa",494 "ro": "ro",495 "rs": "ru",496 "sm": "sm",497 "sa": "sa",498 "sg": "sn",499 "ri": "rs",500 "sn": "sg",501 "lo": "sk",502 "si": "si",503 "sf": "za",504 "sw": "se",505 "sz": "ch",506 "sy": "sy",507 "tw": "tw",508 "th": "th",509 "ts": "tn",510 "tu": "tu",511 "ua": "ua",512 "ae": "ae",513 "uy": "uy",514 "uz": "uz",515 "ve": "ve",516 "vn": "vn",517 "lk": "lk",518 }519
520 gl = gl_map.get(Country_val, Country_val)521
522
523 ##start New Logic524 all_users = []525 query =concatstring526 max_pages = 100527 results = []528 count_result=0529 has_next=True530 531
532 try_count=0533 page=0534 #for page in range(max_pages):535 while page < max_pages: 536 try :537 page=page+1538 Actor.log.info('Check Next Page'+str(page))539 start = page * 10540 url = f"http://www.google.com/search?q={query}&num=10&hl=en&start={start}"541 #print(url)542 # Get fresh proxy URL for every request543 #proxy_url = await proxy_configuration.new_url()544 proxies=None545 if proxyurl:546 proxies = {'http': proxyurl, 'https': proxyurl}547
548 response = requests.get(549 url,550 proxies=proxies,551 headers={552 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "553 "AppleWebKit/537.36 (KHTML, like Gecko) "554 "Chrome/120.0.0.0 Safari/537.36"555 }556 )557
558 if response.status_code != 200:559 Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")560 continue561 562 soup = BeautifulSoup(response.text, "html.parser")563 result_blocks = soup.select("div.g, div.tF2Cxc")564 if(len(result_blocks)==0):565 print('No Result')566 page=page-1567 try_count=try_count+1568 if try_count>2:569 break;570 continue571 if(len(result_blocks)>0):572 try_count=0 573 for j, block in enumerate(result_blocks):574 Email=''575 title_el = block.select_one("h3")576 link_el = block.select_one("a")577 snippet_el = block.select_one(".VwiC3b, .IsZvec, .aCOpRe")578 579 580
581 title= title_el.get_text(strip=True)582 url =link_el.get("href")583 if "/goto" in url:584 url='https://google.com/'+url585 try:586 r = requests.get(url) 587 print(r.url)588 url=r.url589 except:590 pass591 snippet= snippet_el.get_text(strip=True)592 print(title) 593 alltext = title+snippet594 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', alltext) 595 Actor.log.info('match email '+str(len(match)))596 if len(match)>0:597 for i in match:598 Email=i599 Actor.log.info('email '+Email)600 else:601 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', alltext)602 for i in match_website:603 Website = "http://www."+i;604 Actor.log.info('Website '+Website)605 Email=scrape_contact_emails(Website)606 if Email :607 existindb=False608 #Actor.log.info('Email Store '+Email)609 if len(all_users)>0: 610 for item in all_users:611 if item['Email'] == Email :612 existindb=True613 break614 615 if existindb==False:616 all_users.append({'Email': Email}); 617 await Actor.push_data({'Email': Email, 'title': title,'Description':alltext,'Detail_Link':url})618 count_result=count_result+1619 if(Limit_val):620 if Limit_val!='0':621 if(count_result>=int(Limit_val)):622 has_next = False623 print('Limit Exceed')624 break625 # Random polite delay between pages626 if has_next==False:627 break628 await asyncio.sleep(random.uniform(1.5, 3.0))629 except Exception as e:630 print(e)631 continue632 #await Actor.exit() 633 634 print('Check in Yahoo Search Engine')635 query=concatstring.replace("OR", "or")636 #has_next==True637 ## Yahoo Start638 for page in range(max_pages):639 try:640 if has_next==False:641 break642 start = page * 10643 Actor.log.info('Check Page '+str(page))644 #url = f"http://www.google.com/search?q={query}&num=10&hl=en&start={start}"645 url = "https://search.yahoo.com/search"646 params = {647 "p": query,648 "b": start,649 "pz": 10,650 "bct": 0,651 "xargs": 0652 }653
654 # Build request URL manually655 split_url = urlsplit(url)656 query_str = urlencode(params)657 full_url = urlunsplit((split_url.scheme, split_url.netloc, split_url.path, query_str, ""))658
659 #print(f"Fetching page {full_url}")660 # Get fresh proxy URL for every request661 #proxy_url = await proxy_configuration.new_url()662 proxies=None663 if proxyurl_selected:664 proxies = {'http': proxyurl_selected, 'https': proxyurl_selected}665
666 response = requests.get(667 full_url,668 proxies=proxies,669 headers={670 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "671 "AppleWebKit/537.36 (KHTML, like Gecko) "672 "Chrome/120.0.0.0 Safari/537.36"673 }674 )675
676 if response.status_code != 200:677 #Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")678 break 679
680 soup = BeautifulSoup(response.text, "html.parser")681 search_results = soup.select("div.dd.algo.algo-sr")682 print(search_results) 683 if not search_results:684 log("⚠️ No more results found, ending.")685 break686 if(len(search_results)==0):687 print('No Result')688 break;689
690 for j, r in enumerate(search_results):691 try:692 Email='' 693 Website=''694 Address=''695 BusinessName=''696 DetailsLink=''697 Category=''698 link_tag = r.select_one("h3.title")699 700 link_tag1 = r.select_one("a")701 if link_tag:702 BusinessName = link_tag.get_text(strip=True)703 print(BusinessName)704 DetailsLink = link_tag1.get("href")705 Category = urlparse(DetailsLink).netloc706 addr_tag = r.select_one("div.compText.aAbs")707 if addr_tag:708 Address = addr_tag.get_text(separator=" ").strip()709 combined = r.get_text(strip=True)710 print(combined)711 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', combined) 712 Actor.log.info('match email '+str(len(match)))713 if len(match)>0:714 for i in match:715 Email=i716 Actor.log.info('email '+Email)717 else:718 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', combined)719 for i in match_website:720 Website = "http://www."+i;721 Actor.log.info('Website '+Website)722 Email=scrape_contact_emails(Website)723 if Email :724 existindb=False725 #Actor.log.info('Email Store '+Email)726 if len(all_users)>0: 727 for item in all_users:728 if item['Email'] == Email :729 existindb=True730 break731 732 if existindb==False:733 all_users.append({'Email': Email}); 734 await Actor.push_data({'Email': Email, 'title': BusinessName,'Description':combined,'Detail_Link':DetailsLink})735 count_result=count_result+1736 if(Limit_val):737 if Limit_val!='0':738 if(count_result>=int(Limit_val)):739 has_next = False740 print('Limit Exceed')741 break742
743 except Exception as ex:744 print(f"⚠️ Parse error: {ex}")745 continue746 # Random polite delay between pages747 if has_next==False:748 break749 await asyncio.sleep(random.uniform(1.5, 3.0))750 except:751 #print(f"⚠️ Parse error: {ex}")752 break753 754 755 if is_paying=="1":756 print('Start from Method 3')757 try:758 APIFY_USER_ID = os.getenv("APIFY_USER_ID")759 proxyurl=''760 USE_Proxy=False761 my_proxy_settings=None762 # Call the function from the imported module763 qry="https://google.com/search?q="+ concatstring+"&gl="+gl764 Actor.log.info(concatstring)765 await callpython(qry,all_users,social_network_val,count_result,Limit_val)766 767 '''768 await StartProcess(769 "Apify_camp_linkedin_"+APIFY_USER_ID, 770 "ALLINONE", 771 Keyword_val, 772 location_val, 773 social_network_val, 774 Country_val, 775 "Google",776 USE_Proxy, 777 my_proxy_settings,all_users,count_result,Limit_val778 )779 '''780 finally:781 await on_aborting_event()782 783 ##End New Logic1import sys2import asyncio3import json4import argparse5import urllib.parse6from playwright.async_api import async_playwright7from bs4 import BeautifulSoup8from playwright_stealth import Stealth 9import tempfile10import uuid11from datetime import datetime12import os13import zipfile14from apify import Actor15import re16
17# Configure your 2Captcha API Key18API_KEY = "9deb094e9f6ed7e879f716134f8357fc"19#NOPECHA_API_KEY = "sub_1UKxFFCRwBwvt6ptnqUsYL6z"20NOPECHA_API_KEY="sub_1UNmIvCRwBwvt6ptBT1BqAB0"21import os22
23async def handle_google_consent(page):24 """Checks for and accepts the Google consent modal if present."""25 consent_modal_selector = "div#xe7COe, div[aria-label='Before you continue to Google Search']"26 27 try:28 # Wait up to 2 seconds to see if the modal interrupts the session29 if await page.locator(consent_modal_selector).count() > 0:30 print("[INFO] Google Consent Modal detected. Attempting to accept...")31 32 # Target the 'Accept all' or 'Read more' action buttons inside the dialog33 # Google often uses 'button' tags or role="button" inside the dialog container34 accept_button = page.locator('div#xe7COe button#W0wltc').last35 36 await accept_button.click(timeout=3000)37 print("[SUCCESS] Consent modal accepted.")38 await page.wait_for_timeout(1000) # Let the overlay fade out safely39 except Exception as e:40 print(f"[WARNING] Could not bypass consent modal: {e}")41
42 43async def handle_pagination(page, social_network_val):44 45 # 1. Define Selectors based on your social network rules46 if social_network_val in ["youtube.com/", "instagram.com/" , "tiktok.com/"]:47 result_selector = "div.MjjYud"48 else:49 # Playwright alternative for 'div.tF2Cxc' or main organic results50 result_selector = "div.tF2Cxc" 51
52 next_button_selector = "a#pnnext"53 more_results_button = "span.RVQdVd"54
55 # Check if traditional 'Next' button exists56 next_btn = page.locator(next_button_selector)57 next_btn_count = await next_btn.count()58
59 if next_btn_count > 0:60 print("Strategy: Clicking 'Next' Button")61 current_url = page.url62 63 # Force a JavaScript click just like your Selenium execute_script64 await next_btn.first.evaluate("el => el.click()")65 66 # Wait until the URL changes (max 20 seconds)67 tryresult = 068 while page.url == current_url:69 await asyncio.sleep(1)70 tryresult += 171 if tryresult > 20:72 print("Timeout waiting for URL change.")73 break74 return False # Stop pagination loop75 76 if page.url == current_url:77 return False78 await page.wait_for_timeout(5000) 79 # Wait for the new results container to load on the new page80 await page.wait_for_selector(result_selector)81 return True82
83 else:84 print("Strategy: Infinite Scroll / More Results Button")85 # Get previous count of elements86 previous_count = await page.locator(result_selector).count()87 print(f"Previous Element Count: {previous_count}")88
89 results_locator = page.locator(result_selector)90 await results_locator.last.focus()91 92 # Check if the "More Results" button exists and click it93 more_btn = page.locator(more_results_button)94 if await more_btn.count() > 0:95 print("Found 'More Results' button, clicking it...")96 await more_btn.first.click()97 else:98 # If no button exists, scroll down to the bottom of the viewport to trigger lazy loading99 print("No button found, scrolling window down...")100 await page.evaluate("window.scrollTo(0, document.body.scrollHeight)")101
102 # Poll until the total number of items increases (max 20 seconds)103 tryresult = 0104 current_count = await page.locator(result_selector).count()105 106 while previous_count == current_count:107 await asyncio.sleep(1)108 tryresult += 1109 print(f"Waiting for new items to load... Attempt {tryresult}/20")110 111 if tryresult > 20:112 print("Timeout waiting for new elements to load.")113 break114 115 current_count = await page.locator(result_selector).count()116
117 print(f"Updated Element Count: {current_count}")118 119 # If no new items loaded after scrolling/clicking, we have reached the end120 if previous_count == current_count:121 print("No new results found. Ending pagination.")122 return False123 124 return True125
126
127
128async def callpython(query,all_users,network,count_result,Limit_val):129 #captcha_extension = TwoCaptcha(api_key=API_KEY)130 # This automatically downloads, configures with your key, and gives you the path131 #extension_path =captcha_extension.load(with_command_line_option=False)132 extension_path1 = os.path.abspath("./nocaptcha.zip")133 #print(extension_path1)134 unpacked_dir = os.path.abspath("./nopecha")135 136 with zipfile.ZipFile(extension_path1, 'r') as zip_ref:137 zip_ref.extractall(unpacked_dir)138 139 extension_path = os.path.abspath("./nopecha")140
141
142 #print(extension_path)143 if os.path.isdir(extension_path):144 print("Success: The folder exists!")145 else:146 print("Error: The folder does not exist.")147
148 # 2. UPDATE THE API KEY IN THE MANIFEST.JSON149 manifest_path = os.path.join(extension_path, "manifest.json")150 #print(manifest_path)151 # Open and read the current manifest configuration152 with open(manifest_path, "r", encoding="utf-8") as fp:153 manifest_data = json.load(fp)154
155 # In modern versions of NopeCHA, the key structure resides here:156 if "nopecha" not in manifest_data:157 manifest_data["nopecha"] = {}158 manifest_data["nopecha"]["key"] = NOPECHA_API_KEY159
160 # Overwrite and save the updated manifest data back to the file161 with open(manifest_path, "w", encoding="utf-8") as fp:162 json.dump(manifest_data, fp, indent=4) 163
164 #print(f"✅ Successfully updated manifest.json at {manifest_path} with your API key.")165 166 '''167 proxy_config={}168 if proxy_settings:169 print('apply proxy')170 proxy_config = {"server": proxy_settings["ProxyServer"]+":"+str(proxy_settings["ProxyPort"])}171 proxy_config["username"] = proxy_settings["ProxyAuthLogin"]172 proxy_config["password"] = proxy_settings["ProxyAuthPassword"] #APIFY_PROXY_PASSWORD173 '''174
175 async with Stealth().use_async(async_playwright()) as p:176 try:177 unique_id = f"{datetime.now().strftime('%Y%m%d_%H%M%S')}_{os.getpid()}"178
179 # Append the unique ID so every worker gets its own completely isolated folder180 user_data_dir = os.path.join(tempfile.gettempdir(), f"cloned_chrome_profile_{unique_id}")181 182 #3. Base Chromium arguments needed to load extensions cleanly183 chrome_args = [184 f"--disable-extensions-except={extension_path}",185 f"--load-extension={extension_path}",186 ]187 188 #browser = await p.chromium.launch(headless=False)189 context_opts = {190 "headless": False, 191 "viewport": {"width": 1920, "height": 1080},192 "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",193 "locale": "en-US",194 "timezone_id": "America/New_York",195 "permissions": ["geolocation"],196 "geolocation": {"latitude": 40.7128, "longitude": -74.0060},197 "extra_http_headers": {198 "Accept-Language": "en-US,en;q=0.9",199 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"200 },201 "args": chrome_args202
203 }204 #if proxy_settings:205 # context_opts["proxy"] = proxy_config206
207 #context = await browser.new_context(**context_opts)208 209 #page = await context.new_page()210 # 5. Launch using Persistent Context instead of launch() + new_context()211 context = await p.chromium.launch_persistent_context(212 user_data_dir, # <-- Passed explicitly here213 **context_opts # <-- Remaining options unpacked214 )215 # Grab the default open tab or create a new one216 page = context.pages[0] if context.pages else await context.new_page()217
218 await page.add_init_script("""219 Object.defineProperty(navigator, 'webdriver', {220 get: () => undefined221 });222
223 window.navigator.chrome = {224 runtime: {}225 };226
227 Object.defineProperty(navigator, 'plugins', {228 get: () => [1, 2, 3, 4, 5]229 });230
231 Object.defineProperty(navigator, 'languages', {232 get: () => ['en-US', 'en']233 });234 """)235 #await page.goto("https://api.ipify.org/?format=json")236 # Extract only the visible text inside the <body> tag237 #body_text =await page.locator("body").first.inner_text()238 #print(body_text)239 #await page.goto(f"https://nopecha.com/setup#{NOPECHA_API_KEY}")240 await page.wait_for_timeout(5000) 241 print(query)242 await page.goto(query, wait_until="domcontentloaded")243 await page.wait_for_timeout(5000) 244 page_url = page.url245 print(page_url)246 if 'google.com/sorry/index?continue' in page.url:247 print(f"Captcha Deteced solving")248 try_count=0249 while 'google.com/sorry/index?continue' in page.url:250 print(f"Captcha Deteced solving")251 # Non-blocking pause for 1 second to avoid melting your CPU252 await asyncio.sleep(1)253 try_count=try_count+1254 if try_count>200:255 break256 257 while True:258 await page.wait_for_timeout(5000)259 result_selector = "div.tF2Cxc"260 if network in ["youtube.com/", "instagram.com/" , "tiktok.com/"]:261 result_selector = "div.MjjYud:has(h3)"262 else:263 # Playwright alternative for 'div.tF2Cxc' or main organic results264 result_selector = "div.tF2Cxc:has(h3)" 265
266 await page.wait_for_selector(result_selector)267 # 3. Create a locator targeting ALL result items268 results_locator = page.locator(result_selector)269 270 # 4. Extract all matching elements into a Python list271 # .all() is an async method that converts the locator into iterable ElementHandles272 results = await results_locator.all()273 print(f"\nFound {len(results)} organic search results on the first page.\n")274 # 5. Loop/Iterate through each element card275 for index, result in enumerate(results, start=1):276 try:277 # Scrape elements relative to the current 'result' card278 # Google organic titles are inside h3 tags; links are in anchor tags279 title_el = result.locator("h3")280 link_el = result.locator("a")281 desc_el = result.locator(".VwiC3b")282 283 # Extract the text and attributes safely284 # (Using .first in case multiple matching elements exist inside the card layout)285 title = await title_el.first.inner_text()286 link = await link_el.first.get_attribute("href")287 # Check if description element exists before pulling text288 description = await desc_el.first.inner_text()289 # Clean up string text if it contains empty breaks290 description = description.strip().replace("\n", " ")291 # Print out the structured data292 alltext=title+description293 print(alltext)294 Email=None295 #await Actor.push_data({'title': title,'Description':alltext,'Detail_Link':link})296 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', alltext) 297 Actor.log.info('match email '+str(len(match)))298 if len(match)>0:299 for i in match:300 Email=i301 Actor.log.info('email '+Email)302 else:303 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', alltext)304 for i in match_website:305 Website = "http://www."+i;306 Actor.log.info('Website '+Website)307 Email=scrape_contact_emails(Website)308 if Email :309 existindb=False310 #Actor.log.info('Email Store '+Email)311 if len(all_users)>0: 312 for item in all_users:313 if item['Email'] == Email :314 existindb=True315 break316 317 if existindb==False:318 all_users.append({'Email': Email}); 319 await Actor.push_data({'Email': Email, 'title': title,'Description':alltext,'Detail_Link':link})320 count_result=count_result+1321 if(Limit_val):322 if Limit_val!='0':323 if(count_result>=int(Limit_val)):324 has_next = False325 print('Limit Exceed')326 break327 # Random polite delay between pages328 # Print out the structured data329 #print(f"[{index}] {title}")330 #print(f" URL: {link}\n")331
332 except Exception as e:333 print(e)334 # Skip any cards that are ads, people-also-ask widgets, or map blocks335 continue336 Actor.log.info('check for new page')337 if await handle_pagination(page, network)==False:338 break339 340 return341
342 except Exception as e:343 print(json.dumps({"error": str(e)}))# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed files.venv
# git folder.gitroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf# This file tells Git which files shouldn't be added to source control
.idea.DS_Store
apify_storagestorage
.venv/.env/__pypackages__dist/build/*.egg-info/*.egg
__pycache__
.mypy_cache.dmypy.jsondmypy.json.pytest_cache.ruff_cache
.scrapy*.log1# Feel free to add your Python dependencies below. For formatting guidelines, see:2# https://pip.pypa.io/en/latest/reference/requirements-file-format/3
4apify >=4.05playwright >= 1.58.0, < 2.0.06playwright-stealth7beautifulsoup4 ~= 4.12.28requests