Pricing
$10.00/month + usage
Linkedin Email Scraper- Scrap Emails from linkedin specific profile using google search engine
Pricing
$10.00/month + usage
Rating
1.7
(4)
Developer
bhansalisoft
Actor stats
68
Bookmarked
3.1K
Total users
49
Monthly active users
0.22 hours
Issues response
2 months ago
Last modified
Categories
Share
# First, specify the base Docker image.# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.# You can also use any other image from Docker Hub.FROM apify/actor-python-selenium:3.11
# Second, copy just requirements.txt into the Actor image,# since it should be the only file that affects the dependency install in the next step,# in order to speed up the buildCOPY requirements.txt ./
# Install the packages specified in requirements.txt,# Print the installed Python version, pip version# and all installed packages with their versions for debuggingRUN echo "Python version:" \ && python --version \ && echo "Pip version:" \ && pip --version \ && echo "Installing dependencies:" \ && pip install -r requirements.txt \ && echo "All installed Python packages:" \ && pip freeze
# Next, copy the remaining files and directories with the source code.# Since we do this after installing the dependencies, quick build will be really fast# for most source file changes.COPY . ./
# Use compileall to ensure the runnability of the Actor Python code.RUN python3 -m compileall -q .
# Specify how to launch the source code of your Actor.# By default, the "python3 -m src" command is runCMD ["python3", "-m", "src"]{ "actorSpecification": 1, "name": "Linkedin Email Scraper", "title": "Linkedin Email Scraper", "description": "Scrap Emails from Linkedin social media profile using Search Engine", "version": "0.0", "meta": { "templateId": "python-selenium" }, "input": "./input_schema.json", "dockerfile": "./Dockerfile", "storages": { "dataset": { "actorSpecification": 1, "title": "Linkedin Email Scraper", "views": { "titles": { "title": "Linkedin Email Scraper", "transformation": { "fields": [ "Email", "title", "Description", "Detail_Link" ] }, "display": { "component": "table", "properties": { "Email": { "label": "Email", "format": "text" }, "title": { "label": "Title", "format": "text" } , "Description": { "label": "Description", "format": "text" } , "Detail_Link": { "label": "Detail_Link", "format": "text" } } } } } } }}{ "title": "Linkedin Email Scraper", "description": "Linkedin Email Scraper : Scrap Emails from Linkedin Profiles.", "type": "object", "schemaVersion": 1, "properties": { "Keyword": { "title": "Keyword", "description": "Keyword", "type": "string", "editor": "textfield", "prefill":"TestKeyword" }, "location": { "title": "Location(optional)", "description": "Location", "type": "string", "editor": "textfield" }, "social_network": { "title": "Social Network", "description": "Social Network", "type": "string", "editor": "select", "default": "linkedin.com/", "enum": ["linkedin.com/"], "enumTitles": ["Linkedin"] }, "Country": { "title": "Country", "description": "Country", "type": "string", "editor": "select", "default": "www", "enum": ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"], "enumTitles": ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"] }, "Email_Type": { "title": "Email Type", "description": "Email Type", "type": "string", "editor": "select", "default": "0", "enum": ["0", "1"], "enumTitles": ["Popular Emails only(@gmail.com or @yahoo.com or @hotmail.com)", "Other Private Email"] }, "Other_Email_Type": { "title": "Other Email Type(Example @domain.com)", "description": "Other Email Type(Example @domain.com)", "type": "string", "editor": "textfield" }, "Limit": { "title": "Max Number of Result", "description": "Max Number of Result : if this option is blank or 0 then it will scrap unlimited", "type": "string", "editor": "textfield", "prefill":"0" }, "proxySettings": { "title": "Proxy configuration", "type": "object", "description": "Select proxies to be used by your crawler.", "editor": "proxy", "prefill": { "useApifyProxy": false } }
}, "required": ["Keyword","social_network","Email_Type","Country"]}1"""2This module serves as the entry point for executing the Apify Actor. It handles the configuration of logging3settings. The `main()` coroutine is then executed using `asyncio.run()`.4
5Feel free to modify this file to suit your specific needs.6"""7
8import asyncio9import logging10
11from apify.log import ActorLogFormatter12
13from .main import main14
15# Configure loggers16handler = logging.StreamHandler()17handler.setFormatter(ActorLogFormatter())18
19apify_client_logger = logging.getLogger('apify_client')20apify_client_logger.setLevel(logging.INFO)21apify_client_logger.addHandler(handler)22
23apify_logger = logging.getLogger('apify')24apify_logger.setLevel(logging.DEBUG)25apify_logger.addHandler(handler)26
27# Execute the Actor main coroutine28asyncio.run(main())1# datascrapify.py2import json3from urllib.parse import urlparse4import requests5from apify import Actor6import time7processId='0'8api_key = 'nTYunJBzJ1Nc0R8n6I1'9async def StartProcess(camp_name, camp_type, keyword, Location,SocialMedia,Country,search_engine,Use_Proxy,proxy_settings):10 try:11
12 all_users = []13 14 url = 'https://api.datascrapify.com/api/Tool/EmailScraper'15 headers = {16 'Content-Type': 'application/json',17 'Accept': 'application/json',18 'x_api_key': api_key19 }20 21 payload_data = {22 "camp_name": camp_name,23 "camp_type": camp_type,24 "keyword": keyword,25 "Location": Location, # Note: This key is duplicated. Consider using "location" instead.26 "SocialMedia": SocialMedia,27 "Country": Country,28 "search_engine": search_engine,29 "Use_Proxy": Use_Proxy,30 }31 32 # Conditionally add the proxy settings only if a proxy is used33 if Use_Proxy:34 payload_data["_ProxySettings"] = proxy_settings35 36 payload = json.dumps(payload_data)37 print(payload)38 # Make the POST request39 response = requests.request("POST", url, headers=headers, data=payload)40 result = response.json()41 print(result)42 result = json.loads(result) 43 print(result)44 processId=result['processId']45 print(processId)46 if processId:47 print('Start Getting Result from Campaign')48 Fetch_True=True49 while Fetch_True :50 url='https://api.datascrapify.com/api/Tool/GetCampResult?processId='+processId51 response = requests.request("GET", url, headers=headers)52 result = response.json()53 print(result)54 result = json.loads(result)55 new_items = result['data']56 if new_items:57 #await Actor.push_data(new_items)58 for user in new_items:59 print(user)60 exists=False61 if len(all_users)>0: 62 for item in all_users:63 if item['uniqueid'] == user['uniqueid'] :64 exists=True65 break66 67 if exists==False:68 all_users.append({'uniqueid': user['uniqueid'], 'title': user['Name'],'Email':user['Email'],'Description':user['Description'],'Link':user['Link'] ,'Category':user['Category'], 'Website':user['Website'] });69 await Actor.push_data({'uniqueid': user['uniqueid'], 'title': user['Name'],'Email':user['Email'],'Description':user['Description'],'Detail_Link':user['Link'] ,'Category':user['Category'], 'Website':user['Website'] }) 70 71 if result['status']=='Complete':72 Fetch_True=False73 time.sleep(20) 74 75 else:76 print('We cannot process request') 77
78 except:79 Actor.log.info('Aborting process')80 #await on_aborting_event() 81 finally:82 Actor.log.info('Aborting process')83 #await on_aborting_event() 84 85 86 87async def on_aborting_event():88 headers = {89 'Content-Type': 'application/json',90 'Accept': 'application/json',91 'x_api_key': api_key92 }93 """94 This function will be called when the ABORTING event is emitted.95 Perform any necessary cleanup or state persistence here.96 """97 Actor.log.info('Actor is gracefully aborting. Performing cleanup...')98 # Example: Save current state to Key-Value store99 url='https://api.datascrapify.com/api/Tool/StopProcess?processId='+processId100 response = requests.request("GET", url, headers=headers)101 result = response.json()102 print(result)103 Actor.log.info('Cleanup complete.')104
105def parse_proxy_url(proxy_url_string):106 """107 Parses a proxy URL string and returns a dictionary of settings.108 """109 parsed = urlparse(proxy_url_string)110
111 if parsed.username and parsed.password:112 return {113 "ProxyServer": parsed.hostname,114 "ProxyPort": parsed.port,115 "ProxyAuthentification": True,116 "ProxyAuthLogin": parsed.username,117 "ProxyAuthPassword": parsed.password118 }119 elif parsed.hostname and parsed.port:120 return {121 "ProxyServer": parsed.hostname,122 "ProxyPort": parsed.port,123 "ProxyAuthentification": False,124 "ProxyAuthLogin": None,125 "ProxyAuthPassword": None126 }127 else:128 return None1"""2This module defines the `main()` coroutine for the Apify Actor, executed from the `__main__.py` file.3
4Feel free to modify this file to suit your specific needs.5
6To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:7https://docs.apify.com/sdk/python8"""9
10from urllib.parse import urljoin11from seleniumwire import webdriver12#from selenium.webdriver.chrome.options import Options as ChromeOptions13from selenium.webdriver.common.by import By14from selenium.webdriver.common.action_chains import ActionChains15from apify import Actor16import re17from urllib.parse import urljoin18from bs4 import BeautifulSoup19import requests20import requests.exceptions21import time22import undetected_chromedriver as uc23import random24from .datascrapify import StartProcess, parse_proxy_url,on_aborting_event25from apify_shared.consts import ActorEventTypes as Event26import asyncio27from urllib.parse import urlparse, urlsplit, urlunsplit, parse_qs, urlencode28
29# To run this Actor locally, you need to have the Selenium Chromedriver installed.30# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/31# When running on the Apify platform, it is already included in the Actor's Docker image.32
33
34def scrape_contact_emails(link):35 res = requests.get(link)36 domain = link.split(".")37 mailaddr = link38 soup = BeautifulSoup(res.text,"lxml")39 links = soup.find_all("a")40 contact_link = ''41 final_result = ""42 try:43 # Check if there is any email address in the homepage. 44 emails = soup.find_all(text=re.compile('.*@'+domain[1]+'.'+domain[2].replace("/","")))45 emails.sort(key=len)46 print(emails[0].replace("\n",""))47 final_result = emails[0]48 except:49 # Searching for Contact Us Page's url.50 try:51 flag = 052 for link in links:53 if "contact" in link.get("href") or "Contact" in link.get("href") or "CONTACT" in link.get("href") or 'contact' in link.text or 'Contact' in link.text or 'CONTACT' in link.text:54 if len(link.get("href"))>2 and flag<2:55 flag = flag + 156 contact_link = link.get("href")57
58 except:59 pass60
61 domain = domain[0]+"."+domain[1]+"."+domain[2]62 if(len(contact_link)<len(domain)):63 domain = domain+contact_link.replace("/","")64 else:65 domain = contact_link66
67 try:68 # Check if there is any email address in the Contact Us Page. 69 res = requests.get(domain)70 soup = BeautifulSoup(res.text,"lxml")71 emails = soup.find_all(text=re.compile('.*@'+mailaddr[7:].replace("/","")))72 emails.sort(key=len)73 try:74 print(emails[0].replace("\n",""))75 final_result = emails[0]76 return final_result77 except:78 pass79 except Exception as e:80 pass81
82 return ""83
84async def main() -> None:85 """86 The main coroutine is being executed using `asyncio.run()`, so do not attempt to make a normal function87 out of it, it will not work. Asynchronous execution is required for communication with Apify platform,88 and it also enhances performance in the field of web scraping significantly.89 """90 async with Actor:91 # Read the Actor input92 actor_input = await Actor.get_input() or {}93 Keyword_val = actor_input.get('Keyword')94 location_val = actor_input.get('location')95 social_network_val = actor_input.get('social_network')96 Country_val = actor_input.get('Country')97 Email_Type_val = actor_input.get('Email_Type')98 Other_Email_Type_val = actor_input.get('Other_Email_Type')99 proxy_settings = actor_input.get('proxySettings')100 Limit_val =actor_input.get('Limit')101
102 #proxy_configuration = await Actor.create_proxy_configuration(actor_proxy_input=proxy_settings)103 proxy_configuration = await Actor.create_proxy_configuration(groups=['GOOGLE_SERP'])104 #Actor.log.info(proxy_configuration)105 proxyurl =await proxy_configuration.new_url()106 if proxy_configuration and proxy_settings:107 proxyurl =await proxy_configuration.new_url()108 #Actor.log.info(proxyurl)109 110
111 if not Keyword_val:112 Actor.log.info('Please insert keyword')113 await Actor.push_data({'Email': 'Please insert keyword'})114 await Actor.exit()115 return116 117 if Keyword_val=='TestKeyword':118 Actor.log.info('Please insert keyword')119 await Actor.push_data({'Email': 'Please insert Your Keyword'})120 await Actor.exit()121 return122 '''123 me = await Actor.apify_client.user('me').get()124 username=me["username"]125 isPaying='PayingUser'126 if(me["isPaying"]==False):127 isPaying='FreeUser'128
129 try:130 proxyurl=''131 USE_Proxy=False132 if proxyurl:133 USE_Proxy=True134 my_proxy_settings = parse_proxy_url(proxyurl) 135 # Call the function from the imported module136 await StartProcess(137 "Apify_camp_LinkedinEmail_"+username+"_"+isPaying, 138 "ALLINONE", 139 Keyword_val, 140 location_val, 141 social_network_val, 142 Country_val, 143 "Google",144 USE_Proxy, 145 my_proxy_settings146 )147 148 finally:149 # Code that always executes (e.g., cleanup)150 print("This block always runs, regardless of exceptions.")151
152 await Actor.exit();153 ''' 154 155 l1 = ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"]156 l2= ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"]157 select_index=1158 select_country='United States'159 for count, ele in enumerate(l1):160 if(ele==Country_val):161 select_index=count162 break163 #print(count)164 #print(ele)165
166 for count, ele in enumerate(l2):167 if(count==select_index):168 select_country=ele169 break170
171 print(select_country) 172
173 concatstring = ""174 concatstring = concatstring + Keyword_val175 option = "( @gmail.com OR @hotmail.com OR @yahoo.com)";176 if Email_Type_val=="1":177 if not Other_Email_Type_val:178 Actor.log.info('Please insert Email Type Domain')179 await Actor.push_data({'Email': 'Please insert Email Type Domain'})180 await Actor.exit()181 return182 if Other_Email_Type_val.find("@") > -1:183 option = " ( " + Other_Email_Type_val + " )"184 else: 185 option = " ( @" + Other_Email_Type_val + " )" 186 concatstring = concatstring + option187 if location_val:188 concatstring = concatstring+ " in "+ location_val189 190 191 192 if social_network_val:193 concatstring = concatstring + " site:"194
195 if social_network_val == "linkedin.com/" or social_network_val == "pinterest.com/" :196 concatstring = concatstring + Country_val + ".";197 198
199 200
201 if social_network_val == "amazon.com/" :202 if Country_val=='gm':203 Country_val='de'204 elif Country_val=='sp':205 Country_val='es'206 elif Country_val=='fr':207 Country_val='fr' 208 elif Country_val=='uk':209 Country_val='co.uk'210 elif Country_val=='as':211 Country_val='com.au'212 elif Country_val=='www':213 Country_val='com' 214 elif Country_val=='in':215 Country_val='in'216 elif Country_val=='be':217 Country_val='com.be'218 elif Country_val=='br':219 Country_val='com.br'220 elif Country_val=='ca':221 Country_val='ca'222 elif Country_val=='ch':223 Country_val='cn' 224 elif Country_val=='eg': 225 Country_val='eg'226 elif Country_val=='it':227 Country_val='it' 228 elif Country_val=='ja':229 Country_val='co.jp'230 elif Country_val=='mx':231 Country_val='com.mx'232 elif Country_val=='nl':233 Country_val='nl'234 elif Country_val=='pl':235 Country_val='pl' 236 elif Country_val=='sa':237 Country_val='sa'238 elif Country_val=='sn':239 Country_val='sg' 240 elif Country_val=='sw':241 Country_val='se'242 elif Country_val=='tu':243 Country_val='com.tr' 244 elif Country_val=='ae':245 Country_val='ae' 246 elif Country_val=='ae':247 Country_val='ae'248 else : 249 Country_val=='com'250 251 social_network_val=social_network_val.replace('.com','.'+Country_val)252 253
254 concatstring = concatstring + "" + social_network_val + "";255 256 SearchEngine='Google'257 desktop_user_agents = ["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",258 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",259 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.139 Safari/537.36",260 "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",261 # Firefox - Windows262 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",263 "Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0",264 "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0",265 # Edge - Windows266 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.86 Safari/537.36 Edg/123.0.2420.65",267 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.112 Safari/537.36 Edg/122.0.2365.66",268 # Opera - Windows269 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 OPR/96.0.0.0",270 # Chrome - macOS271 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.129 Safari/537.36",272 "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",273 # Firefox - macOS274 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13.5; rv:124.0) Gecko/20100101 Firefox/124.0",275 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0",276 # Safari - macOS277 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",278 "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",279 # Chrome - Linux280 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.184 Safari/537.36",281 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",282 # Firefox - Linux283 "Mozilla/5.0 (X11; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",284 "Mozilla/5.0 (X11; Linux x86_64; rv:124.0) Gecko/20100101 Firefox/124.0",285 "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0",286 # Safari - iMac (intel)287 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15",288 # Edge - macOS289 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.86 Safari/537.36 Edg/123.0.2420.65",290 # Opera - macOS291 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/96.0.0.0"292 ]293
294 mobile_user_agents = [295 # Chrome - Android296 "Mozilla/5.0 (Linux; Android 13; Pixel 7 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.105 Mobile Safari/537.36",297 "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36",298 "Mozilla/5.0 (Linux; Android 11; Pixel 4a) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.140 Mobile Safari/537.36",299
300 # Samsung Internet - Android301 "Mozilla/5.0 (Linux; Android 13; SAMSUNG SM-S911B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/24.0 Chrome/123.0.0.0 Mobile Safari/537.36",302 "Mozilla/5.0 (Linux; Android 12; SAMSUNG SM-A525F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/23.0 Chrome/121.0.0.0 Mobile Safari/537.36",303
304 # Chrome - Xiaomi305 "Mozilla/5.0 (Linux; Android 13; M2101K6G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.105 Mobile Safari/537.36",306
307 # Safari - iPhone308 "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",309 "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",310 "Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/604.1",311
312 # Safari - iPad313 "Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",314 "Mozilla/5.0 (iPad; CPU OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",315
316 # Chrome - iPhone317 "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) CriOS/123.0.0.0 Mobile/15E148 Safari/604.1",318
319 # Opera - Android320 "Mozilla/5.0 (Linux; Android 13; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36 OPR/76.0.4017.123",321
322 # Edge - Android323 "Mozilla/5.0 (Linux; Android 13; Pixel 6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Mobile Safari/537.36 EdgA/123.0.2420.64",324
325 # Firefox - Android326 "Mozilla/5.0 (Android 13; Mobile; rv:124.0) Gecko/124.0 Firefox/124.0",327 "Mozilla/5.0 (Android 12; Mobile; rv:122.0) Gecko/122.0 Firefox/122.0"328] 329
330 331 all_user_agents = desktop_user_agents + mobile_user_agents332 random_user_agent = random.choice(all_user_agents)333
334
335 qry="https://google.com/search?q="+ concatstring336 Actor.log.info(concatstring)337
338
339 ##start New Logic340 all_users = []341 query =concatstring342 max_pages = 100343 results = []344 count_result=0345 has_next=True346 for page in range(max_pages):347 try:348 start = page * 10349 Actor.log.info('Check Page '+str(page))350 url = f"http://www.google.com/search?q={query}&num=10&hl=en&start={start}"351
352 # Get fresh proxy URL for every request353 #proxy_url = await proxy_configuration.new_url()354 proxies=None355 if proxyurl:356 proxies = {'http': proxyurl, 'https': proxyurl}357
358 response = requests.get(359 url,360 proxies=proxies,361 headers={362 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "363 "AppleWebKit/537.36 (KHTML, like Gecko) "364 "Chrome/120.0.0.0 Safari/537.36"365 }366 )367
368 if response.status_code != 200:369 Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")370 continue371
372 soup = BeautifulSoup(response.text, "html.parser")373 result_blocks = soup.select("div.g, div.tF2Cxc")374
375 for j, block in enumerate(result_blocks):376 Email=''377 title_el = block.select_one("h3")378 link_el = block.select_one("a")379 snippet_el = block.select_one(".VwiC3b, .IsZvec, .aCOpRe")380 381 382
383 title= title_el.get_text(strip=True)384 url =link_el.get("href")385 snippet= snippet_el.get_text(strip=True)386 print(title) 387 alltext = title+snippet388 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', alltext) 389 Actor.log.info('match email '+str(len(match)))390 if len(match)>0:391 for i in match:392 Email=i393 Actor.log.info('email '+Email)394 else:395 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', alltext)396 for i in match_website:397 Website = "http://www."+i;398 Actor.log.info('Website '+Website)399 Email=scrape_contact_emails(Website)400 if Email :401 existindb=False402 #Actor.log.info('Email Store '+Email)403 if len(all_users)>0: 404 for item in all_users:405 if item['Email'] == Email :406 existindb=True407 break408 409 if existindb==False:410 all_users.append({'Email': Email}); 411 await Actor.push_data({'Email': Email, 'title': title,'Description':alltext,'Detail_Link':url})412 count_result=count_result+1413 if(Limit_val):414 if Limit_val!='0':415 if(count_result>=int(Limit_val)):416 has_next = False417 print('Limit Exceed')418 break419 # Random polite delay between pages420 if has_next==False:421 break422 await asyncio.sleep(random.uniform(1.5, 3.0))423 except:424 break425 #await Actor.exit()426 427 print('Check in Yahoo Search Engine')428 query=concatstring.replace("OR", "or")429 #has_next==True430 ## Yahoo Start431 for page in range(max_pages):432 try:433 if has_next==False:434 break435 start = page * 10436 Actor.log.info('Check Page '+str(page))437 #url = f"http://www.google.com/search?q={query}&num=10&hl=en&start={start}"438 url = "https://search.yahoo.com/search;_ylt=Awr.2lgGCQ9p1C0DekZXNyoA;_ylu=Y29sbwNncTEEcG9zAzEEdnRpZAMEc2VjA3BhZ2luYXRpb24-"439 params = {440 "p": query,441 "b": start,442 "pz": 10,443 "bct": 0,444 "xargs": 0445 }446
447 # Build request URL manually448 split_url = urlsplit(url)449 query_str = urlencode(params)450 full_url = urlunsplit((split_url.scheme, split_url.netloc, split_url.path, query_str, ""))451
452 print(f"Fetching page {full_url}")453 # Get fresh proxy URL for every request454 #proxy_url = await proxy_configuration.new_url()455 proxies=None456 #if proxyurl:457 # proxies = {'http': proxyurl, 'https': proxyurl}458
459 response = requests.get(460 full_url,461 proxies=proxies,462 headers={463 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "464 "AppleWebKit/537.36 (KHTML, like Gecko) "465 "Chrome/120.0.0.0 Safari/537.36"466 }467 )468
469 if response.status_code != 200:470 Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")471 break 472
473 soup = BeautifulSoup(response.text, "html.parser")474 search_results = soup.select("div.dd.algo.algo-sr")475
476 if not search_results:477 log("⚠️ No more results found, ending.")478 break479
480 for r in search_results:481 try:482 Email='' 483 Website=''484 Address=''485 BusinessName=''486 DetailsLink=''487 Category=''488 link_tag = r.select_one("h3.title")489 link_tag1 = r.select_one("a")490 if link_tag:491 BusinessName = link_tag.get_text(strip=True)492 DetailsLink = link_tag1.get("href")493 Category = urlparse(DetailsLink).netloc494 495 addr_tag = r.select_one("div.compText.aAbs")496 if addr_tag:497 Address = addr_tag.get_text(separator=" ").strip()498
499 combined = r.get_text(strip=True)500 print(combined)501 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', combined) 502 Actor.log.info('match email '+str(len(match)))503 if len(match)>0:504 for i in match:505 Email=i506 Actor.log.info('email '+Email)507 else:508 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', combined)509 for i in match_website:510 Website = "http://www."+i;511 Actor.log.info('Website '+Website)512 Email=scrape_contact_emails(Website)513 if Email :514 existindb=False515 #Actor.log.info('Email Store '+Email)516 if len(all_users)>0: 517 for item in all_users:518 if item['Email'] == Email :519 existindb=True520 break521 522 if existindb==False:523 all_users.append({'Email': Email}); 524 await Actor.push_data({'Email': Email, 'title': BusinessName,'Description':combined,'Detail_Link':DetailsLink})525 count_result=count_result+1526 if(Limit_val):527 if Limit_val!='0':528 if(count_result>=int(Limit_val)):529 has_next = False530 print('Limit Exceed')531 break532
533 except Exception as ex:534 print(f"⚠️ Parse error: {ex}")535 continue536 # Random polite delay between pages537 if has_next==False:538 break539 await asyncio.sleep(random.uniform(1.5, 3.0))540 except:541 #print(f"⚠️ Parse error: {ex}")542 break 543 await Actor.exit() 544 ##End New Logic545 546 # Launch a new Selenium Chrome WebDriver547 Actor.log.info('Launching Chrome WebDriver...')548 chrome_options = uc.ChromeOptions()549 chrome_options.add_argument(f'user-agent={random_user_agent}')550 chrome_options.add_argument("--disable-blink-features=AutomationControlled")551 #chrome_options.add_extension('./NopeCHA-CAPTCHA-Solver.crx')552 #chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36")553 chrome_options.add_argument('--no-sandbox')554 chrome_options.add_argument('--disable-dev-shm-usage')555 chrome_options.add_argument("--disable-blink-features=AutomationControlled")556 #options.add_experimental_option("useAutomationExtension", False)557 # chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])558 #chrome_options.add_experimental_option('useAutomationExtension', False)559 560 seleniumwire_options={}561 if(proxyurl):562 print('apply proxy')563 chrome_options.add_argument(f" - proxy-server={proxyurl}")564 #seleniumwire_options = {'proxy': { 'https': proxyurl,'https': proxyurl} }565 # Initialize SeleniumAuthenticatedProxy566 #proxy_helper = SeleniumAuthenticatedProxy(proxy_url=proxyurl)567 # Enrich Chrome options with proxy authentication568 #proxy_helper.enrich_chrome_options(chrome_options)569
570 #print(seleniumwire_options)571 #driver = webdriver.Chrome(options=chrome_options,seleniumwire_options=seleniumwire_options)572 driver = uc.Chrome(options=chrome_options, use_subprocess=False,version_main = 137)573 driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {574 "source": """575 Object.defineProperty(navigator, 'webdriver', {576 get: () => undefined577 })578 """579 })580
581 #driver.get('https://nopecha.com/setup#aqoz34cu2ob3qmhn')582 #time.sleep(3)583 #print("key : "+driver.execute_script("return document.querySelectorAll('body')[0].innerText"))584
585 # Visit the test URL to check your proxy IP586 driver.get("https://httpbin.io/ip")587 # Select the body tag containing the current IP address588 ip_address = driver.find_element(By.TAG_NAME, "body").text589 # Print your current IP590 print(ip_address)591 try:592 driver.get(qry)593 except Exception as e:594 print(e)595 await Actor.push_data({'Email': e})596 driver.quit()597 598 checkount=0599 try:600 while(driver.current_url.find("sorry/index")>-1 and checkount<100):601 print('captcha')602 print(driver.current_url)603 time.sleep(3)604 checkount=checkount+1605 except Exception as e:606 print(e) 607
608 all_users = []609 start=0610
611 if(SearchEngine=='Yahoo'):612 print('yahoo')613 else:614 try:615 while True:616 consent = driver.find_elements(By.XPATH,"//iframe[contains(@src, 'consent.google.com')]")617 if len(consent)>0 :618 driver.switch_to.frame(consent[0]);619 driver.find_element(By.id("id")).click();620
621 popup = driver.find_elements(By.CSS_SELECTOR,"button[class='yt-spec-button-shape-next yt-spec-button-shape-next--filled yt-spec-button-shape-next--mono yt-spec-button-shape-next--size-m']");622 if (len(popup) > 0):623 Actor.log.info("popup") 624 for n in range(0,len(popup)):625 if (popup[n].text=="Accept all"):626 popup[n].click();627 628 popup_accept = driver.find_elements(By.CSS_SELECTOR,"button[id='L2AGLb']");629 if (len(popup_accept) > 0):630 popup_accept[0].click();631 time.sleep(1)632 633 checkconsent=0;634 while len(driver.find_elements(By.CSS_SELECTOR,"div[class='HTjtHe'][style*='display: block']")) > 0 and checkconsent<100:635 Actor.log.info('checkconsent '+str(checkconsent))636 Actor.log.info("homeurl" + driver.current_url) 637 checkconsent=checkconsent+1638 time.sleep(1)639 640 AllClasses = driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc") 641 print(len(AllClasses)) 642 #checkconsent_youtube=0; 643 #while len(driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]")) <= 0 and checkconsent_youtube<20:644 #Actor.log.info('checkconsent_youtube '+str(checkconsent_youtube))645 #Actor.log.info("homeurl" + driver.current_url) 646 #checkconsent_youtube=checkconsent_youtube+1647 #time.sleep(1)648 homeurl = driver.current_url649 AllClasses = driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc")650 if social_network_val=="youtube.com/" or social_network_val=="instagram.com/":651 AllClasses = driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]")652 Actor.log.info("homeurl" + driver.current_url)653
654 checkount=0655 try:656 while(driver.current_url.index("sorry/index")>-1 and checkount<100):657 print('captcha')658 time.sleep(3)659 checkount=checkount+1660 except Exception as e:661 print(e) 662
663 if(len(AllClasses)==0):664 print('check new attribut')665 AllClasses = driver.find_elements(By.CSS_SELECTOR,"div.fP1Qef")666 print(len(AllClasses))667 print(driver.find_elements(By.CSS_SELECTOR,"div[id='main']"))668 669 #res = requests.get(driver.current_url)670 #await Actor.push_data({'Email': res.text})671 if len(AllClasses)>0:672 start=1673 Actor.log.info("Result" + str(len(AllClasses)))674 for gr in range(0,len(AllClasses)):675 try:676 BusinessName=""677 DetailsLink=""678 Email=""679 Address=""680 businessdetail = AllClasses[gr].find_elements(By.CSS_SELECTOR,"h3.LC20lb")681 if len(businessdetail) > 0:682 BusinessName = businessdetail[0].text;683 DetailsLink = businessdetail[0].find_element(By.XPATH,"parent::*").get_attribute("href")684 if not DetailsLink:685 businessdetailnew = AllClasses[gr].find_elements(By.CSS_SELECTOR,"div.TbwUpd")686 if len(businessdetailnew) > 0:687 DetailsLink = businessdetailnew[0].find_elements(By.XPATH,"parent::*").get_attribute("href")688 Ele_addressdetail = AllClasses[gr].find_elements(By.CSS_SELECTOR,"div.VwiC3b")689 if len(Ele_addressdetail) > 0:690 Address = Ele_addressdetail[0].text.replace(";", "-").replace(",", "-");691 692 alltext = AllClasses[gr].text693 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', alltext) 694 Actor.log.info('match email '+str(len(match)))695 if len(match)>0:696 for i in match:697 Email=i698 Actor.log.info('email '+Email)699 else:700 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', BusinessName+Address)701 for i in match_website:702 Website = "http://www."+i;703 Actor.log.info('Website '+Website)704 Email=scrape_contact_emails(Website)705 if Email :706 existindb=False707 #Actor.log.info('Email Store '+Email)708 if len(all_users)>0: 709 for item in all_users:710 if item['Email'] == Email :711 existindb=True712 break713 if existindb==False:714 all_users.append({'Email': Email}); 715 await Actor.push_data({'Email': Email, 'title': BusinessName,'Description':alltext,'Detail_Link':DetailsLink})716
717 718 except Exception as err:719 Actor.log.info(f"Unexpected {err=}, {type(err)=}") 720
721 if len(driver.find_elements(By.CSS_SELECTOR,"a#pnnext")) > 0:722 Actor.log.info('Click Next')723 str_url = driver.current_url724 #driver.find_elements(By.CSS_SELECTOR,"a#pnnext")[0].click();725 driver.execute_script("arguments[0].click();", driver.find_elements(By.CSS_SELECTOR,"a#pnnext")[0]);726 time.sleep(3)727 while driver.current_url == str_url:728 tryresult = 0;729 time.sleep(1);730 tryresult = tryresult + 1;731 if tryresult > 20:732 driver.quit();733 break;734 else:735 Actor.log.info('Click Next_1')736 AllCOUNT_PREVIOUS = len(driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc"))737 if social_network_val=="youtube.com/" or social_network_val=="instagram.com/":738 AllCOUNT_PREVIOUS = len(driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]"))739 Actor.log.info('Click Next_1_T1')740 str_url = driver.current_url741 action = ActionChains(driver)742 # perform the operation 743 #action.move_to_element(element).click().perform()744 tryresult = 0;745 if len(driver.find_elements(By.CSS_SELECTOR,"span.RVQdVd")) > 0:746 action = ActionChains(driver)747 selectedlink1 = driver.find_element(By.CSS_SELECTOR,"span.RVQdVd")748 action.move_to_element(selectedlink1).click().perform()749 750 time.sleep(3)751 Actor.log.info('Click Next_1_T2')752 AllCOUNT_Now = len(driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc"))753 if social_network_val=="youtube.com/":754 AllCOUNT_Now = len(driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]"))755 756 while (AllCOUNT_PREVIOUS == AllCOUNT_Now):757 time.sleep(1)758 tryresult = tryresult + 1;759 Actor.log.info('Click Next_1_T3_tryresult '+str(tryresult))760 if (tryresult > 20):761 driver.quit();762 break;763 764 AllCOUNT_Now = len(driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc"))765 if social_network_val=="youtube.com/" or social_network_val=="instagram.com/" :766 AllCOUNT_Now = len(driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]")) 767 print('AllCOUNT_PREVIOUS'+str(AllCOUNT_PREVIOUS))768 print('AllCOUNT_Now'+str(AllCOUNT_Now))769 if(AllCOUNT_PREVIOUS==AllCOUNT_Now):770 break 771 time.sleep(3)772 else:773 if(start==0):774 await Actor.push_data({'Email': 'No Data Found, Due to google not respond. May be proxy problem'})775 driver.quit();776 break;777 print('done')778 except Exception:779 Actor.log.exception(f'Cannot extract data from .') 780
781 driver.quit()# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed files.venv
# git folder.gitroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf# This file tells Git which files shouldn't be added to source control
.idea.DS_Store
apify_storagestorage
.venv/.env/__pypackages__dist/build/*.egg-info/*.egg
__pycache__
.mypy_cache.dmypy.jsondmypy.json.pytest_cache.ruff_cache
.scrapy*.log1# Feel free to add your Python dependencies below. For formatting guidelines, see:2# https://pip.pypa.io/en/latest/reference/requirements-file-format/3
4apify < 3.05selenium ~= 4.14.06blinker ~=1.7.07selenium-wire ~=5.1.08beautifulsoup4 ~= 4.12.29httpx ~= 0.25.210types-beautifulsoup4 ~= 4.12.0.711requests12undetected-chromedriver