Pricing
$10.00/month + usage
Linkedin Email Scraper- Scrap Emails from linkedin specific profile using google search engine
1.0 (2)
Pricing
$10.00/month + usage
57
2.6K
225
Issues response
18 days
Last modified
8 days ago
# First, specify the base Docker image.# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.# You can also use any other image from Docker Hub.FROM apify/actor-python-selenium:3.11
# Second, copy just requirements.txt into the Actor image,# since it should be the only file that affects the dependency install in the next step,# in order to speed up the buildCOPY requirements.txt ./
# Install the packages specified in requirements.txt,# Print the installed Python version, pip version# and all installed packages with their versions for debuggingRUN echo "Python version:" \ && python --version \ && echo "Pip version:" \ && pip --version \ && echo "Installing dependencies:" \ && pip install -r requirements.txt \ && echo "All installed Python packages:" \ && pip freeze
# Next, copy the remaining files and directories with the source code.# Since we do this after installing the dependencies, quick build will be really fast# for most source file changes.COPY . ./
# Use compileall to ensure the runnability of the Actor Python code.RUN python3 -m compileall -q .
# Specify how to launch the source code of your Actor.# By default, the "python3 -m src" command is runCMD ["python3", "-m", "src"]{ "actorSpecification": 1, "name": "Linkedin Email Scraper", "title": "Linkedin Email Scraper", "description": "Scrap Emails from Linkedin social media profile using Search Engine", "version": "0.0", "meta": { "templateId": "python-selenium" }, "input": "./input_schema.json", "dockerfile": "./Dockerfile", "storages": { "dataset": { "actorSpecification": 1, "title": "Linkedin Email Scraper", "views": { "titles": { "title": "Linkedin Email Scraper", "transformation": { "fields": [ "Email", "title", "Description", "Detail_Link" ] }, "display": { "component": "table", "properties": { "Email": { "label": "Email", "format": "text" }, "title": { "label": "Title", "format": "text" } , "Description": { "label": "Description", "format": "text" } , "Detail_Link": { "label": "Detail_Link", "format": "text" } } } } } } }}{ "title": "Linkedin Email Scraper", "type": "object", "schemaVersion": 1, "properties": { "Keyword": { "title": "Keyword", "description": "Keyword", "type": "string", "editor": "textfield", "prefill":"TestKeyword" }, "location": { "title": "Location(optional)", "description": "Location", "type": "string", "editor": "textfield" }, "social_network": { "title": "Social Network", "description": "Social Network", "type": "string", "editor": "select", "default": "linkedin.com/", "enum": ["linkedin.com/"], "enumTitles": ["Linkedin"] }, "Country": { "title": "Country", "description": "Country", "type": "string", "editor": "select", "default": "www", "enum": ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"], "enumTitles": ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"] }, "Email_Type": { "title": "Email Type", "description": "Email Type", "type": "string", "editor": "select", "default": "0", "enum": ["0", "1"], "enumTitles": ["Popular Emails only(@gmail.com or @yahoo.com or @hotmail.com)", "Other Private Email"] }, "Other_Email_Type": { "title": "Other Email Type(Example @domain.com)", "description": "Other Email Type(Example @domain.com)", "type": "string", "editor": "textfield" }, "proxySettings": { "title": "Proxy configuration", "type": "object", "description": "Select proxies to be used by your crawler.", "editor": "proxy", "prefill": { "useApifyProxy": false } }
}, "required": ["Keyword","social_network","Email_Type","Country"]}1"""2This module serves as the entry point for executing the Apify Actor. It handles the configuration of logging3settings. The `main()` coroutine is then executed using `asyncio.run()`.4
5Feel free to modify this file to suit your specific needs.6"""7
8import asyncio9import logging10
11from apify.log import ActorLogFormatter12
13from .main import main14
15# Configure loggers16handler = logging.StreamHandler()17handler.setFormatter(ActorLogFormatter())18
19apify_client_logger = logging.getLogger('apify_client')20apify_client_logger.setLevel(logging.INFO)21apify_client_logger.addHandler(handler)22
23apify_logger = logging.getLogger('apify')24apify_logger.setLevel(logging.DEBUG)25apify_logger.addHandler(handler)26
27# Execute the Actor main coroutine28asyncio.run(main())1# datascrapify.py2import json3from urllib.parse import urlparse4import requests5from apify import Actor6import time7processId='0'8api_key = 'nTYunJBzJ1Nc0R8n6I1'9async def StartProcess(camp_name, camp_type, keyword, Location,SocialMedia,Country,search_engine,Use_Proxy,proxy_settings):10 try:11
12 all_users = []13 14 url = 'https://api.datascrapify.com/api/Tool/EmailScraper'15 headers = {16 'Content-Type': 'application/json',17 'Accept': 'application/json',18 'x_api_key': api_key19 }20 21 payload_data = {22 "camp_name": camp_name,23 "camp_type": camp_type,24 "keyword": keyword,25 "Location": Location, # Note: This key is duplicated. Consider using "location" instead.26 "SocialMedia": SocialMedia,27 "Country": Country,28 "search_engine": search_engine,29 "Use_Proxy": Use_Proxy,30 }31 32 # Conditionally add the proxy settings only if a proxy is used33 if Use_Proxy:34 payload_data["_ProxySettings"] = proxy_settings35 36 payload = json.dumps(payload_data)37 print(payload)38 # Make the POST request39 response = requests.request("POST", url, headers=headers, data=payload)40 result = response.json()41 print(result)42 result = json.loads(result) 43 print(result)44 processId=result['processId']45 print(processId)46 if processId:47 print('Start Getting Result from Campaign')48 Fetch_True=True49 while Fetch_True :50 url='https://api.datascrapify.com/api/Tool/GetCampResult?processId='+processId51 response = requests.request("GET", url, headers=headers)52 result = response.json()53 print(result)54 result = json.loads(result)55 new_items = result['data']56 if new_items:57 #await Actor.push_data(new_items)58 for user in new_items:59 print(user)60 exists=False61 if len(all_users)>0: 62 for item in all_users:63 if item['uniqueid'] == user['uniqueid'] :64 exists=True65 break66 67 if exists==False:68 all_users.append({'uniqueid': user['uniqueid'], 'title': user['Name'],'Email':user['Email'],'Description':user['Description'],'Link':user['Link'] ,'Category':user['Category'], 'Website':user['Website'] });69 await Actor.push_data({'uniqueid': user['uniqueid'], 'title': user['Name'],'Email':user['Email'],'Description':user['Description'],'Detail_Link':user['Link'] ,'Category':user['Category'], 'Website':user['Website'] }) 70 71 if result['status']=='Complete':72 Fetch_True=False73 time.sleep(20) 74 75 else:76 print('We cannot process request') 77
78 except:79 Actor.log.info('Aborting process')80 #await on_aborting_event() 81 finally:82 Actor.log.info('Aborting process')83 #await on_aborting_event() 84 85 86 87async def on_aborting_event():88 headers = {89 'Content-Type': 'application/json',90 'Accept': 'application/json',91 'x_api_key': api_key92 }93 """94 This function will be called when the ABORTING event is emitted.95 Perform any necessary cleanup or state persistence here.96 """97 Actor.log.info('Actor is gracefully aborting. Performing cleanup...')98 # Example: Save current state to Key-Value store99 url='https://api.datascrapify.com/api/Tool/StopProcess?processId='+processId100 response = requests.request("GET", url, headers=headers)101 result = response.json()102 print(result)103 Actor.log.info('Cleanup complete.')104
105def parse_proxy_url(proxy_url_string):106 """107 Parses a proxy URL string and returns a dictionary of settings.108 """109 parsed = urlparse(proxy_url_string)110
111 if parsed.username and parsed.password:112 return {113 "ProxyServer": parsed.hostname,114 "ProxyPort": parsed.port,115 "ProxyAuthentification": True,116 "ProxyAuthLogin": parsed.username,117 "ProxyAuthPassword": parsed.password118 }119 elif parsed.hostname and parsed.port:120 return {121 "ProxyServer": parsed.hostname,122 "ProxyPort": parsed.port,123 "ProxyAuthentification": False,124 "ProxyAuthLogin": None,125 "ProxyAuthPassword": None126 }127 else:128 return None1"""2This module defines the `main()` coroutine for the Apify Actor, executed from the `__main__.py` file.3
4Feel free to modify this file to suit your specific needs.5
6To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:7https://docs.apify.com/sdk/python8"""9
10from urllib.parse import urljoin11from seleniumwire import webdriver12#from selenium.webdriver.chrome.options import Options as ChromeOptions13from selenium.webdriver.common.by import By14from selenium.webdriver.common.action_chains import ActionChains15from apify import Actor16import re17from urllib.parse import urljoin18from bs4 import BeautifulSoup19import requests20import requests.exceptions21import time22import undetected_chromedriver as uc23import random24from .datascrapify import StartProcess, parse_proxy_url,on_aborting_event25from apify_shared.consts import ActorEventTypes as Event26import asyncio27
28# To run this Actor locally, you need to have the Selenium Chromedriver installed.29# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/30# When running on the Apify platform, it is already included in the Actor's Docker image.31
32
33def scrape_contact_emails(link):34 res = requests.get(link)35 domain = link.split(".")36 mailaddr = link37 soup = BeautifulSoup(res.text,"lxml")38 links = soup.find_all("a")39 contact_link = ''40 final_result = ""41 try:42 # Check if there is any email address in the homepage. 43 emails = soup.find_all(text=re.compile('.*@'+domain[1]+'.'+domain[2].replace("/","")))44 emails.sort(key=len)45 print(emails[0].replace("\n",""))46 final_result = emails[0]47 except:48 # Searching for Contact Us Page's url.49 try:50 flag = 051 for link in links:52 if "contact" in link.get("href") or "Contact" in link.get("href") or "CONTACT" in link.get("href") or 'contact' in link.text or 'Contact' in link.text or 'CONTACT' in link.text:53 if len(link.get("href"))>2 and flag<2:54 flag = flag + 155 contact_link = link.get("href")56
57 except:58 pass59
60 domain = domain[0]+"."+domain[1]+"."+domain[2]61 if(len(contact_link)<len(domain)):62 domain = domain+contact_link.replace("/","")63 else:64 domain = contact_link65
66 try:67 # Check if there is any email address in the Contact Us Page. 68 res = requests.get(domain)69 soup = BeautifulSoup(res.text,"lxml")70 emails = soup.find_all(text=re.compile('.*@'+mailaddr[7:].replace("/","")))71 emails.sort(key=len)72 try:73 print(emails[0].replace("\n",""))74 final_result = emails[0]75 return final_result76 except:77 pass78 except Exception as e:79 pass80
81 return ""82
83async def main() -> None:84 """85 The main coroutine is being executed using `asyncio.run()`, so do not attempt to make a normal function86 out of it, it will not work. Asynchronous execution is required for communication with Apify platform,87 and it also enhances performance in the field of web scraping significantly.88 """89 async with Actor:90 # Read the Actor input91 actor_input = await Actor.get_input() or {}92 Keyword_val = actor_input.get('Keyword')93 location_val = actor_input.get('location')94 social_network_val = actor_input.get('social_network')95 Country_val = actor_input.get('Country')96 Email_Type_val = actor_input.get('Email_Type')97 Other_Email_Type_val = actor_input.get('Other_Email_Type')98 proxy_settings = actor_input.get('proxySettings')99 #proxy_configuration = await Actor.create_proxy_configuration(actor_proxy_input=proxy_settings)100 proxy_configuration = await Actor.create_proxy_configuration(groups=['GOOGLE_SERP'])101 #Actor.log.info(proxy_configuration)102 proxyurl = ''103 if proxy_configuration and proxy_settings:104 proxyurl =await proxy_configuration.new_url()105 #Actor.log.info(proxyurl)106 107
108 if not Keyword_val:109 Actor.log.info('Please insert keyword')110 await Actor.push_data({'Email': 'Please insert keyword'})111 await Actor.exit()112 return113 114 if Keyword_val=='TestKeyword':115 Actor.log.info('Please insert keyword')116 await Actor.push_data({'Email': 'Please insert Your Keyword'})117 await Actor.exit()118 return119 '''120 me = await Actor.apify_client.user('me').get()121 username=me["username"]122 isPaying='PayingUser'123 if(me["isPaying"]==False):124 isPaying='FreeUser'125
126 try:127 proxyurl=''128 USE_Proxy=False129 if proxyurl:130 USE_Proxy=True131 my_proxy_settings = parse_proxy_url(proxyurl) 132 # Call the function from the imported module133 await StartProcess(134 "Apify_camp_LinkedinEmail_"+username+"_"+isPaying, 135 "ALLINONE", 136 Keyword_val, 137 location_val, 138 social_network_val, 139 Country_val, 140 "Google",141 USE_Proxy, 142 my_proxy_settings143 )144 145 finally:146 # Code that always executes (e.g., cleanup)147 print("This block always runs, regardless of exceptions.")148
149 await Actor.exit();150 ''' 151 152 l1 = ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"]153 l2= ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"]154 select_index=1155 select_country='United States'156 for count, ele in enumerate(l1):157 if(ele==Country_val):158 select_index=count159 break160 #print(count)161 #print(ele)162
163 for count, ele in enumerate(l2):164 if(count==select_index):165 select_country=ele166 break167
168 print(select_country) 169
170 concatstring = ""171 concatstring = concatstring + Keyword_val172 option = "( @gmail.com OR @hotmail.com OR @yahoo.com)";173 if Email_Type_val=="1":174 if not Other_Email_Type_val:175 Actor.log.info('Please insert Email Type Domain')176 await Actor.push_data({'Email': 'Please insert Email Type Domain'})177 await Actor.exit()178 return179 if Other_Email_Type_val.find("@") > -1:180 option = " ( " + Other_Email_Type_val + " )"181 else: 182 option = " ( @" + Other_Email_Type_val + " )" 183 concatstring = concatstring + option184 if location_val:185 concatstring = concatstring+ " in "+ location_val186 187 188 189 if social_network_val:190 concatstring = concatstring + " site:"191
192 if social_network_val == "linkedin.com/" or social_network_val == "pinterest.com/" :193 concatstring = concatstring + Country_val + ".";194 195
196 197
198 if social_network_val == "amazon.com/" :199 if Country_val=='gm':200 Country_val='de'201 elif Country_val=='sp':202 Country_val='es'203 elif Country_val=='fr':204 Country_val='fr' 205 elif Country_val=='uk':206 Country_val='co.uk'207 elif Country_val=='as':208 Country_val='com.au'209 elif Country_val=='www':210 Country_val='com' 211 elif Country_val=='in':212 Country_val='in'213 elif Country_val=='be':214 Country_val='com.be'215 elif Country_val=='br':216 Country_val='com.br'217 elif Country_val=='ca':218 Country_val='ca'219 elif Country_val=='ch':220 Country_val='cn' 221 elif Country_val=='eg': 222 Country_val='eg'223 elif Country_val=='it':224 Country_val='it' 225 elif Country_val=='ja':226 Country_val='co.jp'227 elif Country_val=='mx':228 Country_val='com.mx'229 elif Country_val=='nl':230 Country_val='nl'231 elif Country_val=='pl':232 Country_val='pl' 233 elif Country_val=='sa':234 Country_val='sa'235 elif Country_val=='sn':236 Country_val='sg' 237 elif Country_val=='sw':238 Country_val='se'239 elif Country_val=='tu':240 Country_val='com.tr' 241 elif Country_val=='ae':242 Country_val='ae' 243 elif Country_val=='ae':244 Country_val='ae'245 else : 246 Country_val=='com'247 248 social_network_val=social_network_val.replace('.com','.'+Country_val)249 250
251 concatstring = concatstring + "" + social_network_val + "";252 253 SearchEngine='Google'254 desktop_user_agents = ["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",255 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",256 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.139 Safari/537.36",257 "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",258 # Firefox - Windows259 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",260 "Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0",261 "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0",262 # Edge - Windows263 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.86 Safari/537.36 Edg/123.0.2420.65",264 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.112 Safari/537.36 Edg/122.0.2365.66",265 # Opera - Windows266 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 OPR/96.0.0.0",267 # Chrome - macOS268 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.129 Safari/537.36",269 "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",270 # Firefox - macOS271 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13.5; rv:124.0) Gecko/20100101 Firefox/124.0",272 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0",273 # Safari - macOS274 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",275 "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",276 # Chrome - Linux277 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.184 Safari/537.36",278 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",279 # Firefox - Linux280 "Mozilla/5.0 (X11; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",281 "Mozilla/5.0 (X11; Linux x86_64; rv:124.0) Gecko/20100101 Firefox/124.0",282 "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0",283 # Safari - iMac (intel)284 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15",285 # Edge - macOS286 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.86 Safari/537.36 Edg/123.0.2420.65",287 # Opera - macOS288 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/96.0.0.0"289 ]290
291 mobile_user_agents = [292 # Chrome - Android293 "Mozilla/5.0 (Linux; Android 13; Pixel 7 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.105 Mobile Safari/537.36",294 "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36",295 "Mozilla/5.0 (Linux; Android 11; Pixel 4a) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.140 Mobile Safari/537.36",296
297 # Samsung Internet - Android298 "Mozilla/5.0 (Linux; Android 13; SAMSUNG SM-S911B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/24.0 Chrome/123.0.0.0 Mobile Safari/537.36",299 "Mozilla/5.0 (Linux; Android 12; SAMSUNG SM-A525F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/23.0 Chrome/121.0.0.0 Mobile Safari/537.36",300
301 # Chrome - Xiaomi302 "Mozilla/5.0 (Linux; Android 13; M2101K6G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.105 Mobile Safari/537.36",303
304 # Safari - iPhone305 "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",306 "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",307 "Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/604.1",308
309 # Safari - iPad310 "Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",311 "Mozilla/5.0 (iPad; CPU OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",312
313 # Chrome - iPhone314 "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) CriOS/123.0.0.0 Mobile/15E148 Safari/604.1",315
316 # Opera - Android317 "Mozilla/5.0 (Linux; Android 13; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36 OPR/76.0.4017.123",318
319 # Edge - Android320 "Mozilla/5.0 (Linux; Android 13; Pixel 6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Mobile Safari/537.36 EdgA/123.0.2420.64",321
322 # Firefox - Android323 "Mozilla/5.0 (Android 13; Mobile; rv:124.0) Gecko/124.0 Firefox/124.0",324 "Mozilla/5.0 (Android 12; Mobile; rv:122.0) Gecko/122.0 Firefox/122.0"325] 326
327 328 all_user_agents = desktop_user_agents + mobile_user_agents329 random_user_agent = random.choice(all_user_agents)330
331
332 qry="https://google.com/search?q="+ concatstring333 Actor.log.info(concatstring)334
335
336 ##start New Logic337 all_users = []338 query =concatstring339 max_pages = 100340 results = []341 for page in range(max_pages):342 start = page * 10343 url = f"http://www.google.com/search?q={query}&num=10&hl=en&start={start}"344
345 # Get fresh proxy URL for every request346 #proxy_url = await proxy_configuration.new_url()347 proxies=None348 if proxyurl:349 proxies = {'http': proxyurl, 'https': proxyurl}350
351 response = requests.get(352 url,353 proxies=proxies,354 headers={355 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "356 "AppleWebKit/537.36 (KHTML, like Gecko) "357 "Chrome/120.0.0.0 Safari/537.36"358 },359 timeout=30,360 )361
362 if response.status_code != 200:363 Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")364 continue365
366 soup = BeautifulSoup(response.text, "html.parser")367 result_blocks = soup.select("div.g, div.tF2Cxc")368
369 for j, block in enumerate(result_blocks):370 Email=''371 title_el = block.select_one("h3")372 link_el = block.select_one("a")373 snippet_el = block.select_one(".VwiC3b, .IsZvec, .aCOpRe")374 375 376
377 title= title_el.get_text(strip=True)378 url =link_el.get("href")379 snippet= snippet_el.get_text(strip=True)380 print(title) 381 alltext = title+snippet382 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', alltext) 383 Actor.log.info('match email '+str(len(match)))384 if len(match)>0:385 for i in match:386 Email=i387 Actor.log.info('email '+Email)388 else:389 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', alltext)390 for i in match_website:391 Website = "http://www."+i;392 Actor.log.info('Website '+Website)393 Email=scrape_contact_emails(Website)394 if Email :395 existindb=False396 #Actor.log.info('Email Store '+Email)397 if len(all_users)>0: 398 for item in all_users:399 if item['Email'] == Email :400 existindb=True401 break402 403 if existindb==False:404 all_users.append({'Email': Email}); 405 await Actor.push_data({'Email': Email, 'title': title,'Description':alltext,'Detail_Link':url})406
407 # Random polite delay between pages408 await asyncio.sleep(random.uniform(1.5, 3.0))409 410 await Actor.exit() 411 ##End New Logic412 413 # Launch a new Selenium Chrome WebDriver414 Actor.log.info('Launching Chrome WebDriver...')415 chrome_options = uc.ChromeOptions()416 chrome_options.add_argument(f'user-agent={random_user_agent}')417 chrome_options.add_argument("--disable-blink-features=AutomationControlled")418 #chrome_options.add_extension('./NopeCHA-CAPTCHA-Solver.crx')419 #chrome_options.add_argument("user-agent=Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36")420 chrome_options.add_argument('--no-sandbox')421 chrome_options.add_argument('--disable-dev-shm-usage')422 chrome_options.add_argument("--disable-blink-features=AutomationControlled")423 #options.add_experimental_option("useAutomationExtension", False)424 # chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])425 #chrome_options.add_experimental_option('useAutomationExtension', False)426 427 seleniumwire_options={}428 if(proxyurl):429 print('apply proxy')430 chrome_options.add_argument(f" - proxy-server={proxyurl}")431 #seleniumwire_options = {'proxy': { 'https': proxyurl,'https': proxyurl} }432 # Initialize SeleniumAuthenticatedProxy433 #proxy_helper = SeleniumAuthenticatedProxy(proxy_url=proxyurl)434 # Enrich Chrome options with proxy authentication435 #proxy_helper.enrich_chrome_options(chrome_options)436
437 #print(seleniumwire_options)438 #driver = webdriver.Chrome(options=chrome_options,seleniumwire_options=seleniumwire_options)439 driver = uc.Chrome(options=chrome_options, use_subprocess=False,version_main = 137)440 driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {441 "source": """442 Object.defineProperty(navigator, 'webdriver', {443 get: () => undefined444 })445 """446 })447
448 #driver.get('https://nopecha.com/setup#aqoz34cu2ob3qmhn')449 #time.sleep(3)450 #print("key : "+driver.execute_script("return document.querySelectorAll('body')[0].innerText"))451
452 # Visit the test URL to check your proxy IP453 driver.get("https://httpbin.io/ip")454 # Select the body tag containing the current IP address455 ip_address = driver.find_element(By.TAG_NAME, "body").text456 # Print your current IP457 print(ip_address)458 try:459 driver.get(qry)460 except Exception as e:461 print(e)462 await Actor.push_data({'Email': e})463 driver.quit()464 465 checkount=0466 try:467 while(driver.current_url.find("sorry/index")>-1 and checkount<100):468 print('captcha')469 print(driver.current_url)470 time.sleep(3)471 checkount=checkount+1472 except Exception as e:473 print(e) 474
475 all_users = []476 start=0477
478 if(SearchEngine=='Yahoo'):479 print('yahoo')480 else:481 try:482 while True:483 consent = driver.find_elements(By.XPATH,"//iframe[contains(@src, 'consent.google.com')]")484 if len(consent)>0 :485 driver.switch_to.frame(consent[0]);486 driver.find_element(By.id("id")).click();487
488 popup = driver.find_elements(By.CSS_SELECTOR,"button[class='yt-spec-button-shape-next yt-spec-button-shape-next--filled yt-spec-button-shape-next--mono yt-spec-button-shape-next--size-m']");489 if (len(popup) > 0):490 Actor.log.info("popup") 491 for n in range(0,len(popup)):492 if (popup[n].text=="Accept all"):493 popup[n].click();494 495 popup_accept = driver.find_elements(By.CSS_SELECTOR,"button[id='L2AGLb']");496 if (len(popup_accept) > 0):497 popup_accept[0].click();498 time.sleep(1)499 500 checkconsent=0;501 while len(driver.find_elements(By.CSS_SELECTOR,"div[class='HTjtHe'][style*='display: block']")) > 0 and checkconsent<100:502 Actor.log.info('checkconsent '+str(checkconsent))503 Actor.log.info("homeurl" + driver.current_url) 504 checkconsent=checkconsent+1505 time.sleep(1)506 507 AllClasses = driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc") 508 print(len(AllClasses)) 509 #checkconsent_youtube=0; 510 #while len(driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]")) <= 0 and checkconsent_youtube<20:511 #Actor.log.info('checkconsent_youtube '+str(checkconsent_youtube))512 #Actor.log.info("homeurl" + driver.current_url) 513 #checkconsent_youtube=checkconsent_youtube+1514 #time.sleep(1)515 homeurl = driver.current_url516 AllClasses = driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc")517 if social_network_val=="youtube.com/" or social_network_val=="instagram.com/":518 AllClasses = driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]")519 Actor.log.info("homeurl" + driver.current_url)520
521 checkount=0522 try:523 while(driver.current_url.index("sorry/index")>-1 and checkount<100):524 print('captcha')525 time.sleep(3)526 checkount=checkount+1527 except Exception as e:528 print(e) 529
530 if(len(AllClasses)==0):531 print('check new attribut')532 AllClasses = driver.find_elements(By.CSS_SELECTOR,"div.fP1Qef")533 print(len(AllClasses))534 print(driver.find_elements(By.CSS_SELECTOR,"div[id='main']"))535 536 #res = requests.get(driver.current_url)537 #await Actor.push_data({'Email': res.text})538 if len(AllClasses)>0:539 start=1540 Actor.log.info("Result" + str(len(AllClasses)))541 for gr in range(0,len(AllClasses)):542 try:543 BusinessName=""544 DetailsLink=""545 Email=""546 Address=""547 businessdetail = AllClasses[gr].find_elements(By.CSS_SELECTOR,"h3.LC20lb")548 if len(businessdetail) > 0:549 BusinessName = businessdetail[0].text;550 DetailsLink = businessdetail[0].find_element(By.XPATH,"parent::*").get_attribute("href")551 if not DetailsLink:552 businessdetailnew = AllClasses[gr].find_elements(By.CSS_SELECTOR,"div.TbwUpd")553 if len(businessdetailnew) > 0:554 DetailsLink = businessdetailnew[0].find_elements(By.XPATH,"parent::*").get_attribute("href")555 Ele_addressdetail = AllClasses[gr].find_elements(By.CSS_SELECTOR,"div.VwiC3b")556 if len(Ele_addressdetail) > 0:557 Address = Ele_addressdetail[0].text.replace(";", "-").replace(",", "-");558 559 alltext = AllClasses[gr].text560 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', alltext) 561 Actor.log.info('match email '+str(len(match)))562 if len(match)>0:563 for i in match:564 Email=i565 Actor.log.info('email '+Email)566 else:567 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', BusinessName+Address)568 for i in match_website:569 Website = "http://www."+i;570 Actor.log.info('Website '+Website)571 Email=scrape_contact_emails(Website)572 if Email :573 existindb=False574 #Actor.log.info('Email Store '+Email)575 if len(all_users)>0: 576 for item in all_users:577 if item['Email'] == Email :578 existindb=True579 break580 if existindb==False:581 all_users.append({'Email': Email}); 582 await Actor.push_data({'Email': Email, 'title': BusinessName,'Description':alltext,'Detail_Link':DetailsLink})583
584 585 except Exception as err:586 Actor.log.info(f"Unexpected {err=}, {type(err)=}") 587
588 if len(driver.find_elements(By.CSS_SELECTOR,"a#pnnext")) > 0:589 Actor.log.info('Click Next')590 str_url = driver.current_url591 #driver.find_elements(By.CSS_SELECTOR,"a#pnnext")[0].click();592 driver.execute_script("arguments[0].click();", driver.find_elements(By.CSS_SELECTOR,"a#pnnext")[0]);593 time.sleep(3)594 while driver.current_url == str_url:595 tryresult = 0;596 time.sleep(1);597 tryresult = tryresult + 1;598 if tryresult > 20:599 driver.quit();600 break;601 else:602 Actor.log.info('Click Next_1')603 AllCOUNT_PREVIOUS = len(driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc"))604 if social_network_val=="youtube.com/" or social_network_val=="instagram.com/":605 AllCOUNT_PREVIOUS = len(driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]"))606 Actor.log.info('Click Next_1_T1')607 str_url = driver.current_url608 action = ActionChains(driver)609 # perform the operation 610 #action.move_to_element(element).click().perform()611 tryresult = 0;612 if len(driver.find_elements(By.CSS_SELECTOR,"span.RVQdVd")) > 0:613 action = ActionChains(driver)614 selectedlink1 = driver.find_element(By.CSS_SELECTOR,"span.RVQdVd")615 action.move_to_element(selectedlink1).click().perform()616 617 time.sleep(3)618 Actor.log.info('Click Next_1_T2')619 AllCOUNT_Now = len(driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc"))620 if social_network_val=="youtube.com/":621 AllCOUNT_Now = len(driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]"))622 623 while (AllCOUNT_PREVIOUS == AllCOUNT_Now):624 time.sleep(1)625 tryresult = tryresult + 1;626 Actor.log.info('Click Next_1_T3_tryresult '+str(tryresult))627 if (tryresult > 20):628 driver.quit();629 break;630 631 AllCOUNT_Now = len(driver.find_elements(By.CSS_SELECTOR,"div.tF2Cxc"))632 if social_network_val=="youtube.com/" or social_network_val=="instagram.com/" :633 AllCOUNT_Now = len(driver.find_elements(By.XPATH,"//div[contains(@class,'MjjYud')]")) 634 print('AllCOUNT_PREVIOUS'+str(AllCOUNT_PREVIOUS))635 print('AllCOUNT_Now'+str(AllCOUNT_Now))636 if(AllCOUNT_PREVIOUS==AllCOUNT_Now):637 break 638 time.sleep(3)639 else:640 if(start==0):641 await Actor.push_data({'Email': 'No Data Found, Due to google not respond. May be proxy problem'})642 driver.quit();643 break;644 print('done')645 except Exception:646 Actor.log.exception(f'Cannot extract data from .') 647
648 driver.quit()# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed files.venv
# git folder.gitroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf# This file tells Git which files shouldn't be added to source control
.idea.DS_Store
apify_storagestorage
.venv/.env/__pypackages__dist/build/*.egg-info/*.egg
__pycache__
.mypy_cache.dmypy.jsondmypy.json.pytest_cache.ruff_cache
.scrapy*.log1# Feel free to add your Python dependencies below. For formatting guidelines, see:2# https://pip.pypa.io/en/latest/reference/requirements-file-format/3
4apify < 3.05selenium ~= 4.14.06blinker ~=1.7.07selenium-wire ~=5.1.08beautifulsoup4 ~= 4.12.29httpx ~= 0.25.210types-beautifulsoup4 ~= 4.12.0.711requests12undetected-chromedriver