Linkedin Email Scraper
Pricing
$10.00/month + usage
Linkedin Email Scraper- Scrap Emails from linkedin specific profile using google search engine
Pricing
$10.00/month + usage
Rating
3.0
(4)
Developer
bhansalisoft
Maintained by CommunityActor stats
74
Bookmarked
3.5K
Total users
17
Monthly active users
12 days ago
Last modified
Categories
Share
# First, specify the base Docker image.# You can see the Docker images from Apify at https://hub.docker.com/r/apify/.# You can also use any other image from Docker Hub.FROM apify/actor-python-selenium:3.11
# Second, copy just requirements.txt into the Actor image,# since it should be the only file that affects the dependency install in the next step,# in order to speed up the buildCOPY requirements.txt ./
# Install the packages specified in requirements.txt,# Print the installed Python version, pip version# and all installed packages with their versions for debuggingRUN echo "Python version:" \ && python --version \ && echo "Pip version:" \ && pip --version \ && echo "Installing dependencies:" \ && pip install -r requirements.txt \ && echo "All installed Python packages:" \ && pip freeze
# Next, copy the remaining files and directories with the source code.# Since we do this after installing the dependencies, quick build will be really fast# for most source file changes.COPY . ./
# Use compileall to ensure the runnability of the Actor Python code.RUN python3 -m compileall -q .
# Specify how to launch the source code of your Actor.# By default, the "python3 -m src" command is runCMD ["python3", "-m", "src"]{ "actorSpecification": 1, "name": "Linkedin Email Scraper", "title": "Linkedin Email Scraper", "description": "Scrap Emails from Linkedin social media profile using Search Engine", "version": "0.0", "meta": { "templateId": "python-selenium" }, "input": "./input_schema.json", "output": "./output_schema.json", "dockerfile": "./Dockerfile", "storages": { "dataset": { "actorSpecification": 1, "title": "Linkedin Email Scraper", "views": { "titles": { "title": "Linkedin Email Scraper", "transformation": { "fields": [ "Email", "title", "Description", "Detail_Link" ] }, "display": { "component": "table", "properties": { "Email": { "label": "Email", "format": "text" }, "title": { "label": "Title", "format": "text" } , "Description": { "label": "Description", "format": "text" } , "Detail_Link": { "label": "Detail_Link", "format": "text" } } } } } } }}{ "title": "Linkedin Email Scraper", "description": "Linkedin Email Scraper : Scrap Emails from Linkedin Profiles.", "type": "object", "schemaVersion": 1, "properties": { "Keyword": { "title": "Keyword", "description": "Keyword", "type": "string", "editor": "textfield", "prefill":"TestKeyword" }, "location": { "title": "Location(optional)", "description": "Location", "type": "string", "editor": "textfield" }, "social_network": { "title": "Social Network", "description": "Social Network", "type": "string", "editor": "select", "default": "linkedin.com/", "enum": ["linkedin.com/"], "enumTitles": ["Linkedin"] }, "Country": { "title": "Country", "description": "Country", "type": "string", "editor": "select", "default": "www", "enum": ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"], "enumTitles": ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"] }, "Email_Type": { "title": "Email Type", "description": "Email Type", "type": "string", "editor": "select", "default": "0", "enum": ["0", "1"], "enumTitles": ["Popular Emails only(@gmail.com or @yahoo.com or @hotmail.com)", "Other Private Email"] }, "Other_Email_Type": { "title": "Other Email Type(Example @domain.com)", "description": "Other Email Type(Example @domain.com)", "type": "string", "editor": "textfield" }, "Limit": { "title": "Max Number of Result", "description": "Max Number of Result : if this option is blank or 0 then it will scrap unlimited", "type": "string", "editor": "textfield", "prefill":"0" }, "proxySettings": { "title": "Proxy configuration", "type": "object", "description": "Select proxies to be used by your crawler.", "editor": "proxy", "prefill": { "useApifyProxy": false } }
}, "required": ["Keyword","social_network","Email_Type","Country"]}{ "actorOutputSchemaVersion": 1, "title": "Output schema", "properties": { "overview": { "type": "string", "title": "Overview", "template": "{{links.apiDefaultDatasetUrl}}/items?view=overview" } }}1"""2This module serves as the entry point for executing the Apify Actor. It handles the configuration of logging3settings. The `main()` coroutine is then executed using `asyncio.run()`.4
5Feel free to modify this file to suit your specific needs.6"""7
8import asyncio9import logging10
11from apify.log import ActorLogFormatter12
13from .main import main14
15# Configure loggers16handler = logging.StreamHandler()17handler.setFormatter(ActorLogFormatter())18
19apify_client_logger = logging.getLogger('apify_client')20apify_client_logger.setLevel(logging.INFO)21apify_client_logger.addHandler(handler)22
23apify_logger = logging.getLogger('apify')24apify_logger.setLevel(logging.DEBUG)25apify_logger.addHandler(handler)26
27# Execute the Actor main coroutine28asyncio.run(main())1# datascrapify.py2import json3from urllib.parse import urlparse4import requests5from apify import Actor6import time7processId='0'8api_key = 'nTYunJBzJ1Nc0R8n6I1'9async def StartProcess(camp_name, camp_type, keyword, Location,SocialMedia,Country,search_engine,Use_Proxy,proxy_settings):10 try:11
12 all_users = []13 14 url = 'https://api.datascrapify.com/api/Tool/EmailScraper'15 headers = {16 'Content-Type': 'application/json',17 'Accept': 'application/json',18 'x_api_key': api_key19 }20 21 payload_data = {22 "camp_name": camp_name,23 "camp_type": camp_type,24 "keyword": keyword,25 "Location": Location, # Note: This key is duplicated. Consider using "location" instead.26 "SocialMedia": SocialMedia,27 "Country": Country,28 "search_engine": search_engine,29 "Use_Proxy": Use_Proxy,30 }31 32 # Conditionally add the proxy settings only if a proxy is used33 if Use_Proxy:34 payload_data["_ProxySettings"] = proxy_settings35 36 payload = json.dumps(payload_data)37 print(payload)38 # Make the POST request39 response = requests.request("POST", url, headers=headers, data=payload)40 result = response.json()41 print(result)42 result = json.loads(result) 43 print(result)44 processId=result['processId']45 print(processId)46 if processId:47 print('Start Getting Result from Campaign')48 Fetch_True=True49 while Fetch_True :50 url='https://api.datascrapify.com/api/Tool/GetCampResult?processId='+processId51 response = requests.request("GET", url, headers=headers)52 result = response.json()53 print(result)54 result = json.loads(result)55 new_items = result['data']56 if new_items:57 #await Actor.push_data(new_items)58 for user in new_items:59 print(user)60 exists=False61 if len(all_users)>0: 62 for item in all_users:63 if item['uniqueid'] == user['uniqueid'] :64 exists=True65 break66 67 if exists==False:68 all_users.append({'uniqueid': user['uniqueid'], 'title': user['Name'],'Email':user['Email'],'Description':user['Description'],'Link':user['Link'] ,'Category':user['Category'], 'Website':user['Website'] });69 await Actor.push_data({'uniqueid': user['uniqueid'], 'title': user['Name'],'Email':user['Email'],'Description':user['Description'],'Detail_Link':user['Link'] ,'Category':user['Category'], 'Website':user['Website'] }) 70 71 if result['status']=='Complete':72 Fetch_True=False73 time.sleep(20) 74 75 else:76 print('We cannot process request') 77
78 except:79 Actor.log.info('Aborting process')80 #await on_aborting_event() 81 finally:82 Actor.log.info('Aborting process')83 #await on_aborting_event() 84 85 86 87async def on_aborting_event():88 headers = {89 'Content-Type': 'application/json',90 'Accept': 'application/json',91 'x_api_key': api_key92 }93 """94 This function will be called when the ABORTING event is emitted.95 Perform any necessary cleanup or state persistence here.96 """97 Actor.log.info('Actor is gracefully aborting. Performing cleanup...')98 # Example: Save current state to Key-Value store99 url='https://api.datascrapify.com/api/Tool/StopProcess?processId='+processId100 response = requests.request("GET", url, headers=headers)101 result = response.json()102 print(result)103 Actor.log.info('Cleanup complete.')104
105def parse_proxy_url(proxy_url_string):106 """107 Parses a proxy URL string and returns a dictionary of settings.108 """109 parsed = urlparse(proxy_url_string)110
111 if parsed.username and parsed.password:112 return {113 "ProxyServer": parsed.hostname,114 "ProxyPort": parsed.port,115 "ProxyAuthentification": True,116 "ProxyAuthLogin": parsed.username,117 "ProxyAuthPassword": parsed.password118 }119 elif parsed.hostname and parsed.port:120 return {121 "ProxyServer": parsed.hostname,122 "ProxyPort": parsed.port,123 "ProxyAuthentification": False,124 "ProxyAuthLogin": None,125 "ProxyAuthPassword": None126 }127 else:128 return None1"""2This module defines the `main()` coroutine for the Apify Actor, executed from the `__main__.py` file.3
4Feel free to modify this file to suit your specific needs.5
6To build Apify Actors, utilize the Apify SDK toolkit, read more at the official documentation:7https://docs.apify.com/sdk/python8"""9
10from urllib.parse import urljoin11from selenium.webdriver.common.action_chains import ActionChains12from apify import Actor13import re14from urllib.parse import urljoin15from bs4 import BeautifulSoup16import requests17import requests.exceptions18import time19import random20from .datascrapify import StartProcess, parse_proxy_url,on_aborting_event21from apify_shared.consts import ActorEventTypes as Event22import asyncio23from urllib.parse import urlparse, urlsplit, urlunsplit, parse_qs, urlencode24import os25
26# To run this Actor locally, you need to have the Selenium Chromedriver installed.27# https://www.selenium.dev/documentation/webdriver/getting_started/install_drivers/28# When running on the Apify platform, it is already included in the Actor's Docker image.29
30
31def scrape_contact_emails(link):32 res = requests.get(link)33 domain = link.split(".")34 mailaddr = link35 soup = BeautifulSoup(res.text,"lxml")36 links = soup.find_all("a")37 contact_link = ''38 final_result = ""39 try:40 # Check if there is any email address in the homepage. 41 emails = soup.find_all(text=re.compile('.*@'+domain[1]+'.'+domain[2].replace("/","")))42 emails.sort(key=len)43 print(emails[0].replace("\n",""))44 final_result = emails[0]45 except:46 # Searching for Contact Us Page's url.47 try:48 flag = 049 for link in links:50 if "contact" in link.get("href") or "Contact" in link.get("href") or "CONTACT" in link.get("href") or 'contact' in link.text or 'Contact' in link.text or 'CONTACT' in link.text:51 if len(link.get("href"))>2 and flag<2:52 flag = flag + 153 contact_link = link.get("href")54
55 except:56 pass57
58 domain = domain[0]+"."+domain[1]+"."+domain[2]59 if(len(contact_link)<len(domain)):60 domain = domain+contact_link.replace("/","")61 else:62 domain = contact_link63
64 try:65 # Check if there is any email address in the Contact Us Page. 66 res = requests.get(domain)67 soup = BeautifulSoup(res.text,"lxml")68 emails = soup.find_all(text=re.compile('.*@'+mailaddr[7:].replace("/","")))69 emails.sort(key=len)70 try:71 print(emails[0].replace("\n",""))72 final_result = emails[0]73 return final_result74 except:75 pass76 except Exception as e:77 pass78
79 return ""80
81async def main() -> None:82 """83 The main coroutine is being executed using `asyncio.run()`, so do not attempt to make a normal function84 out of it, it will not work. Asynchronous execution is required for communication with Apify platform,85 and it also enhances performance in the field of web scraping significantly.86 """87 async with Actor:88 # Read the Actor input89 actor_input = await Actor.get_input() or {}90 Keyword_val = actor_input.get('Keyword')91 location_val = actor_input.get('location')92 social_network_val = actor_input.get('social_network')93 Country_val = actor_input.get('Country')94 Email_Type_val = actor_input.get('Email_Type')95 Other_Email_Type_val = actor_input.get('Other_Email_Type')96 proxy_settings = actor_input.get('proxySettings')97 Limit_val =actor_input.get('Limit')98
99 #proxy_configuration = await Actor.create_proxy_configuration(actor_proxy_input=proxy_settings)100 proxy_configuration = await Actor.create_proxy_configuration(groups=['GOOGLE_SERP'])101 #Actor.log.info(proxy_configuration)102 proxyurl =await proxy_configuration.new_url()103 if proxy_configuration and proxy_settings:104 proxyurl =await proxy_configuration.new_url()105 #Actor.log.info(proxyurl)106 107
108 if not Keyword_val:109 Actor.log.info('Please insert keyword')110 await Actor.push_data({'Email': 'Please insert keyword'})111 await Actor.exit()112 return113 114 if Keyword_val=='TestKeyword':115 Actor.log.info('Please insert keyword')116 await Actor.push_data({'Email': 'Please insert Your Keyword'})117 await Actor.exit()118 return119 120 is_paying = os.getenv("APIFY_USER_IS_PAYING")121 print(is_paying) 122 if is_paying=="1":123 print('Paid User')124 else:125 Limit_val=10126 isPaying='FreeUser'127 await Actor.push_data({'Email': 'Free User Can Get Only 10 Records'})128 129 '''130 try:131 proxyurl=''132 USE_Proxy=False133 if proxyurl:134 USE_Proxy=True135 my_proxy_settings = parse_proxy_url(proxyurl) 136 # Call the function from the imported module137 await StartProcess(138 "Apify_camp_LinkedinEmail_"+username+"_"+isPaying, 139 "ALLINONE", 140 Keyword_val, 141 location_val, 142 social_network_val, 143 Country_val, 144 "Google",145 USE_Proxy, 146 my_proxy_settings147 )148 149 finally:150 # Code that always executes (e.g., cleanup)151 print("This block always runs, regardless of exceptions.")152
153 await Actor.exit();154 ''' 155 156 l1 = ["it","www","gm","fr","sp","uk","al","ag","ar","am","as","au","aj","bg","bo","be","bh","bk","br","bu","ca","ci","ch","co","cs","hr","cu","ez","dk","ec","eg","en","fi","gg","gr","hk","hu","ic","in","id","ir","iz","ei","is","jm","ja","ke","kn","ks","ku","lg","ly","ls","lh","lu","mc","mk","my","mt","mx","md","mn","mj","mo","np","nl","nz","ni","no","pk","we","pm","pa","pe","rp","pl","po","rq","qa","ro","rs","sm","sa","sg","ri","sn","lo","si","sf","sw","sz","sy","tw","th","ts","tu","ua","ae","uy","uz","ve"]157 l2= ["Italy","United States","Germany","France","Spain","United Kingdom","Albania","Algeria","Argentina","Armenia","Australia","Austria","Azerbaijan","Bangladesh","Belarus","Belgium","Belize","Bosnia and Herzegovina","Brazil","Bulgaria","Canada","Chile","China","Colombia","Costa Rica","Croatia","Cuba","Czechia","Denmark","Ecuador","Egypt","Estonia","Finland","Georgia","Greece","Hong Kong","Hungary","Iceland","India","Indonesia","Iran","Iraq","Ireland","Israel","Jamaica","Japan","Kenya","Korea","Korea, Republic of","Kuwait","Latvia","Libya","Liechtenstein","Lithuania","Luxembourg","Macao","Macedonia","Malaysia","Malta","Mexico","Moldova, Republic of","Monaco","Montenegro","Morocco","Nepal","Netherlands","New Zealand","Nigeria","Norway","Pakistan","Palestine, State of","Panama","Paraguay","Peru","Philippines","Poland","Portugal","Puerto Rico","Qatar","Romania","Russia","San Marino","Saudi Arabia","Senegal","Serbia","Singapore","Slovakia","Slovenia","South Africa","Sweden","Switzerland","Syrian Arab Republic","Taiwan","Thailand","Tunisia","Turkey","Ukraine","United Arab Emirates","Uruguay","Uzbekistan","Venezuela"]158 select_index=1159 select_country='United States'160 for count, ele in enumerate(l1):161 if(ele==Country_val):162 select_index=count163 break164 #print(count)165 #print(ele)166
167 for count, ele in enumerate(l2):168 if(count==select_index):169 select_country=ele170 break171
172 print(select_country) 173
174 concatstring = ""175 concatstring = concatstring + Keyword_val176 option = "( @gmail.com OR @hotmail.com OR @yahoo.com)";177 if Email_Type_val=="1":178 if not Other_Email_Type_val:179 Actor.log.info('Please insert Email Type Domain')180 await Actor.push_data({'Email': 'Please insert Email Type Domain'})181 await Actor.exit()182 return183 if Other_Email_Type_val.find("@") > -1:184 option = " ( " + Other_Email_Type_val + " )"185 else: 186 option = " ( @" + Other_Email_Type_val + " )" 187 concatstring = concatstring + option188 if location_val:189 concatstring = concatstring+ " in "+ location_val190 191 192 193 if social_network_val:194 concatstring = concatstring + " site:"195
196 if social_network_val == "linkedin.com/" or social_network_val == "pinterest.com/" :197 concatstring = concatstring + Country_val + ".";198 199
200 201
202 if social_network_val == "amazon.com/" :203 if Country_val=='gm':204 Country_val='de'205 elif Country_val=='sp':206 Country_val='es'207 elif Country_val=='fr':208 Country_val='fr' 209 elif Country_val=='uk':210 Country_val='co.uk'211 elif Country_val=='as':212 Country_val='com.au'213 elif Country_val=='www':214 Country_val='com' 215 elif Country_val=='in':216 Country_val='in'217 elif Country_val=='be':218 Country_val='com.be'219 elif Country_val=='br':220 Country_val='com.br'221 elif Country_val=='ca':222 Country_val='ca'223 elif Country_val=='ch':224 Country_val='cn' 225 elif Country_val=='eg': 226 Country_val='eg'227 elif Country_val=='it':228 Country_val='it' 229 elif Country_val=='ja':230 Country_val='co.jp'231 elif Country_val=='mx':232 Country_val='com.mx'233 elif Country_val=='nl':234 Country_val='nl'235 elif Country_val=='pl':236 Country_val='pl' 237 elif Country_val=='sa':238 Country_val='sa'239 elif Country_val=='sn':240 Country_val='sg' 241 elif Country_val=='sw':242 Country_val='se'243 elif Country_val=='tu':244 Country_val='com.tr' 245 elif Country_val=='ae':246 Country_val='ae' 247 elif Country_val=='ae':248 Country_val='ae'249 else : 250 Country_val=='com'251 252 social_network_val=social_network_val.replace('.com','.'+Country_val)253 254
255 concatstring = concatstring + "" + social_network_val + "";256 257 SearchEngine='Google'258 desktop_user_agents = ["Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36",259 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36",260 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.139 Safari/537.36",261 "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",262 # Firefox - Windows263 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:125.0) Gecko/20100101 Firefox/125.0",264 "Mozilla/5.0 (Windows NT 10.0; rv:124.0) Gecko/20100101 Firefox/124.0",265 "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:122.0) Gecko/20100101 Firefox/122.0",266 # Edge - Windows267 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.86 Safari/537.36 Edg/123.0.2420.65",268 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.112 Safari/537.36 Edg/122.0.2365.66",269 # Opera - Windows270 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36 OPR/96.0.0.0",271 # Chrome - macOS272 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_5_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.6261.129 Safari/537.36",273 "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_6_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",274 # Firefox - macOS275 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13.5; rv:124.0) Gecko/20100101 Firefox/124.0",276 "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:123.0) Gecko/20100101 Firefox/123.0",277 # Safari - macOS278 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.4 Safari/605.1.15",279 "Mozilla/5.0 (Macintosh; Intel Mac OS X 12_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15",280 # Chrome - Linux281 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.184 Safari/537.36",282 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",283 # Firefox - Linux284 "Mozilla/5.0 (X11; Linux x86_64; rv:125.0) Gecko/20100101 Firefox/125.0",285 "Mozilla/5.0 (X11; Linux x86_64; rv:124.0) Gecko/20100101 Firefox/124.0",286 "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0",287 # Safari - iMac (intel)288 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.3 Safari/605.1.15",289 # Edge - macOS290 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_4_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.86 Safari/537.36 Edg/123.0.2420.65",291 # Opera - macOS292 "Mozilla/5.0 (Macintosh; Intel Mac OS X 13_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36 OPR/96.0.0.0"293 ]294
295 mobile_user_agents = [296 # Chrome - Android297 "Mozilla/5.0 (Linux; Android 13; Pixel 7 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.105 Mobile Safari/537.36",298 "Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36",299 "Mozilla/5.0 (Linux; Android 11; Pixel 4a) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.6167.140 Mobile Safari/537.36",300
301 # Samsung Internet - Android302 "Mozilla/5.0 (Linux; Android 13; SAMSUNG SM-S911B) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/24.0 Chrome/123.0.0.0 Mobile Safari/537.36",303 "Mozilla/5.0 (Linux; Android 12; SAMSUNG SM-A525F) AppleWebKit/537.36 (KHTML, like Gecko) SamsungBrowser/23.0 Chrome/121.0.0.0 Mobile Safari/537.36",304
305 # Chrome - Xiaomi306 "Mozilla/5.0 (Linux; Android 13; M2101K6G) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.6312.105 Mobile Safari/537.36",307
308 # Safari - iPhone309 "Mozilla/5.0 (iPhone; CPU iPhone OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",310 "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.0 Mobile/15E148 Safari/604.1",311 "Mozilla/5.0 (iPhone; CPU iPhone OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Mobile/15E148 Safari/604.1",312
313 # Safari - iPad314 "Mozilla/5.0 (iPad; CPU OS 17_4 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Mobile/15E148 Safari/604.1",315 "Mozilla/5.0 (iPad; CPU OS 15_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.0 Mobile/15E148 Safari/604.1",316
317 # Chrome - iPhone318 "Mozilla/5.0 (iPhone; CPU iPhone OS 17_0 like Mac OS X) AppleWebKit/537.36 (KHTML, like Gecko) CriOS/123.0.0.0 Mobile/15E148 Safari/604.1",319
320 # Opera - Android321 "Mozilla/5.0 (Linux; Android 13; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Mobile Safari/537.36 OPR/76.0.4017.123",322
323 # Edge - Android324 "Mozilla/5.0 (Linux; Android 13; Pixel 6 Pro) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Mobile Safari/537.36 EdgA/123.0.2420.64",325
326 # Firefox - Android327 "Mozilla/5.0 (Android 13; Mobile; rv:124.0) Gecko/124.0 Firefox/124.0",328 "Mozilla/5.0 (Android 12; Mobile; rv:122.0) Gecko/122.0 Firefox/122.0"329] 330
331 332 all_user_agents = desktop_user_agents + mobile_user_agents333 random_user_agent = random.choice(all_user_agents)334
335
336 qry="https://google.com/search?q="+ concatstring337 Actor.log.info(concatstring)338
339 gl_map = {340 "it": "it",341 "www": "us",342 "gm": "de",343 "fr": "fr",344 "sp": "es",345 "uk": "uk",346 "al": "al",347 "ag": "dz",348 "ar": "ar",349 "am": "am",350 "as": "au",351 "au": "at",352 "aj": "az",353 "bg": "bd",354 "bo": "by",355 "be": "be",356 "bh": "bz",357 "bk": "ba",358 "br": "br",359 "bu": "bg",360 "ca": "ca",361 "ci": "cl",362 "ch": "cn",363 "co": "co",364 "cs": "cr",365 "hr": "hr",366 "cu": "cu",367 "ez": "ez",368 "dk": "dk",369 "ec": "ec",370 "eg": "eg",371 "en": "ee",372 "fi": "fi",373 "gg": "ge",374 "gr": "gr",375 "hk": "hk",376 "hu": "hu",377 "ic": "is",378 "in": "in",379 "id": "id",380 "ir": "ir",381 "iz": "iq",382 "ei": "ie",383 "is": "il",384 "jm": "jm",385 "ja": "jp",386
387 "ke": "ke",388 "kn": "kp",389 "ks": "kr",390 "ku": "kw",391 "lg": "lv",392 "ly": "ly",393 "ls": "li",394 "lh": "lt",395 "lu": "lu",396 "mc": "mo",397 "mk": "mk",398 "my": "my",399 "mt": "mt",400 "mx": "mx",401 "md": "md",402 "mn": "mc",403 "mj": "me",404 "mo": "ma",405 "np": "np",406 "nl": "nl",407 "nz": "nz",408 "ni": "ng",409 "no": "no",410 "pk": "pk",411 "we": "ps",412 "pm": "pa",413 "pa": "py",414 "pe": "pe",415 "rp": "ph",416 "pl": "pl",417 "po": "pt",418 "rq": "pr",419 "qa": "qa",420 "ro": "ro",421 "rs": "ru",422 "sm": "sm",423 "sa": "sa",424 "sg": "sn",425 "ri": "rs",426 "sn": "sg",427 "lo": "sk",428 "si": "si",429 "sf": "za",430 "sw": "se",431 "sz": "ch",432 "sy": "sy",433 "tw": "tw",434 "th": "th",435 "ts": "tn",436 "tu": "tu",437 "ua": "ua",438 "ae": "ae",439 "uy": "uy",440 "uz": "uz",441 "ve": "ve",442 "vn": "vn",443 "lk": "lk",444 }445
446 gl = gl_map.get(Country_val, Country_val)447
448
449 ##start New Logic450 all_users = []451 query =concatstring452 max_pages = 100453 results = []454 count_result=0455 has_next=True456 for page in range(max_pages):457 try:458 start = page * 10459 Actor.log.info('Check Page '+str(page))460 url = f"http://www.google.com/search?q={query}&num=10&hl=en&start={start}&gl={gl}"461
462 # Get fresh proxy URL for every request463 #proxy_url = await proxy_configuration.new_url()464 proxies=None465 if proxyurl:466 proxies = {'http': proxyurl, 'https': proxyurl}467
468 response = requests.get(469 url,470 proxies=proxies,471 headers={472 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "473 "AppleWebKit/537.36 (KHTML, like Gecko) "474 "Chrome/120.0.0.0 Safari/537.36"475 }476 )477
478 if response.status_code != 200:479 Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")480 continue481
482 soup = BeautifulSoup(response.text, "html.parser")483 result_blocks = soup.select("div.g, div.tF2Cxc")484
485 for j, block in enumerate(result_blocks):486 Email=''487 title_el = block.select_one("h3")488 link_el = block.select_one("a")489 snippet_el = block.select_one(".VwiC3b, .IsZvec, .aCOpRe")490 491 492
493 title= title_el.get_text(strip=True)494 url =link_el.get("href")495 snippet= snippet_el.get_text(strip=True)496 print(title) 497 alltext = title+snippet498 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', alltext) 499 Actor.log.info('match email '+str(len(match)))500 if len(match)>0:501 for i in match:502 Email=i503 Actor.log.info('email '+Email)504 else:505 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', alltext)506 for i in match_website:507 Website = "http://www."+i;508 Actor.log.info('Website '+Website)509 Email=scrape_contact_emails(Website)510 if Email :511 existindb=False512 #Actor.log.info('Email Store '+Email)513 if len(all_users)>0: 514 for item in all_users:515 if item['Email'] == Email :516 existindb=True517 break518 519 if existindb==False:520 all_users.append({'Email': Email}); 521 await Actor.push_data({'Email': Email, 'title': title,'Description':alltext,'Detail_Link':url})522 count_result=count_result+1523 if(Limit_val):524 if Limit_val!='0':525 if(count_result>=int(Limit_val)):526 has_next = False527 print('Limit Exceed')528 break529 # Random polite delay between pages530 if has_next==False:531 break532 await asyncio.sleep(random.uniform(1.5, 3.0))533 except:534 break535 #await Actor.exit()536 537 print('Check in Yahoo Search Engine')538 query=concatstring.replace("OR", "or")539 #has_next==True540 ## Yahoo Start541 for page in range(max_pages):542 try:543 if has_next==False:544 break545 start = page * 10546 Actor.log.info('Check Page '+str(page))547 #url = f"http://www.google.com/search?q={query}&num=10&hl=en&start={start}"548 url = "https://search.yahoo.com/search;_ylt=Awr.2lgGCQ9p1C0DekZXNyoA;_ylu=Y29sbwNncTEEcG9zAzEEdnRpZAMEc2VjA3BhZ2luYXRpb24-"549 params = {550 "p": query,551 "b": start,552 "pz": 10,553 "bct": 0,554 "xargs": 0555 }556
557 # Build request URL manually558 split_url = urlsplit(url)559 query_str = urlencode(params)560 full_url = urlunsplit((split_url.scheme, split_url.netloc, split_url.path, query_str, ""))561
562 print(f"Fetching page {full_url}")563 # Get fresh proxy URL for every request564 #proxy_url = await proxy_configuration.new_url()565 proxies=None566 #if proxyurl:567 # proxies = {'http': proxyurl, 'https': proxyurl}568
569 response = requests.get(570 full_url,571 proxies=proxies,572 headers={573 "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "574 "AppleWebKit/537.36 (KHTML, like Gecko) "575 "Chrome/120.0.0.0 Safari/537.36"576 }577 )578
579 if response.status_code != 200:580 Actor.log.warning(f"⚠️ Request failed: {response.status_code} {response.reason}")581 break 582
583 soup = BeautifulSoup(response.text, "html.parser")584 search_results = soup.select("div.dd.algo.algo-sr")585
586 if not search_results:587 log("⚠️ No more results found, ending.")588 break589
590 for r in search_results:591 try:592 Email='' 593 Website=''594 Address=''595 BusinessName=''596 DetailsLink=''597 Category=''598 link_tag = r.select_one("h3.title")599 link_tag1 = r.select_one("a")600 if link_tag:601 BusinessName = link_tag.get_text(strip=True)602 DetailsLink = link_tag1.get("href")603 Category = urlparse(DetailsLink).netloc604 605 addr_tag = r.select_one("div.compText.aAbs")606 if addr_tag:607 Address = addr_tag.get_text(separator=" ").strip()608
609 combined = r.get_text(strip=True)610 print(combined)611 match = re.findall(r'[a-zA-Z0-9\.\-+_]+@[a-zA-Z0-9\.\-+_]+\.[a-zA-Z]+', combined) 612 Actor.log.info('match email '+str(len(match)))613 if len(match)>0:614 for i in match:615 Email=i616 Actor.log.info('email '+Email)617 else:618 match_website = re.findall(r'\\b(?:https?://|www\\.)\\S+\\b', combined)619 for i in match_website:620 Website = "http://www."+i;621 Actor.log.info('Website '+Website)622 Email=scrape_contact_emails(Website)623 if Email :624 existindb=False625 #Actor.log.info('Email Store '+Email)626 if len(all_users)>0: 627 for item in all_users:628 if item['Email'] == Email :629 existindb=True630 break631 632 if existindb==False:633 all_users.append({'Email': Email}); 634 await Actor.push_data({'Email': Email, 'title': BusinessName,'Description':combined,'Detail_Link':DetailsLink})635 count_result=count_result+1636 if(Limit_val):637 if Limit_val!='0':638 if(count_result>=int(Limit_val)):639 has_next = False640 print('Limit Exceed')641 break642
643 except Exception as ex:644 print(f"⚠️ Parse error: {ex}")645 continue646 # Random polite delay between pages647 if has_next==False:648 break649 await asyncio.sleep(random.uniform(1.5, 3.0))650 except:651 #print(f"⚠️ Parse error: {ex}")652 break 653 await Actor.exit() 654 ##End New Logic# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed files.venv
# git folder.gitroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf# This file tells Git which files shouldn't be added to source control
.idea.DS_Store
apify_storagestorage
.venv/.env/__pypackages__dist/build/*.egg-info/*.egg
__pycache__
.mypy_cache.dmypy.jsondmypy.json.pytest_cache.ruff_cache
.scrapy*.log1# Feel free to add your Python dependencies below. For formatting guidelines, see:2# https://pip.pypa.io/en/latest/reference/requirements-file-format/3
4apify < 4.05beautifulsoup4 ~= 4.12.26requests