Expert Finder Experts Scraper
Pricing
from $4.99 / 1,000 results
Search experts by keyword and extract full profile details including expertise, bio, contact info, and links. 🔎
Pricing
from $4.99 / 1,000 results
Rating
0.0
(0)
Developer
EasyApi
Maintained by CommunityActor stats
1
Bookmarked
2
Total users
1
Monthly active users
3 days ago
Last modified
Share
# Specify the base Docker image. You can read more about# the available images at https://docs.apify.com/sdk/js/docs/guides/docker-images# You can also use any other image from Docker Hub.FROM apify/actor-node-puppeteer-chrome# Check preinstalled packagesRUN npm ls crawlee apify puppeteer playwright
# Copy just package.json and package-lock.json# to speed up the build using Docker layer cache.COPY package*.json ./
# Install NPM packages, skip optional and development dependencies to# keep the image small. Avoid logging too much and print the dependency# tree for debuggingRUN npm --quiet set progress=false \ && npm install --omit=dev --omit=optional \ && echo "Installed NPM packages:" \ && (npm list --omit=dev --all || true) \ && echo "Node.js version:" \ && node --version \ && echo "NPM version:" \ && npm --version \ && rm -r ~/.npm
# Next, copy the remaining files and directories with the source code.# Since we do this after NPM install, quick build will be really fast# for most source file changes.COPY . ./
# Run the image.CMD npm start --silent{ "actorSpecification": 1, "name": "expert-finder-experts-scraper", "title": "Empty JavaScript project", "description": "Empty project in JavaScript.", "version": "0.0", "buildTag": "latest", "meta": { "templateId": "js-empty" }, "input": "./input_schema.json", "dockerfile": "./Dockerfile"}{ "title": "Expert Finder Experts Scraper Input", "type": "object", "schemaVersion": 1, "properties": { "keyword": { "title": "Keyword", "type": "string", "description": "Keyword to search experts by name or expertise", "editor": "textfield", "default": "financial", "prefill": "financial" }, "maxItems": { "title": "Maximum Results", "type": "integer", "description": "Maximum number of expert profiles to scrape", "default": 100, "prefill": 100, "minimum": 1, "maximum": 50000 }, "proxyConfiguration": { "title": "Proxy Configuration", "type": "object", "description": "Proxy settings for HTTP requests", "default": { "useApifyProxy": false }, "editor": "proxy" } }, "required": ["keyword"]}1import { Actor } from 'apify';2import axios from 'axios';3import * as cheerio from 'cheerio';4
5const SITE_BASE = 'https://network.expertisefinder.com';6
7const DEFAULT_HEADERS = {8 accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',9 'accept-language': 'en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7',10 'cache-control': 'no-cache',11 pragma: 'no-cache',12 'user-agent':13 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/148.0.0.0 Safari/537.36',14};15
16class ExpertFinderExpertsScraper {17 getAxiosConfig(proxyUrl) {18 if (!proxyUrl) return {};19
20 const parsed = new URL(proxyUrl);21 return {22 proxy: {23 protocol: parsed.protocol.replace(':', ''),24 host: parsed.hostname,25 port: Number(parsed.port),26 ...(parsed.username27 ? {28 auth: {29 username: decodeURIComponent(parsed.username),30 password: decodeURIComponent(parsed.password),31 },32 }33 : {}),34 },35 };36 }37
38 buildSearchUrl(keyword, page = 1) {39 const params = new URLSearchParams({ query: String(keyword).trim().toLowerCase() });40 if (page > 1) {41 params.set('page', String(page));42 }43 return `${SITE_BASE}/search-experts?${params.toString()}`;44 }45
46 buildProfileUrl(path) {47 if (!path) return null;48 if (path.startsWith('http')) return path;49 return `${SITE_BASE}${path.startsWith('/') ? path : `/${path}`}`;50 }51
52 normalizeText(value) {53 return String(value || '').replace(/\s+/g, ' ').trim() || null;54 }55
56 parsePagination($) {57 const paginateText = $('#desktop_paginate').text().replace(/\s+/g, ' ');58 const match = paginateText.match(/Page\s+(\d+)\s+of\s+(\d+)/i);59
60 const hasNextPage = $('#desktop_paginate a[href*="page="]').toArray().some((element) => {61 const link = $(element);62 const text = link.text().replace(/\s+/g, ' ').trim();63 return /next/i.test(text) || link.find('.fa-chevron-right').length > 0;64 });65
66 return {67 currentPage: match ? Number(match[1]) : 1,68 totalPages: match ? Number(match[2]) : 1,69 hasNextPage,70 };71 }72
73 parseSearchProfiles(html) {74 const $ = cheerio.load(html);75 const profiles = [];76
77 $('a.viewprofile').each((_, element) => {78 const href = $(element).attr('href');79 const profileUrl = this.buildProfileUrl(href);80 if (!profileUrl) return;81
82 profiles.push({ profileUrl });83 });84
85 return {86 profiles,87 pagination: this.parsePagination($),88 };89 }90
91 parseProfile(html, searchKeyword, profileUrl) {92 const $ = cheerio.load(html);93 const heading = this.normalizeText($('#nameSection h1').text());94 const [name, university] = heading95 ? heading.split(',').map((part) => part.trim())96 : [null, null];97
98 const generalDetails = $('#generalDetails');99 const position = this.normalizeText(100 generalDetails.children('.spacify').first().text(),101 );102
103 const contactInfo = $('#contactInfo');104 const location = this.normalizeText(105 contactInfo106 .find('.fa-location-arrow')107 .closest('.spacify')108 .text()109 .replace(/\s+/g, ' ')110 .trim(),111 );112 const email = this.normalizeText(113 contactInfo.find('a[href^="mailto:"]').first().text(),114 );115 const phone = this.normalizeText(116 contactInfo117 .find('.fa-phone')118 .closest('.spacify')119 .text()120 .replace(/\s+/g, ' ')121 .trim(),122 );123
124 const expertIn = [];125 $('.singleCard')126 .filter((_, card) => {127 return $(card).find('.sectionHeader').text().trim() === 'Expert In';128 })129 .find('.cardPadder a')130 .each((_, link) => {131 const text = this.normalizeText($(link).text());132 if (text) expertIn.push(text);133 });134
135 const links = [];136 $('.singleCard')137 .filter((_, card) => {138 return $(card).find('.sectionHeader').text().trim() === 'Links';139 })140 .find('.cardPadder a')141 .each((_, link) => {142 const title = this.normalizeText($(link).text());143 const url = $(link).attr('href') || null;144 if (title && url) {145 links.push({ title, url });146 }147 });148
149 const bio =150 this.normalizeText($('#maxbio .cardPadder > span').first().text()) ||151 this.normalizeText($('#minbio .cardPadder > span').first().text());152
153 return {154 searchKeyword,155 profileUrl,156 name: name || heading || null,157 university: university || null,158 position,159 location,160 email,161 phone,162 expertIn,163 bio,164 links,165 };166 }167
168 async fetchHtml(url, axiosConfig) {169 const response = await axios.get(url, {170 ...axiosConfig,171 headers: DEFAULT_HEADERS,172 timeout: 60000,173 });174 return response.data;175 }176
177 async run(input) {178 const { keyword, maxItems = 100, proxyConfiguration } = input;179 const searchKeyword = String(keyword || '').trim();180 const itemLimit = Number(maxItems);181
182 if (!searchKeyword) {183 throw new Error('Input must include a keyword');184 }185
186 if (!Number.isFinite(itemLimit) || itemLimit < 1) {187 throw new Error('maxItems must be a positive number');188 }189
190 const proxyConfig = proxyConfiguration191 ? await Actor.createProxyConfiguration(proxyConfiguration)192 : undefined;193
194 const proxyUrl = proxyConfig ? await proxyConfig.newUrl() : undefined;195 const axiosConfig = this.getAxiosConfig(proxyUrl);196
197 await this.scrapeKeyword(searchKeyword, itemLimit, axiosConfig);198 }199
200 async scrapeKeyword(keyword, maxItems, axiosConfig) {201 console.log(`Searching experts for "${keyword}"...`);202
203 let pageNumber = 1;204 let totalSaved = 0;205 const seen = new Set();206
207 while (totalSaved < maxItems) {208 const searchUrl = this.buildSearchUrl(keyword, pageNumber);209 let html;210
211 try {212 html = await this.fetchHtml(searchUrl, axiosConfig);213 } catch (error) {214 const message = error.response?.status215 ? `Request failed with status ${error.response.status}`216 : 'Request failed';217 throw new Error(message);218 }219
220 const { profiles, pagination } = this.parseSearchProfiles(html);221 if (profiles.length === 0) {222 break;223 }224
225 let savedThisPage = 0;226
227 for (const profile of profiles) {228 if (totalSaved >= maxItems) break;229 if (seen.has(profile.profileUrl)) continue;230 seen.add(profile.profileUrl);231
232 try {233 const profileHtml = await this.fetchHtml(profile.profileUrl, axiosConfig);234 const result = this.parseProfile(profileHtml, keyword, profile.profileUrl);235
236 await Actor.pushData([237 {238 ...result,239 scrapedAt: new Date().toISOString(),240 },241 ]);242 totalSaved++;243 savedThisPage++;244 console.log(`Saved ${totalSaved} experts for "${keyword}"`);245 } catch {246 continue;247 }248
249 await this.randomDelay(400, 800);250 }251
252 if (totalSaved >= maxItems) {253 break;254 }255
256 if (savedThisPage === 0) {257 break;258 }259
260 if (!pagination.hasNextPage || pagination.currentPage >= pagination.totalPages) {261 break;262 }263
264 pageNumber++;265 await this.randomDelay(800, 1500);266 }267
268 if (totalSaved === 0) {269 console.log(`No experts found for "${keyword}"`);270 await Actor.pushData([271 {272 searchKeyword: keyword,273 error: 'No experts found',274 scrapedAt: new Date().toISOString(),275 },276 ]);277 }278 }279
280 async randomDelay(min = 500, max = 1500) {281 const delay = Math.floor(Math.random() * (max - min + 1) + min);282 await new Promise((resolve) => setTimeout(resolve, delay));283 }284}285
286await Actor.init();287
288Actor.main(async () => {289 const input = await Actor.getInput();290
291 // const input = {292 // keyword: 'AI',293 // maxItems: 100294 // }295
296 const scraper = new ExpertFinderExpertsScraper();297 await scraper.run(input);298});# configurations.idea
# crawlee and apify storage foldersapify_storagecrawlee_storagestorage
# installed filesnode_modules
# git folder.gitroot = true
[*]indent_style = spaceindent_size = 4charset = utf-8trim_trailing_whitespace = trueinsert_final_newline = trueend_of_line = lf{ "extends": "@apify", "root": true}# This file tells Git which files shouldn't be added to source control
.DS_Store.ideanode_modulesstorage{ "name": "js-scrape-single-page", "version": "0.0.1", "type": "module", "description": "This is an example of an Apify actor.", "engines": { "node": ">=18.0.0" }, "dependencies": { "apify": "^3.4.4", "axios": "^1.7.7", "cheerio": "^1.0.0", "crypto-js": "^4.1.1", "puppeteer-real-browser": "^1.4.4" }, "scripts": { "start": "node ./src/main.js", "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1" }, "author": "It's not you it's me", "license": "ISC"}