Product Hunt Scraper (with Email Enrichment)
Pricing
from $2.00 / 1,000 results
Extract product names, upvotes, makers, taglines, and website URLs from ProductHunt MCP-ready. $2.00/1K.
Pricing
from $2.00 / 1,000 results
Rating
0.0
(0)
Developer
Khadin Akbar
Maintained by CommunityActor stats
0
Bookmarked
34
Total users
15
Monthly active users
5 days ago
Last modified
Categories
Share
FROM apify/actor-node-playwright-chrome:20
# Detect and save the image's pre-installed playwright version BEFORE any COPY.# The image ships playwright + matching Chromium. We must install the exact same# playwright version so browser executable paths always match.RUN node -e "try{process.stdout.write(require('./node_modules/playwright/package.json').version)}catch(e){process.stdout.write('unknown')}" > /tmp/IMAGE_PW_VERSION && \ echo "Image playwright: $(cat /tmp/IMAGE_PW_VERSION)"
# Copy ALL source (including package.json, package-lock.json, src/, .actor/)COPY . .
# Patch package.json + regenerate lock file to use the image's playwright versionRUN IMAGE_PW=$(cat /tmp/IMAGE_PW_VERSION) && \ npm pkg set dependencies.playwright="$IMAGE_PW" && \ npm install --package-lock-only --ignore-scripts 2>/dev/null && \ echo "Playwright pinned to $IMAGE_PW (matches image browsers)"
# Install ALL deps (devDependencies included so tsc is available)RUN NODE_ENV=development npm install --audit=false
# Compile TypeScriptRUN npm run build
# Remove devDependencies from the final imageRUN npm prune --omit=dev
CMD ["node", "dist/main.js"]{ "name": "producthunt-scraper-pro", "version": "1.0.0", "type": "module", "description": "Scrape ProductHunt listings — products, makers, upvotes, topics, and website URLs", "main": "dist/main.js", "scripts": { "start": "node dist/main.js", "build": "tsc", "dev": "tsc && node dist/main.js" }, "dependencies": { "apify": "^3.0.0", "crawlee": "^3.0.0", "playwright": "1.44.0", "cheerio": "^1.0.0" }, "devDependencies": { "typescript": "^5.3.0", "@types/node": "^20.0.0" }}{ "compilerOptions": { "target": "ES2022", "module": "NodeNext", "moduleResolution": "NodeNext", "outDir": "dist", "rootDir": "src", "strict": true, "esModuleInterop": true, "skipLibCheck": true, "resolveJsonModule": true }, "include": ["src/**/*"]}{ "title": "ProductHunt Scraper Pro Input", "type": "object", "schemaVersion": 1, "properties": { "mode": { "title": "Scraping Mode", "type": "string", "description": "How to find products. Use 'leaderboard' for top-ranked products by time period. Use 'search' when the user provides a keyword. Use 'topic' for a ProductHunt topic slug. Use 'urls' for specific ProductHunt URLs.", "enum": ["leaderboard", "search", "topic", "urls"], "default": "leaderboard", "editor": "select" }, "leaderboardPeriod": { "title": "Leaderboard Period", "type": "string", "description": "Only used when mode='leaderboard'. Choose 'daily', 'weekly', 'monthly', or 'yearly'.", "enum": ["daily", "weekly", "monthly", "yearly"], "default": "daily", "editor": "select" }, "searchQuery": { "title": "Search Query", "type": "string", "description": "Used when mode='search'. Enter a keyword or niche to search ProductHunt (e.g. 'AI writing tools', 'developer tools').", "editor": "textfield" }, "topic": { "title": "Topic Slug", "type": "string", "description": "Used when mode='topic'. The ProductHunt topic URL slug (e.g. 'artificial-intelligence', 'developer-tools', 'productivity').", "editor": "textfield" }, "startUrls": { "title": "Start URLs", "type": "array", "description": "Used when mode='urls'. List of specific ProductHunt URLs to scrape.", "editor": "requestListSources", "prefill": [{"url": "https://www.producthunt.com/"}] }, "maxResults": { "title": "Max Results", "type": "integer", "description": "Maximum number of products to return. Default is 100.", "default": 100, "minimum": 1, "maximum": 20000, "editor": "number" }, "maxConcurrency": { "title": "Max Concurrency", "type": "integer", "description": "Number of pages to scrape simultaneously. Default is 5.", "default": 5, "minimum": 1, "maximum": 10, "editor": "number" }, "enrichEmails": { "title": "Find Email Addresses", "type": "boolean", "description": "When enabled, discovers emails for each product's website using a 3-layer waterfall: Hunter.io API → Snov.io API → page scraping fallback. Default is true.", "default": true, "editor": "checkbox" }, "startDate": { "title": "Start Date (YYYY-MM-DD)", "type": "string", "description": "For leaderboard daily mode: query a specific date. Leave blank for today. Use with endDate to scrape a date range in one run (e.g. startDate=2026-03-01, endDate=2026-03-07 → full week).", "editor": "textfield" }, "endDate": { "title": "End Date (YYYY-MM-DD)", "type": "string", "description": "Optional end date for multi-day scraping in one run. Only used in daily leaderboard mode. Example: startDate=2026-03-01, endDate=2026-03-07 returns all products launched across that 7-day window.", "editor": "textfield" }, "lookbackDays": { "title": "Lookback Days", "type": "integer", "description": "Alternative to startDate: scrape the last N days from today. E.g. lookbackDays=7 pulls the last week. Ignored when startDate is provided.", "default": 1, "minimum": 1, "maximum": 365, "editor": "number" }, "outputMode": { "title": "Output Mode", "type": "string", "description": "Controls what gets returned. 'full' = all fields (default). 'lean' = minimal fields optimized for LLM/AI agent context windows (name, tagline, website, upvotes, rank, emails, topics). 'leads' = full fields but only products where at least 1 email was found — perfect for outreach lists.", "enum": ["full", "lean", "leads"], "default": "full", "editor": "select" }, "hunterApiKey": { "title": "Hunter.io API Key (recommended)", "type": "string", "description": "Hunter.io is the industry-standard email enrichment API with ~65-80% hit rate on SaaS/startup domains. Get a free key at https://hunter.io (25 domain searches/month free). Without this key, the actor falls back to basic page scraping which has much lower success on modern JS-rendered sites.", "editor": "textfield", "isSecret": true }, "snovClientId": { "title": "Snov.io Client ID (optional fallback)", "type": "string", "description": "Snov.io client ID for email enrichment — used as fallback if Hunter.io finds nothing. Get credentials at https://snov.io → Settings → API (50 searches/month free).", "editor": "textfield" }, "snovClientSecret": { "title": "Snov.io Client Secret (optional fallback)", "type": "string", "description": "Snov.io client secret. Required alongside snovClientId.", "editor": "textfield", "isSecret": true } }}{ "actorSpecification": 1, "name": "producthunt-scraper-pro", "title": "ProductHunt Scraper Pro — Daily/Weekly/Monthly + Email Finder", "description": "Scrape ProductHunt leaderboards (daily, weekly, monthly, yearly) and extract product names, upvotes, makers, taglines, topics, website URLs, and founder emails. Supports date ranges (startDate + endDate), outputMode (full/lean/leads), keyword search, topic filtering, and multi-day scraping in one run. Uses the official ProductHunt API for fast, block-free data. Email enrichment via Hunter.io API or Playwright browser crawl.", "version": "1.1", "buildTag": "latest", "dockerfile": "./Dockerfile", "output": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }, "pricingInfo": { "pricingModel": "PAY_PER_EVENT", "pricingPerEvent": { "actorChargeEvents": [ { "eventName": "product-scraped", "eventDescription": "One ProductHunt product record scraped and pushed to the dataset.", "eventPriceUsd": 0.001 } ] } }, "input": { "title": "ProductHunt Scraper Pro Input", "type": "object", "schemaVersion": 1, "description": "Configure how ProductHunt is scraped.", "properties": { "mode": { "title": "Scraping Mode", "type": "string", "description": "How to find products. Use 'leaderboard' for top-ranked products by time period. Use 'search' for a keyword. Use 'topic' for a topic slug. Use 'urls' for specific ProductHunt URLs.", "enum": [ "leaderboard", "search", "topic", "urls" ], "default": "leaderboard", "editor": "select" }, "leaderboardPeriod": { "title": "Leaderboard Period", "type": "string", "description": "Only used when mode='leaderboard'. daily/weekly/monthly/yearly.", "enum": [ "daily", "weekly", "monthly", "yearly" ], "default": "daily", "editor": "select" }, "searchQuery": { "title": "Search Query", "type": "string", "description": "Used when mode='search'. Keyword or niche to search (e.g. 'AI writing tools').", "editor": "textfield" }, "topic": { "title": "Topic Slug", "type": "string", "description": "Used when mode='topic'. ProductHunt topic slug (e.g. 'artificial-intelligence', 'developer-tools').", "editor": "textfield" }, "startUrls": { "title": "Start URLs", "type": "array", "description": "Used when mode='urls'. List of specific ProductHunt URLs to scrape.", "editor": "requestListSources", "prefill": [ { "url": "https://www.producthunt.com/" } ] }, "maxResults": { "title": "Max Results", "type": "integer", "description": "Maximum number of products to return. Default is 100.", "default": 100, "prefill": 10, "minimum": 1, "maximum": 20000, "editor": "number" }, "allProducts": { "title": "Include All Products (not just featured)", "type": "boolean", "description": "Used when mode='leaderboard'. When false (default), scrapes only the featured/highlighted products for that period — the curated PH front page view. When true, appends /all to the URL and scrapes every product launched in that period, including non-featured ones. Use true when the user wants the full launch archive, all submissions, or non-featured products.", "default": false, "editor": "checkbox" }, "maxConcurrency": { "title": "Max Concurrency", "type": "integer", "description": "Number of pages to scrape simultaneously. Default is 5.", "default": 5, "minimum": 1, "maximum": 10, "editor": "number" }, "enrichEmails": { "title": "Find Email Addresses", "type": "boolean", "description": "When enabled, discovers emails for each product's website using a 3-layer waterfall: Hunter.io API → Snov.io API → page scraping fallback. Default is true.", "default": true, "prefill": false, "editor": "checkbox" }, "hunterApiKey": { "title": "Hunter.io API Key (recommended)", "type": "string", "description": "Hunter.io is the industry-standard email enrichment API with ~65-80% hit rate on SaaS/startup domains. Get a free key at https://hunter.io (25 domain searches/month free). Without this key the actor falls back to basic page scraping which has much lower success on modern JS-rendered sites.", "editor": "textfield", "isSecret": true }, "snovClientId": { "title": "Snov.io Client ID (optional fallback)", "type": "string", "description": "Snov.io client ID for email enrichment — used if Hunter.io finds nothing. Get credentials at https://snov.io → Settings → API (50 searches/month free).", "editor": "textfield" }, "snovClientSecret": { "title": "Snov.io Client Secret (optional fallback)", "type": "string", "description": "Snov.io client secret. Required alongside snovClientId.", "editor": "textfield", "isSecret": true }, "phApiToken": { "title": "ProductHunt API Token (optional — works out of the box)", "type": "string", "description": "Optional. A default token is built in so the actor works immediately with no setup. To use your own free token: go to https://www.producthunt.com/v2/oauth/applications, create an app, and paste the Developer Token here.", "editor": "textfield", "isSecret": true }, "startDate": { "title": "Start Date (YYYY-MM-DD)", "type": "string", "description": "For daily leaderboard: specific date to scrape. Leave blank for today. Use with endDate for multi-day ranges.", "editor": "textfield" }, "endDate": { "title": "End Date (YYYY-MM-DD)", "type": "string", "description": "Optional end date for multi-day scraping in one run (daily mode only). Example: startDate=2026-03-01 + endDate=2026-03-07 pulls an entire week.", "editor": "textfield" }, "lookbackDays": { "title": "Lookback Days", "type": "integer", "description": "Scrape the last N days from today. Alternative to startDate. Ignored when startDate is set.", "default": 1, "minimum": 1, "maximum": 365, "editor": "number" }, "outputMode": { "title": "Output Mode", "type": "string", "description": "Controls output fields. 'full' = all fields (default). 'lean' = minimal fields for LLM/AI agents (name, tagline, website, upvotes, rank, emails, topics). 'leads' = full fields but ONLY products with emails found.", "enum": [ "full", "lean", "leads" ], "default": "full", "editor": "select" } } }, "changelog": "\n\nv1.1 (2026-05-22) — Soft-fail user-input validation: convert Actor.fail on missing/invalid input to Actor.exit with WARNING status. Prevents user-input errors from counting as actor failures and dragging down 30-day success rate."}{ "actorSpecification": 1, "fields": { "type": "object", "properties": { "product_name": { "type": "string", "title": "Product Name", "description": "The official product name as listed on ProductHunt.", "example": "Notion AI" }, "tagline": { "type": ["string", "null"], "title": "Tagline", "description": "Short one-line pitch shown beneath the product name on ProductHunt.", "example": "AI-powered writing for everyone" }, "description": { "type": ["string", "null"], "title": "Description", "description": "Full product description written by the makers on ProductHunt.", "example": "Notion AI helps you write faster, think bigger, and augment your knowledge base." }, "upvote_count": { "type": ["integer", "null"], "title": "Upvotes", "description": "Total upvotes received. Higher = more popular on launch day.", "example": 1284 }, "comment_count": { "type": ["integer", "null"], "title": "Comments", "description": "Total number of comments on the ProductHunt listing.", "example": 87 }, "reviews_count": { "type": ["integer", "null"], "title": "Review Count", "description": "Number of user reviews the product has received on ProductHunt.", "example": 42 }, "reviews_rating": { "type": ["number", "null"], "title": "Avg Rating", "description": "Average user review rating on ProductHunt (typically 1–5 scale).", "example": 4.7 }, "daily_rank": { "type": ["integer", "null"], "title": "Daily Rank", "description": "#1 means most upvoted product of the day. Populated for daily leaderboard scrapes.", "example": 1 }, "weekly_rank": { "type": ["integer", "null"], "title": "Weekly Rank", "description": "Product's position in the weekly leaderboard.", "example": 3 }, "monthly_rank": { "type": ["integer", "null"], "title": "Monthly Rank", "description": "Product's position in the monthly leaderboard.", "example": 12 }, "yearly_rank": { "type": ["integer", "null"], "title": "Yearly Rank", "description": "Product's position in the annual leaderboard.", "example": 47 }, "launch_date": { "type": ["string", "null"], "title": "Launch Date", "description": "Date the product was featured on ProductHunt (YYYY-MM-DD).", "example": "2026-03-28" }, "product_hunt_url": { "type": ["string", "null"], "title": "ProductHunt URL", "description": "Full URL to the product's page on ProductHunt.", "example": "https://www.producthunt.com/posts/notion-ai" }, "website_url": { "type": ["string", "null"], "title": "Website", "description": "Official product website URL (PH redirect resolved to final destination).", "example": "https://notion.so" }, "emails": { "type": "array", "title": "Emails Found", "description": "Email addresses discovered on the product website via multi-layer extraction: mailto links → JSON-LD structured data → body text scan. Empty when enrichEmails is off or no emails were found.", "example": ["hello@notion.so", "support@notion.so"] }, "email_source": { "type": "string", "title": "Email Source", "description": "Where the emails came from. 'hunter_io' = verified by Hunter.io API (highest confidence, 65-80% hit rate on SaaS domains). 'page_scrape' = found by Playwright crawling the product website. 'none' = no emails found or enrichment was skipped.", "example": "hunter_io" }, "topics": { "type": "array", "title": "Topics", "description": "ProductHunt category tags (e.g. Artificial Intelligence, Developer Tools, Productivity).", "example": ["Productivity", "Artificial Intelligence", "Writing"] }, "thumbnail_url": { "type": ["string", "null"], "title": "Thumbnail", "description": "URL of the product logo/thumbnail hosted on ProductHunt's CDN.", "example": "https://ph-files.imgix.net/abc123.png?auto=format&fit=crop&h=256&w=256" }, "makers": { "type": "array", "title": "Makers", "description": "People who built the product. Each entry includes maker_name, maker_id (PH username), maker_ph_id (internal PH ID), twitter_url, and maker_url (PH profile).", "example": [{"maker_name": "Ivan Zhao", "maker_id": "ivanhzhao", "maker_ph_id": "1234567", "twitter_url": "https://twitter.com/ivanhzhao", "maker_url": "https://www.producthunt.com/@ivanhzhao"}] }, "featured": { "type": "boolean", "title": "Featured", "description": "True if the product was editorially featured on ProductHunt's homepage.", "example": true }, "scraped_at": { "type": "string", "title": "Scraped At", "description": "ISO 8601 timestamp of when this record was extracted.", "example": "2026-03-28T14:30:00.000Z" } } }, "views": { "overview": { "title": "🚀 Products Overview", "transformation": { "fields": [ "daily_rank", "product_name", "tagline", "upvote_count", "comment_count", "reviews_rating", "launch_date", "topics", "website_url", "product_hunt_url", "featured" ] }, "display": { "component": "table", "properties": { "daily_rank": { "label": "Rank" }, "product_name": { "label": "Product" }, "tagline": { "label": "Tagline" }, "upvote_count": { "label": "Upvotes" }, "comment_count": { "label": "Comments" }, "reviews_rating": { "label": "Rating" }, "launch_date": { "label": "Launch Date" }, "topics": { "label": "Topics" }, "website_url": { "label": "Website" }, "product_hunt_url": { "label": "PH Link" }, "featured": { "label": "Featured?" } } } }, "leads": { "title": "✉️ Email Leads", "transformation": { "fields": [ "product_name", "tagline", "emails", "email_source", "website_url", "topics", "upvote_count", "launch_date", "product_hunt_url" ] }, "display": { "component": "table", "properties": { "product_name": { "label": "Product" }, "tagline": { "label": "Tagline" }, "emails": { "label": "Emails Found" }, "email_source": { "label": "Source" }, "website_url": { "label": "Website" }, "topics": { "label": "Topics" }, "upvote_count": { "label": "Upvotes" }, "launch_date": { "label": "Launch Date" }, "product_hunt_url": { "label": "PH Link" } } } }, "rankings": { "title": "🏆 Rankings", "transformation": { "fields": [ "daily_rank", "weekly_rank", "monthly_rank", "yearly_rank", "product_name", "upvote_count", "comment_count", "reviews_count", "reviews_rating", "launch_date", "featured" ] }, "display": { "component": "table", "properties": { "daily_rank": { "label": "Daily #" }, "weekly_rank": { "label": "Weekly #" }, "monthly_rank": { "label": "Monthly #" }, "yearly_rank": { "label": "Yearly #" }, "product_name": { "label": "Product" }, "upvote_count": { "label": "Upvotes" }, "comment_count": { "label": "Comments" }, "reviews_count": { "label": "Reviews" }, "reviews_rating":{ "label": "Avg Rating" }, "launch_date": { "label": "Launch Date" }, "featured": { "label": "Featured?" } } } }, "makers": { "title": "👥 Makers", "transformation": { "fields": [ "product_name", "makers", "upvote_count", "launch_date", "website_url", "product_hunt_url" ] }, "display": { "component": "table", "properties": { "product_name": { "label": "Product" }, "makers": { "label": "Makers" }, "upvote_count": { "label": "Upvotes" }, "launch_date": { "label": "Launch Date" }, "website_url": { "label": "Website" }, "product_hunt_url": { "label": "PH Link" } } } } }}{ "actorOutputSchemaVersion": 1, "title": "ProductHunt Scraper Pro Output", "description": "Scraped ProductHunt products including name, upvotes, tagline, website URL, emails, makers, topics, and rank data. Results are stored in the default dataset.", "properties": { "results": { "type": "string", "title": "Scraped Products", "description": "All scraped ProductHunt products. Each item contains product_name, tagline, upvote_count, rank, website_url, emails, makers, topics, and more.", "template": "{{links.apiDefaultDatasetUrl}}/items" } }}1import { Actor } from 'apify';2import { log } from 'crawlee';3import { chromium } from 'playwright';4// ─── INIT ────────────────────────────────────────────────────────────────────5await Actor.init();6// ─── INPUT PARSING ───────────────────────────────────────────────────────────7const rawInput = (await Actor.getInput() ?? {});8const mode = (rawInput.mode ?? rawInput.scrapeMode ?? 'leaderboard');9const leaderboardPeriod = (rawInput.leaderboardPeriod ?? rawInput.period ?? 'daily');10const startDate = (rawInput.startDate ?? rawInput.date ?? '');11const endDate = (rawInput.endDate ?? '');12const lookbackDays = Number(rawInput.lookbackDays ?? 1);13const outputMode = (rawInput.outputMode ?? 'full');14const searchQuery = (rawInput.searchQuery ?? rawInput.query ?? rawInput.q);15const topic = (rawInput.topic ?? rawInput.topicSlug);16const startUrls = (rawInput.startUrls ?? []);17const maxResults = Number(rawInput.maxResults ?? rawInput.maxItems ?? rawInput.limit ?? 1000);18const allProducts = Boolean(rawInput.allProducts ?? rawInput.includeAll ?? false);19const enrichEmails = Boolean(rawInput.enrichEmails ?? rawInput.findEmails ?? true);20const hunterApiKey = (rawInput.hunterApiKey ?? rawInput.hunterKey ?? process.env.HUNTER_API_KEY ?? '');21const maxConcurrency = Math.min(Number(rawInput.maxConcurrency ?? 3), 5);22const DEFAULT_PH_TOKEN = 'siHY_lSwsPoDFCx71pKL0Gj_jOvDIYTcaLKfgvVC8Y4';23const phApiToken = (rawInput.phApiToken ?? rawInput.apiToken ?? process.env.PH_API_TOKEN ?? DEFAULT_PH_TOKEN);24// ─── INPUT VALIDATION ────────────────────────────────────────────────────────25if (!rawInput.phApiToken && !rawInput.apiToken && !process.env.PH_API_TOKEN) {26 log.info('Using built-in PH API token. Supply your own via phApiToken input or PH_API_TOKEN env var.');27}28if (mode === 'search' && !searchQuery)29 await Actor.fail('searchQuery required for search mode.');30if (mode === 'topic' && !topic)31 await Actor.fail('topic required for topic mode.');32if (mode === 'urls' && startUrls.length === 0)33 await Actor.fail('startUrls required for urls mode.');34log.info('ProductHunt Scraper starting', {35 mode, leaderboardPeriod, startDate, endDate: endDate || '(same as startDate)',36 lookbackDays, maxResults, allProducts, enrichEmails, outputMode,37 emailMethod: hunterApiKey ? 'Hunter.io + Playwright fallback' : 'Playwright browser scraping',38});39// ─── DATE HELPERS ─────────────────────────────────────────────────────────────40function isoDate(d) { return d.toISOString().split('T')[0]; }41function getWeekBounds(d) {42 const s = new Date(d);43 s.setDate(d.getDate() - d.getDay());44 const e = new Date(s);45 e.setDate(s.getDate() + 6);46 return { start: isoDate(s), end: isoDate(e) };47}48function getMonthBounds(d) {49 const y = d.getFullYear(), m = d.getMonth();50 return { start: isoDate(new Date(y, m, 1)), end: isoDate(new Date(y, m + 1, 0)) };51}52function getYearBounds(d) {53 return { start: `${d.getFullYear()}-01-01`, end: `${d.getFullYear()}-12-31` };54}55function buildDateRange() {56 if (mode !== 'leaderboard')57 return null;58 if (lookbackDays > 1) {59 const end = new Date();60 const begin = new Date();61 begin.setDate(end.getDate() - lookbackDays);62 return { postedAfter: `${isoDate(begin)}T00:00:00+00:00`, postedBefore: `${isoDate(end)}T23:59:59+00:00` };63 }64 const base = startDate ? new Date(startDate + 'T12:00:00Z') : new Date();65 switch (leaderboardPeriod) {66 case 'daily': {67 const from = startDate || isoDate(base);68 // endDate allows multi-day ranges in one run (e.g. startDate=2026-03-25 endDate=2026-03-30)69 const to = endDate || from;70 return { postedAfter: `${from}T00:00:00+00:00`, postedBefore: `${to}T23:59:59+00:00` };71 }72 case 'weekly': {73 const { start, end } = getWeekBounds(base);74 return { postedAfter: `${start}T00:00:00+00:00`, postedBefore: `${end}T23:59:59+00:00` };75 }76 case 'monthly': {77 const { start, end } = getMonthBounds(base);78 return { postedAfter: `${start}T00:00:00+00:00`, postedBefore: `${end}T23:59:59+00:00` };79 }80 case 'yearly': {81 const { start, end } = getYearBounds(base);82 return { postedAfter: `${start}T00:00:00+00:00`, postedBefore: `${end}T23:59:59+00:00` };83 }84 default: return null;85 }86}87// ─── PH GRAPHQL API ───────────────────────────────────────────────────────────88const PH_API = 'https://api.producthunt.com/v2/api/graphql';89async function phQuery(query, variables = {}) {90 const res = await fetch(PH_API, {91 method: 'POST',92 headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': `Bearer ${phApiToken}` },93 body: JSON.stringify({ query, variables }),94 });95 if (!res.ok) {96 const b = await res.text().catch(() => '');97 throw new Error(`PH API ${res.status}: ${b.slice(0, 200)}`);98 }99 const json = await res.json();100 if (json.errors?.length)101 throw new Error(`GraphQL: ${JSON.stringify(json.errors[0])}`);102 return json.data;103}104const POST_FIELDS = `105 id name tagline description votesCount commentsCount url website slug106 featuredAt createdAt dailyRank weeklyRank monthlyRank yearlyRank107 reviewsCount reviewsRating108 thumbnail { url }109 topics(first: 10) { edges { node { name } } }110 makers { id name username twitterUsername url }111`;112const POSTS_QUERY = `query GetPosts($first:Int $after:String $order:PostsOrder $featured:Boolean $postedAfter:DateTime $postedBefore:DateTime $topic:String) {113 posts(first:$first after:$after order:$order featured:$featured postedAfter:$postedAfter postedBefore:$postedBefore topic:$topic) {114 edges { node { ${POST_FIELDS} } }115 pageInfo { endCursor hasNextPage }116 }117}`;118const SEARCH_QUERY = `query SearchPosts($query:String! $first:Int $after:String) {119 posts(first:$first after:$after order:VOTES search:$query) {120 edges { node { ${POST_FIELDS} } }121 pageInfo { endCursor hasNextPage }122 }123}`;124// ─── REDIRECT RESOLVER ───────────────────────────────────────────────────────125// Fallback resolver (used only when no browser is running, i.e. enrichEmails=false).126// Strips UTM params before fetching — PH /r/XXXX redirects can fail with UTM garbage.127async function resolveToRealUrl(rawUrl) {128 if (!rawUrl)129 return rawUrl;130 try {131 const cleanUrl = rawUrl.split('?')[0]; // strip ?utm_campaign=... etc.132 let res = await fetch(cleanUrl, {133 method: 'HEAD', redirect: 'follow',134 headers: { 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1)' },135 signal: AbortSignal.timeout(8000),136 });137 // HEAD sometimes doesn't follow PH redirects — fall back to GET138 if (res.url.includes('producthunt.com')) {139 res = await fetch(cleanUrl, {140 method: 'GET', redirect: 'follow',141 headers: { 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1)' },142 signal: AbortSignal.timeout(10000),143 });144 }145 const resolved = res.url.split('?')[0].replace(/\/$/, '');146 return (resolved && !resolved.includes('producthunt.com')) ? resolved : rawUrl;147 }148 catch {149 return rawUrl;150 }151}152// ─── EMAIL ENRICHMENT ────────────────────────────────────────────────────────153const MAX_PAGES_PER_DOMAIN = 8;154const SKIP_DOMAINS = new Set([155 'apps.apple.com', 'play.google.com', 'github.com', 'youtube.com',156 'twitter.com', 'x.com', 'linkedin.com', 'facebook.com', 'instagram.com',157 'reddit.com', 'discord.com', 'discord.gg', 'slack.com',158 'medium.com', 'substack.com', 'producthunt.com', 'amazon.com',159 'chrome.google.com', 'marketplace.visualstudio.com',160 'notion.so', 'figma.com', 'trello.com', 'airtable.com',161]);162const UA_POOL = [163 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',164 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',165 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',166 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.15',167];168// Seed paths — always queued first, ordered by email-finding probability169const SEED_PATHS = [170 '/contact',171 '/contact-us',172 '/about',173 '/about-us',174 '/team',175 '/support',176 '/help',177 '/privacy',178 '/privacy-policy',179 '/',180];181// Priority score for a URL — higher = visit sooner when discovered via links182function urlPriority(url) {183 const u = url.toLowerCase();184 if (u.includes('contact'))185 return 100;186 if (u.includes('about'))187 return 90;188 if (u.includes('team'))189 return 80;190 if (u.includes('support'))191 return 75;192 if (u.includes('help'))193 return 70;194 if (u.includes('company'))195 return 65;196 if (u.includes('privacy'))197 return 60;198 if (u.includes('legal'))199 return 55;200 if (u.includes('imprint'))201 return 50;202 if (u.includes('faq'))203 return 45;204 if (u.includes('humans.txt'))205 return 40;206 if (/^\/$|\/index/.test(u))207 return 5; // homepage — lowest208 return 20;209}210// Runs inside the live browser DOM — returns extracted emails211function browserExtractEmails(baseDomain) {212 const junkRe = /example\.|test@|placeholder|noreply|no-reply|\.(png|jpg|gif|svg|pdf|js|css|woff)$|@sentry\.|@logrocket\.|@segment\.|your@|email@email|\d{10,}@/i;213 const emailRe = /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g;214 const found = new Set();215 // 1. mailto: links — highest precision, JS may inject these dynamically216 document.querySelectorAll('a[href^="mailto:"]').forEach((el) => {217 const a = el;218 const e = a.href.replace('mailto:', '').split('?')[0].trim().toLowerCase();219 if (e && e.includes('@') && !junkRe.test(e))220 found.add(e);221 });222 // 2. JSON-LD structured data (schema.org Organization / Person .email)223 document.querySelectorAll('script[type="application/ld+json"]').forEach((el) => {224 try {225 const data = JSON.parse(el.textContent ?? '{}');226 const items = Array.isArray(data) ? data : [data];227 for (const item of items) {228 if (item?.email && typeof item.email === 'string') {229 const e = item.email.trim().toLowerCase();230 if (!junkRe.test(e))231 found.add(e);232 }233 for (const cp of (item?.contactPoint ?? [])) {234 if (cp?.email && typeof cp.email === 'string') {235 const e = cp.email.trim().toLowerCase();236 if (!junkRe.test(e))237 found.add(e);238 }239 }240 }241 }242 catch { /* malformed JSON-LD */ }243 });244 // 3. Full rendered text + raw HTML regex scan245 const combined = (document.body?.innerText ?? '') + ' ' + document.documentElement.innerHTML;246 const all = combined.match(emailRe) ?? [];247 // Domain-matched first (highest precision)248 for (const m of all) {249 const e = m.toLowerCase();250 if (junkRe.test(e))251 continue;252 const d = e.split('@')[1];253 if (d && (d === baseDomain || d.endsWith('.' + baseDomain) || baseDomain.endsWith('.' + d))) {254 found.add(e);255 }256 }257 // Fallback: any valid email if nothing domain-matched258 if (found.size === 0) {259 for (const m of all) {260 const e = m.toLowerCase();261 if (!junkRe.test(e))262 found.add(e);263 }264 }265 return [...found];266}267// Runs inside the browser — returns internal links from the current page268function browserExtractInternalLinks(origin) {269 const links = new Set();270 document.querySelectorAll('a[href]').forEach((el) => {271 const a = el;272 try {273 const href = a.href;274 if (!href.startsWith(origin))275 return; // external link276 const url = new URL(href);277 if (url.hash)278 return; // skip anchors279 // Strip query string and trailing slash for dedup280 const clean = url.origin + url.pathname.replace(/\/$/, '');281 if (clean !== origin)282 links.add(clean);283 }284 catch { /* ignore malformed hrefs */ }285 });286 return [...links];287}288// ── Optional fast-path: Hunter.io (only if API key is configured) ─────────────289// Returns emails with their confidence scores from Hunter.io290async function tryHunterIo(domain) {291 if (!hunterApiKey)292 return { emails: [], confidence: [] };293 try {294 const res = await fetch(`https://api.hunter.io/v2/domain-search?domain=${encodeURIComponent(domain)}&limit=10&api_key=${encodeURIComponent(hunterApiKey)}`, { signal: AbortSignal.timeout(10000) });295 if (!res.ok)296 return { emails: [], confidence: [] };297 const data = await res.json();298 const hits = (data?.data?.emails ?? [])299 .filter((e) => (e.confidence ?? 0) >= 50)300 .sort((a, b) => (b.confidence ?? 0) - (a.confidence ?? 0));301 return {302 emails: hits.map((e) => String(e.value).toLowerCase().trim()).filter(Boolean),303 confidence: hits.map((e) => e.confidence ?? 0),304 };305 }306 catch {307 return { emails: [], confidence: [] };308 }309}310// ── Master email finder — resolves PH redirect URLs + smart crawl ─────────────311// Returns emails found, the real resolved URL, and the source of the emails.312// phProductUrl: the /products/SLUG page on PH — used to establish session cookies313// and extract the real website URL from __NEXT_DATA__ as a fallback.314async function findEmails(rawWebsiteUrl, browser, phProductUrl) {315 // Validate the input URL is parseable before launching a context316 try {317 new URL(rawWebsiteUrl);318 }319 catch {320 return { emails: [], resolvedUrl: rawWebsiteUrl, emailSource: 'none' };321 }322 // Fresh isolated context per product (clean cookies, no cross-contamination)323 const context = await browser.newContext({324 userAgent: UA_POOL[Math.floor(Math.random() * UA_POOL.length)],325 locale: 'en-US',326 viewport: { width: 1280, height: 800 },327 extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' },328 serviceWorkers: 'block',329 });330 // Hide automation detection — prevents PH bot-protection from firing331 await context.addInitScript(() => {332 Object.defineProperty(navigator, 'webdriver', { get: () => undefined });333 Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });334 window.chrome = { runtime: {} };335 });336 // Block heavy assets — only DOM text needed337 await context.route('**/*', (route) => {338 const rt = route.request().resourceType();339 if (['image', 'media', 'font', 'stylesheet'].includes(rt))340 return route.abort();341 return route.continue();342 });343 const page = await context.newPage();344 let emails = [];345 let resolvedUrl = rawWebsiteUrl;346 let origin = '';347 let baseDomain = '';348 let pagesVisited = 0;349 try {350 // ── Step 1: Resolve the real website URL ─────────────────────────────────351 // PH /r/XXXX redirect URLs require PH session cookies to follow.352 // Strategy:353 // (a) Fast-path: try HEAD fetch (works if network allows)354 // (b) Visit the PH product page to seed PH session cookies, then extract355 // the real URL from __NEXT_DATA__ OR follow the redirect with cookies356 let realPageUrl = rawWebsiteUrl;357 const cleanRedirectUrl = rawWebsiteUrl.split('?')[0];358 // (a) Fast-path HTTP fetch — works without browser359 try {360 const headRes = await fetch(cleanRedirectUrl, {361 method: 'HEAD', redirect: 'follow',362 headers: {363 'User-Agent': UA_POOL[0],364 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',365 'Accept-Language': 'en-US,en;q=0.9',366 },367 signal: AbortSignal.timeout(8000),368 });369 const fetchResolved = headRes.url.split('?')[0].replace(/\/$/, '');370 if (fetchResolved && !fetchResolved.includes('producthunt.com')) {371 realPageUrl = fetchResolved;372 log.info(` 🔗 ${cleanRedirectUrl} → ${realPageUrl} (fetch)`);373 }374 }375 catch { /* fall through to browser */ }376 // (b) Browser-based resolution — visit PH product page first for cookies377 if ((realPageUrl === rawWebsiteUrl || realPageUrl.includes('producthunt.com')) && phProductUrl) {378 try {379 // Visit PH product page — this sets session cookies AND loads __NEXT_DATA__380 await page.goto(phProductUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });381 await page.waitForTimeout(600);382 // Try to extract the real website URL from PH's __NEXT_DATA__ JSON blob383 const nextDataUrl = await page.evaluate(() => {384 const nd = document.getElementById('__NEXT_DATA__');385 if (!nd)386 return null;387 try {388 const d = JSON.parse(nd.textContent ?? '');389 const product = d?.props?.pageProps?.post ??390 d?.props?.pageProps?.product ??391 d?.props?.pageProps?.productData ??392 d?.props?.pageProps?.data?.post;393 // Try every URL-like field394 const candidates = [395 product?.website,396 product?.websiteUrl,397 product?.externalUrl,398 product?.originalUrl,399 ];400 for (const c of candidates) {401 if (c && typeof c === 'string' && c.startsWith('http'))402 return c;403 }404 }405 catch { }406 return null;407 });408 if (nextDataUrl && !nextDataUrl.includes('producthunt.com')) {409 realPageUrl = nextDataUrl.split('?')[0].replace(/\/$/, '');410 log.info(` 📦 ${cleanRedirectUrl} → ${realPageUrl} (__NEXT_DATA__)`);411 }412 else {413 // We now have PH session cookies — try the redirect URL414 const redirectTarget = (nextDataUrl ?? cleanRedirectUrl).split('?')[0];415 let capturedLoc = null;416 page.on('response', (r) => {417 if (r.status() >= 300 && r.status() < 400) {418 const loc = r.headers()['location'];419 if (loc && !loc.includes('producthunt.com') && !capturedLoc) {420 try {421 const abs = loc.startsWith('http') ? loc : new URL(loc, redirectTarget).href;422 capturedLoc = abs.split('?')[0].replace(/\/$/, '');423 }424 catch { }425 }426 }427 });428 await page.goto(redirectTarget, { waitUntil: 'networkidle', timeout: 20000 });429 const navUrl = page.url().split('?')[0].replace(/\/$/, '');430 const resolved = capturedLoc || (navUrl && !navUrl.includes('producthunt.com') ? navUrl : null);431 if (resolved) {432 realPageUrl = resolved;433 log.info(` 🍪 ${cleanRedirectUrl} → ${realPageUrl} (cookie+redirect)`);434 }435 else {436 log.info(` ⚠️ Could not resolve ${cleanRedirectUrl} — staying on PH`);437 }438 }439 }440 catch (e) {441 log.debug(` URL resolution via PH page failed: ${e.message?.slice(0, 80)}`);442 }443 }444 resolvedUrl = realPageUrl;445 // If we couldn't resolve past PH, skip email enrichment446 if (!realPageUrl || realPageUrl.includes('producthunt.com')) {447 return { emails: [], resolvedUrl, emailSource: 'none' };448 }449 let parsedReal;450 try {451 parsedReal = new URL(realPageUrl);452 }453 catch {454 return { emails: [], resolvedUrl, emailSource: 'none' };455 }456 origin = parsedReal.origin;457 baseDomain = parsedReal.hostname.replace(/^www\./, '');458 // Skip known junk domains (app stores, social media, etc.) — now against REAL domain459 const rootDomain = baseDomain.split('.').slice(-2).join('.');460 if (SKIP_DOMAINS.has(rootDomain) || SKIP_DOMAINS.has(baseDomain)) {461 log.debug(` [skip-domain] ${baseDomain} is in skip list — no email search`);462 return { emails: [], resolvedUrl, emailSource: 'none' };463 }464 // ── Step 2: Hunter.io fast-path (if API key configured) ──────────────────465 const { emails: hunterEmails } = await tryHunterIo(baseDomain);466 if (hunterEmails.length > 0) {467 log.info(` 🎯 Hunter.io → ${hunterEmails.length} email(s) for ${baseDomain}`);468 return { emails: hunterEmails, resolvedUrl, emailSource: 'hunter_io' };469 }470 // ── Navigate to the real website for email scraping ───────────────────────471 // After URL resolution, page may be at PH or unnavigated — go to real site.472 const currentPageUrl = page.url().split('?')[0].replace(/\/$/, '');473 const alreadyAtRealSite = currentPageUrl.startsWith(origin);474 if (!alreadyAtRealSite) {475 try {476 await page.goto(realPageUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });477 await page.waitForTimeout(1500);478 }479 catch (e) {480 const tag = e.message?.includes('Timeout') ? '[timeout]' : '[network]';481 log.debug(` ${tag} Could not load ${realPageUrl}: ${e.message?.slice(0, 80)}`);482 return { emails: [], resolvedUrl, emailSource: 'none' };483 }484 }485 pagesVisited = 1; // landing page loaded486 // ── Step 3: Extract emails from already-loaded landing page ───────────────487 const landingEmails = await page.evaluate(browserExtractEmails, baseDomain);488 if (landingEmails.length > 0)489 return { emails: landingEmails, resolvedUrl, emailSource: 'page_scrape' };490 const visited = new Set([realPageUrl, rawWebsiteUrl]);491 const queue = [];492 const landingLinks = await page.evaluate(browserExtractInternalLinks, origin);493 for (const link of landingLinks) {494 if (!visited.has(link)) {495 const path = link.replace(origin, '') || '/';496 queue.push({ url: link, priority: urlPriority(path) });497 }498 }499 // Also inject seed paths that weren't discovered naturally500 for (const path of SEED_PATHS) {501 const url = `${origin}${path}`;502 if (!visited.has(url) && !queue.some(q => q.url === url)) {503 queue.push({ url, priority: urlPriority(path) });504 }505 }506 // Sort descending — highest priority visited first507 queue.sort((a, b) => b.priority - a.priority);508 // ── Step 5: Crawl up to MAX_PAGES_PER_DOMAIN pages ───────────────────────509 while (queue.length > 0 && pagesVisited < MAX_PAGES_PER_DOMAIN) {510 const entry = queue.shift();511 if (visited.has(entry.url))512 continue;513 visited.add(entry.url);514 pagesVisited++;515 try {516 const response = await page.goto(entry.url, {517 waitUntil: 'domcontentloaded',518 timeout: 14000,519 });520 const ct = response?.headers()['content-type'] ?? '';521 if (!ct.includes('text/') && !ct.includes('html'))522 continue;523 // Let React / Vue / Svelte hydrate524 await page.waitForTimeout(1500);525 const found = await page.evaluate(browserExtractEmails, baseDomain);526 if (found.length > 0) {527 emails = found;528 break; // stop crawling — email found, mark source below529 }530 // Expand queue with newly discovered internal links531 if (pagesVisited < MAX_PAGES_PER_DOMAIN) {532 const links = await page.evaluate(browserExtractInternalLinks, origin);533 for (const link of links) {534 if (!visited.has(link) && !queue.some(q => q.url === link)) {535 const path = link.replace(origin, '') || '/';536 queue.push({ url: link, priority: urlPriority(path) });537 }538 }539 queue.sort((a, b) => b.priority - a.priority);540 }541 }542 catch {543 continue;544 }545 }546 }547 catch (err) {548 const tag = err.message?.includes('Timeout') ? '[timeout]'549 : err.message?.includes('net::ERR') ? '[network]'550 : '[error]';551 log.debug(` ${tag} Navigation failed for ${rawWebsiteUrl}: ${err.message?.slice(0, 100)}`);552 }553 finally {554 await context.close();555 }556 const emailSource = emails.length > 0 ? 'page_scrape' : 'none';557 log.debug(` crawled ${pagesVisited} page(s) for ${baseDomain || rawWebsiteUrl} → ${emailSource}`);558 return { emails, resolvedUrl, emailSource };559}560// ─── TYPE HELPERS ─────────────────────────────────────────────────────────────561function toInt(v) {562 if (v === null || v === undefined || v === '')563 return null;564 const n = parseInt(String(v), 10);565 return isNaN(n) ? null : n;566}567function toFloat(v) {568 if (v === null || v === undefined || v === '')569 return null;570 const n = parseFloat(String(v));571 return isNaN(n) ? null : n;572}573// ─── NODE → RECORD ────────────────────────────────────────────────────────────574function nodeToRecord(node, rank) {575 const makers = (node.makers ?? []).map((m) => ({576 maker_name: m.name ?? null,577 maker_id: m.username ?? null,578 maker_ph_id: m.id ?? null,579 twitter_url: m.twitterUsername ? `https://twitter.com/${m.twitterUsername}` : null,580 maker_url: m.url ?? null,581 }));582 const topics = (node.topics?.edges ?? []).map((e) => e?.node?.name).filter(Boolean);583 return {584 product_name: node.name ?? null,585 tagline: node.tagline ?? null,586 description: node.description ?? null,587 upvote_count: toInt(node.votesCount),588 comment_count: toInt(node.commentsCount),589 reviews_count: toInt(node.reviewsCount),590 reviews_rating: toFloat(node.reviewsRating),591 daily_rank: toInt(node.dailyRank) ?? rank,592 weekly_rank: toInt(node.weeklyRank),593 monthly_rank: toInt(node.monthlyRank),594 yearly_rank: toInt(node.yearlyRank),595 launch_date: node.featuredAt ? new Date(node.featuredAt).toISOString().split('T')[0]596 : node.createdAt ? new Date(node.createdAt).toISOString().split('T')[0] : null,597 product_hunt_url: node.url ?? (node.slug ? `https://www.producthunt.com/posts/${node.slug}` : null),598 website_url: node.website ?? null, // resolved to real domain in Phase 2599 topics,600 thumbnail_url: node.thumbnail?.url ?? null,601 makers,602 featured: !!node.featuredAt,603 emails: [],604 email_source: 'none', // updated in processNode after enrichment605 scraped_at: new Date().toISOString(),606 };607}608// ─── OUTPUT MODE FILTER ───────────────────────────────────────────────────────609// lean — minimal fields for fast browsing / LLM context windows610// leads — full fields, but only products where emails were found611// full — everything (default)612const LEAN_FIELDS = new Set([613 'product_name', 'tagline', 'website_url', 'upvote_count', 'daily_rank',614 'emails', 'email_source', 'topics', 'launch_date', 'product_hunt_url',615 'featured', 'scraped_at',616]);617function applyOutputMode(record, mode) {618 if (mode === 'leads' && record.emails.length === 0)619 return null;620 if (mode === 'lean') {621 const out = {};622 for (const k of LEAN_FIELDS)623 out[k] = record[k] ?? null;624 return out;625 }626 return record;627}628// ─── QUERY VAR BUILDER ────────────────────────────────────────────────────────629const dateRange = buildDateRange();630function buildVars(after, remaining) {631 const pageSize = Math.min(20, remaining);632 if (mode === 'search')633 return { query: searchQuery, first: pageSize, after };634 const vars = { first: pageSize, after, order: 'VOTES' };635 if (!allProducts)636 vars.featured = true;637 if (dateRange) {638 vars.postedAfter = dateRange.postedAfter;639 vars.postedBefore = dateRange.postedBefore;640 }641 if (mode === 'topic')642 vars.topic = topic;643 return vars;644}645// ─── PHASE 1: FETCH FROM PH API ──────────────────────────────────────────────646log.info('Phase 1: Fetching products from ProductHunt API...', {647 dateRange: dateRange ?? 'no date filter', allProducts, maxResults,648});649const collectedNodes = [];650const seenIds = new Set();651let cursor = null;652let pageNum = 1;653while (collectedNodes.length < maxResults) {654 log.info(`Page ${pageNum} (collected ${collectedNodes.length}/${maxResults})...`);655 let data;656 try {657 data = await phQuery(mode === 'search' ? SEARCH_QUERY : POSTS_QUERY, buildVars(cursor, maxResults - collectedNodes.length));658 }659 catch (err) {660 log.error(`PH API error on page ${pageNum}: ${err.message}`);661 break;662 }663 const edges = data?.posts?.edges ?? [];664 const pageInfo = data?.posts?.pageInfo ?? {};665 if (!edges.length) {666 log.info('No more results from PH API.');667 break;668 }669 let added = 0;670 for (const { node } of edges) {671 if (collectedNodes.length >= maxResults)672 break;673 if (!node?.id || seenIds.has(node.id))674 continue;675 seenIds.add(node.id);676 collectedNodes.push(node);677 added++;678 }679 log.info(`Page ${pageNum}: +${added} products (total: ${collectedNodes.length})`);680 if (!pageInfo.hasNextPage || !pageInfo.endCursor) {681 log.info('No more pages.');682 break;683 }684 cursor = pageInfo.endCursor;685 pageNum++;686}687log.info(`Phase 1 complete: ${collectedNodes.length} products`);688// ─── PHASE 2: RESOLVE REAL DOMAINS + EMAIL ENRICHMENT ────────────────────────689log.info(`Phase 2: Resolving real domains${enrichEmails ? ' + Playwright email scraping' : ''}...`);690let browser = null;691if (enrichEmails) {692 browser = await chromium.launch({693 headless: true,694 args: [695 '--no-sandbox',696 '--disable-setuid-sandbox',697 '--disable-dev-shm-usage',698 '--disable-accelerated-2d-canvas',699 '--disable-gpu',700 '--no-zygote',701 '--disable-extensions',702 '--disable-background-networking',703 '--disable-default-apps',704 '--disable-blink-features=AutomationControlled', // hide navigator.webdriver705 ],706 });707 log.info('Playwright browser launched.');708}709async function processNode(node, rank) {710 const record = nodeToRecord(node, rank);711 const rawUrl = typeof record.website_url === 'string' ? record.website_url : '';712 // PH product page URL — used to get session cookies + extract URL from __NEXT_DATA__713 const phProductUrl = typeof record.product_hunt_url === 'string'714 ? record.product_hunt_url.split('?')[0]715 : undefined;716 if (enrichEmails && browser && rawUrl.startsWith('http')) {717 // findEmails handles URL resolution (via Playwright navigation) AND email scraping.718 // It returns the real domain URL after following PH redirect wrappers.719 try {720 const { emails, resolvedUrl, emailSource } = await findEmails(rawUrl, browser, phProductUrl);721 record.website_url = resolvedUrl;722 record.emails = emails;723 record.email_source = emailSource;724 if (emails.length > 0) {725 log.info(` ✉ ${record.product_name}: ${emails.length} email(s) [${emailSource}] → ${emails.join(', ')}`);726 }727 }728 catch (err) {729 const msg = err.message ?? '';730 const tag = msg.includes('Timeout') || msg.includes('timeout') ? '[timeout]'731 : msg.includes('net::ERR') || msg.includes('ECONNREFUSED') ? '[network]'732 : msg.includes('Navigation') ? '[navigation]'733 : '[error]';734 log.warning(` Email enrichment failed ${tag} for ${record.product_name}: ${msg.slice(0, 100)}`);735 record.website_url = await resolveToRealUrl(rawUrl);736 }737 }738 else if (rawUrl.startsWith('http')) {739 // No browser (enrichEmails=false) — use HTTP redirect resolver740 record.website_url = await resolveToRealUrl(rawUrl);741 }742 // Apply outputMode filter: lean strips to key fields, leads drops products with no emails743 const outputRecord = applyOutputMode(record, outputMode);744 if (outputRecord !== null)745 await Actor.pushData(outputRecord);746}747// Batched concurrency — cap at 3 concurrent pages when browser is active748const batchSize = enrichEmails ? Math.min(maxConcurrency, 3) : maxConcurrency;749for (let i = 0; i < collectedNodes.length; i += batchSize) {750 const batch = collectedNodes.slice(i, i + batchSize);751 await Promise.all(batch.map((node, j) => processNode(node, i + j + 1)));752 const done = Math.min(i + batchSize, collectedNodes.length);753 if (done % 30 === 0 || done === collectedNodes.length) {754 log.info(`✓ Processed ${done}/${collectedNodes.length}`);755 }756}757// ─── CLEANUP ──────────────────────────────────────────────────────────────────758if (browser) {759 await browser.close();760 log.info('Playwright browser closed.');761}762log.info(`Done! ${collectedNodes.length} products scraped and pushed.`);763await Actor.exit();1import { Actor } from 'apify';2import { log } from 'crawlee';3import { chromium, type Browser } from 'playwright';4
5// ─── INIT ────────────────────────────────────────────────────────────────────6await Actor.init();7
8// ─── INPUT PARSING ───────────────────────────────────────────────────────────9const rawInput = (await Actor.getInput() ?? {}) as Record<string, unknown>;10
11const mode = (rawInput.mode ?? rawInput.scrapeMode ?? 'leaderboard') as12 'leaderboard' | 'search' | 'topic' | 'urls';13const leaderboardPeriod = (rawInput.leaderboardPeriod ?? rawInput.period ?? 'daily') as14 'daily' | 'weekly' | 'monthly' | 'yearly';15const startDate = (rawInput.startDate ?? rawInput.date ?? '') as string;16const endDate = (rawInput.endDate ?? '') as string;17const lookbackDays = Number(rawInput.lookbackDays ?? 1);18const outputMode = (rawInput.outputMode ?? 'full') as 'full' | 'lean' | 'leads';19const searchQuery = (rawInput.searchQuery ?? rawInput.query ?? rawInput.q) as string | undefined;20const topic = (rawInput.topic ?? rawInput.topicSlug) as string | undefined;21const startUrls = (rawInput.startUrls ?? []) as Array<{ url: string } | string>;22const maxResults = Number(rawInput.maxResults ?? rawInput.maxItems ?? rawInput.limit ?? 1000);23const allProducts = Boolean(rawInput.allProducts ?? rawInput.includeAll ?? false);24const enrichEmails = Boolean(rawInput.enrichEmails ?? rawInput.findEmails ?? true);25const hunterApiKey = (rawInput.hunterApiKey ?? rawInput.hunterKey ?? process.env.HUNTER_API_KEY ?? '') as string;26const maxConcurrency = Math.min(Number(rawInput.maxConcurrency ?? 3), 5);27const DEFAULT_PH_TOKEN = 'siHY_lSwsPoDFCx71pKL0Gj_jOvDIYTcaLKfgvVC8Y4';28const phApiToken = (rawInput.phApiToken ?? rawInput.apiToken ?? process.env.PH_API_TOKEN ?? DEFAULT_PH_TOKEN) as string;29
30// ─── INPUT VALIDATION ────────────────────────────────────────────────────────31if (!rawInput.phApiToken && !rawInput.apiToken && !process.env.PH_API_TOKEN) {32 log.info('Using built-in PH API token. Supply your own via phApiToken input or PH_API_TOKEN env var.');33}34async function softExit(msg: string): Promise<never> {35 await Actor.setStatusMessage(msg, { isStatusMessageTerminal: true, level: 'WARNING' });36 await Actor.exit();37 process.exit(0);38}39if (mode === 'search' && !searchQuery) await softExit('searchQuery required for search mode. Provide a search keyword and re-run.');40if (mode === 'topic' && !topic) await softExit('topic required for topic mode. Provide a topic slug (e.g. "ai") and re-run.');41if (mode === 'urls' && startUrls.length === 0) await softExit('startUrls required for urls mode. Provide one or more ProductHunt URLs and re-run.');42
43log.info('ProductHunt Scraper starting', {44 mode, leaderboardPeriod, startDate, endDate: endDate || '(same as startDate)',45 lookbackDays, maxResults, allProducts, enrichEmails, outputMode,46 emailMethod: hunterApiKey ? 'Hunter.io + Playwright fallback' : 'Playwright browser scraping',47});48
49// ─── DATE HELPERS ─────────────────────────────────────────────────────────────50function isoDate(d: Date) { return d.toISOString().split('T')[0]; }51
52function getWeekBounds(d: Date) {53 const s = new Date(d); s.setDate(d.getDate() - d.getDay());54 const e = new Date(s); e.setDate(s.getDate() + 6);55 return { start: isoDate(s), end: isoDate(e) };56}57function getMonthBounds(d: Date) {58 const y = d.getFullYear(), m = d.getMonth();59 return { start: isoDate(new Date(y,m,1)), end: isoDate(new Date(y,m+1,0)) };60}61function getYearBounds(d: Date) {62 return { start: `${d.getFullYear()}-01-01`, end: `${d.getFullYear()}-12-31` };63}64
65function buildDateRange(): { postedAfter: string; postedBefore: string } | null {66 if (mode !== 'leaderboard') return null;67 if (lookbackDays > 1) {68 const end = new Date(); const begin = new Date();69 begin.setDate(end.getDate() - lookbackDays);70 return { postedAfter: `${isoDate(begin)}T00:00:00+00:00`, postedBefore: `${isoDate(end)}T23:59:59+00:00` };71 }72 const base = startDate ? new Date(startDate + 'T12:00:00Z') : new Date();73 switch (leaderboardPeriod) {74 case 'daily': {75 const from = startDate || isoDate(base);76 // endDate allows multi-day ranges in one run (e.g. startDate=2026-03-25 endDate=2026-03-30)77 const to = endDate || from;78 return { postedAfter: `${from}T00:00:00+00:00`, postedBefore: `${to}T23:59:59+00:00` };79 }80 case 'weekly': {81 const { start, end } = getWeekBounds(base);82 return { postedAfter: `${start}T00:00:00+00:00`, postedBefore: `${end}T23:59:59+00:00` };83 }84 case 'monthly': {85 const { start, end } = getMonthBounds(base);86 return { postedAfter: `${start}T00:00:00+00:00`, postedBefore: `${end}T23:59:59+00:00` };87 }88 case 'yearly': {89 const { start, end } = getYearBounds(base);90 return { postedAfter: `${start}T00:00:00+00:00`, postedBefore: `${end}T23:59:59+00:00` };91 }92 default: return null;93 }94}95
96// ─── PH GRAPHQL API ───────────────────────────────────────────────────────────97const PH_API = 'https://api.producthunt.com/v2/api/graphql';98async function phQuery(query: string, variables: Record<string, unknown> = {}, attempt = 0): Promise<any> {99 const res = await fetch(PH_API, {100 method: 'POST',101 headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', 'Authorization': `Bearer ${phApiToken}` },102 body: JSON.stringify({ query, variables }),103 });104 // 429 rate limit — parse reset_in from the error body and wait, then retry105 if (res.status === 429) {106 const b = await res.text().catch(() => '{}');107 let waitSecs = 30 * (attempt + 1); // fallback backoff: 30s, 60s, 90s108 try {109 const errData = JSON.parse(b);110 const resetIn = errData?.errors?.[0]?.details?.reset_in;111 if (resetIn && typeof resetIn === 'number') waitSecs = Math.min(resetIn + 5, 120);112 } catch { /* use fallback */ }113 if (attempt >= 4) throw new Error(`PH API 429 after ${attempt + 1} retries: ${b.slice(0, 200)}`);114 log.warning(`PH API rate limited — waiting ${waitSecs}s before retry (attempt ${attempt + 1}/5)...`);115 await new Promise(r => setTimeout(r, waitSecs * 1000));116 return phQuery(query, variables, attempt + 1);117 }118 if (!res.ok) { const b = await res.text().catch(() => ''); throw new Error(`PH API ${res.status}: ${b.slice(0,200)}`); }119 const json = await res.json() as { data?: any; errors?: any[] };120 if (json.errors?.length) throw new Error(`GraphQL: ${JSON.stringify(json.errors[0])}`);121 return json.data;122}123
124const POST_FIELDS = `125 id name tagline description votesCount commentsCount url website slug126 featuredAt createdAt dailyRank weeklyRank monthlyRank yearlyRank127 reviewsCount reviewsRating128 thumbnail { url }129 topics(first: 10) { edges { node { name } } }130 makers { id name username twitterUsername url }131`;132
133const POSTS_QUERY = `query GetPosts($first:Int $after:String $order:PostsOrder $featured:Boolean $postedAfter:DateTime $postedBefore:DateTime $topic:String) {134 posts(first:$first after:$after order:$order featured:$featured postedAfter:$postedAfter postedBefore:$postedBefore topic:$topic) {135 edges { node { ${POST_FIELDS} } }136 pageInfo { endCursor hasNextPage }137 }138}`;139const SEARCH_QUERY = `query SearchPosts($query:String! $first:Int $after:String) {140 posts(first:$first after:$after order:VOTES search:$query) {141 edges { node { ${POST_FIELDS} } }142 pageInfo { endCursor hasNextPage }143 }144}`;145
146// ─── REDIRECT RESOLVER ───────────────────────────────────────────────────────147// Fallback resolver (used only when no browser is running, i.e. enrichEmails=false).148// Strips UTM params before fetching — PH /r/XXXX redirects can fail with UTM garbage.149async function resolveToRealUrl(rawUrl: string): Promise<string> {150 if (!rawUrl) return rawUrl;151 try {152 const cleanUrl = rawUrl.split('?')[0]; // strip ?utm_campaign=... etc.153 let res = await fetch(cleanUrl, {154 method: 'HEAD', redirect: 'follow',155 headers: { 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1)' },156 signal: AbortSignal.timeout(8000),157 });158 // HEAD sometimes doesn't follow PH redirects — fall back to GET159 if (res.url.includes('producthunt.com')) {160 res = await fetch(cleanUrl, {161 method: 'GET', redirect: 'follow',162 headers: { 'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1)' },163 signal: AbortSignal.timeout(10000),164 });165 }166 const resolved = res.url.split('?')[0].replace(/\/$/, '');167 return (resolved && !resolved.includes('producthunt.com')) ? resolved : rawUrl;168 } catch { return rawUrl; }169}170
171// ─── EMAIL ENRICHMENT ────────────────────────────────────────────────────────172
173const MAX_PAGES_PER_DOMAIN = 8;174
175const SKIP_DOMAINS = new Set([176 'apps.apple.com', 'play.google.com', 'github.com', 'youtube.com',177 'twitter.com', 'x.com', 'linkedin.com', 'facebook.com', 'instagram.com',178 'reddit.com', 'discord.com', 'discord.gg', 'slack.com',179 'medium.com', 'substack.com', 'producthunt.com', 'amazon.com',180 'chrome.google.com', 'marketplace.visualstudio.com',181 'notion.so', 'figma.com', 'trello.com', 'airtable.com',182]);183
184const UA_POOL = [185 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',186 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36',187 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36',188 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_3_1) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.15',189];190
191// Seed paths — always queued first, ordered by email-finding probability192const SEED_PATHS = [193 '/contact',194 '/contact-us',195 '/about',196 '/about-us',197 '/team',198 '/support',199 '/help',200 '/privacy',201 '/privacy-policy',202 '/',203];204
205// Priority score for a URL — higher = visit sooner when discovered via links206function urlPriority(url: string): number {207 const u = url.toLowerCase();208 if (u.includes('contact')) return 100;209 if (u.includes('about')) return 90;210 if (u.includes('team')) return 80;211 if (u.includes('support')) return 75;212 if (u.includes('help')) return 70;213 if (u.includes('company')) return 65;214 if (u.includes('privacy')) return 60;215 if (u.includes('legal')) return 55;216 if (u.includes('imprint')) return 50;217 if (u.includes('faq')) return 45;218 if (u.includes('humans.txt')) return 40;219 if (/^\/$|\/index/.test(u)) return 5; // homepage — lowest220 return 20;221}222
223// Runs inside the live browser DOM — returns extracted emails224function browserExtractEmails(baseDomain: string): string[] {225 const junkRe = /example\.|test@|placeholder|noreply|no-reply|\.(png|jpg|gif|svg|pdf|js|css|woff)$|@sentry\.|@logrocket\.|@segment\.|your@|email@email|\d{10,}@/i;226 const emailRe = /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g;227 const found = new Set<string>();228
229 // 1. mailto: links — highest precision, JS may inject these dynamically230 document.querySelectorAll('a[href^="mailto:"]').forEach((el) => {231 const a = el as HTMLAnchorElement;232 const e = a.href.replace('mailto:', '').split('?')[0].trim().toLowerCase();233 if (e && e.includes('@') && !junkRe.test(e)) found.add(e);234 });235
236 // 2. JSON-LD structured data (schema.org Organization / Person .email)237 document.querySelectorAll('script[type="application/ld+json"]').forEach((el) => {238 try {239 const data = JSON.parse((el as HTMLElement).textContent ?? '{}');240 const items: any[] = Array.isArray(data) ? data : [data];241 for (const item of items) {242 if (item?.email && typeof item.email === 'string') {243 const e = item.email.trim().toLowerCase();244 if (!junkRe.test(e)) found.add(e);245 }246 for (const cp of (item?.contactPoint ?? [])) {247 if (cp?.email && typeof cp.email === 'string') {248 const e = cp.email.trim().toLowerCase();249 if (!junkRe.test(e)) found.add(e);250 }251 }252 }253 } catch { /* malformed JSON-LD */ }254 });255
256 // 3. Full rendered text + raw HTML regex scan257 const combined = (document.body?.innerText ?? '') + ' ' + document.documentElement.innerHTML;258 const all = combined.match(emailRe) ?? [];259
260 // Domain-matched first (highest precision)261 for (const m of all) {262 const e = m.toLowerCase();263 if (junkRe.test(e)) continue;264 const d = e.split('@')[1];265 if (d && (d === baseDomain || d.endsWith('.' + baseDomain) || baseDomain.endsWith('.' + d))) {266 found.add(e);267 }268 }269 // Fallback: any valid email if nothing domain-matched270 if (found.size === 0) {271 for (const m of all) {272 const e = m.toLowerCase();273 if (!junkRe.test(e)) found.add(e);274 }275 }276
277 return [...found];278}279
280// Runs inside the browser — returns internal links from the current page281function browserExtractInternalLinks(origin: string): string[] {282 const links = new Set<string>();283 document.querySelectorAll('a[href]').forEach((el) => {284 const a = el as HTMLAnchorElement;285 try {286 const href = a.href;287 if (!href.startsWith(origin)) return; // external link288 const url = new URL(href);289 if (url.hash) return; // skip anchors290 // Strip query string and trailing slash for dedup291 const clean = url.origin + url.pathname.replace(/\/$/, '');292 if (clean !== origin) links.add(clean);293 } catch { /* ignore malformed hrefs */ }294 });295 return [...links];296}297
298// ── Optional fast-path: Hunter.io (only if API key is configured) ─────────────299// Returns emails with their confidence scores from Hunter.io300async function tryHunterIo(domain: string): Promise<{ emails: string[]; confidence: number[] }> {301 if (!hunterApiKey) return { emails: [], confidence: [] };302 try {303 const res = await fetch(304 `https://api.hunter.io/v2/domain-search?domain=${encodeURIComponent(domain)}&limit=10&api_key=${encodeURIComponent(hunterApiKey)}`,305 { signal: AbortSignal.timeout(10000) },306 );307 if (!res.ok) return { emails: [], confidence: [] };308 const data = await res.json() as any;309 const hits = (data?.data?.emails ?? [])310 .filter((e: any) => (e.confidence ?? 0) >= 50)311 .sort((a: any, b: any) => (b.confidence ?? 0) - (a.confidence ?? 0));312 return {313 emails: hits.map((e: any) => String(e.value).toLowerCase().trim()).filter(Boolean),314 confidence: hits.map((e: any) => e.confidence ?? 0),315 };316 } catch { return { emails: [], confidence: [] }; }317}318
319// Email source tags — tells callers (and users) where emails came from320// "hunter_io" = Hunter.io API returned them (highest confidence, verified)321// "page_scrape" = Found via Playwright: mailto links, JSON-LD, or text scan322// "none" = No emails found after all attempts323type EmailSource = 'hunter_io' | 'page_scrape' | 'none';324
325// ── Master email finder — resolves PH redirect URLs + smart crawl ─────────────326// Returns emails found, the real resolved URL, and the source of the emails.327// phProductUrl: the /products/SLUG page on PH — used to establish session cookies328// and extract the real website URL from __NEXT_DATA__ as a fallback.329async function findEmails(330 rawWebsiteUrl: string,331 browser: Browser,332 phProductUrl?: string,333): Promise<{ emails: string[]; resolvedUrl: string; emailSource: EmailSource }> {334 // Validate the input URL is parseable before launching a context335 try { new URL(rawWebsiteUrl); } catch { return { emails: [], resolvedUrl: rawWebsiteUrl, emailSource: 'none' }; }336
337 // Fresh isolated context per product (clean cookies, no cross-contamination)338 const context = await browser.newContext({339 userAgent: UA_POOL[Math.floor(Math.random() * UA_POOL.length)],340 locale: 'en-US',341 viewport: { width: 1280, height: 800 },342 extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' },343 serviceWorkers: 'block',344 });345
346 // Hide automation detection — prevents PH bot-protection from firing347 await context.addInitScript(() => {348 Object.defineProperty(navigator, 'webdriver', { get: () => undefined });349 Object.defineProperty(navigator, 'plugins', { get: () => [1, 2, 3] });350 (window as any).chrome = { runtime: {} };351 });352
353 // Block heavy assets — only DOM text needed354 await context.route('**/*', (route) => {355 const rt = route.request().resourceType();356 if (['image', 'media', 'font', 'stylesheet'].includes(rt)) return route.abort();357 return route.continue();358 });359
360 const page = await context.newPage();361 let emails: string[] = [];362 let resolvedUrl = rawWebsiteUrl;363 let origin = '';364 let baseDomain = '';365 let pagesVisited = 0;366
367 try {368 // ── Step 1: Resolve the real website URL ─────────────────────────────────369 // PH /r/XXXX redirect URLs require PH session cookies to follow.370 // Strategy:371 // (a) Fast-path: try HEAD fetch (works if network allows)372 // (b) Visit the PH product page to seed PH session cookies, then extract373 // the real URL from __NEXT_DATA__ OR follow the redirect with cookies374 let realPageUrl = rawWebsiteUrl;375 const cleanRedirectUrl = rawWebsiteUrl.split('?')[0];376
377 // (a) Fast-path HTTP fetch — works without browser378 try {379 const headRes = await fetch(cleanRedirectUrl, {380 method: 'HEAD', redirect: 'follow',381 headers: {382 'User-Agent': UA_POOL[0],383 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',384 'Accept-Language': 'en-US,en;q=0.9',385 },386 signal: AbortSignal.timeout(8000),387 });388 const fetchResolved = headRes.url.split('?')[0].replace(/\/$/, '');389 if (fetchResolved && !fetchResolved.includes('producthunt.com')) {390 realPageUrl = fetchResolved;391 log.info(` 🔗 ${cleanRedirectUrl} → ${realPageUrl} (fetch)`);392 }393 } catch { /* fall through to browser */ }394
395 // (b) Browser-based resolution — visit PH product page first for cookies396 if ((realPageUrl === rawWebsiteUrl || realPageUrl.includes('producthunt.com')) && phProductUrl) {397 try {398 // Visit PH product page — this sets session cookies AND loads __NEXT_DATA__399 await page.goto(phProductUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });400 await page.waitForTimeout(600);401
402 // Try to extract the real website URL from PH's __NEXT_DATA__ JSON blob403 const nextDataUrl = await page.evaluate(() => {404 const nd = document.getElementById('__NEXT_DATA__');405 if (!nd) return null;406 try {407 const d = JSON.parse(nd.textContent ?? '');408 const product =409 d?.props?.pageProps?.post ??410 d?.props?.pageProps?.product ??411 d?.props?.pageProps?.productData ??412 d?.props?.pageProps?.data?.post;413 // Try every URL-like field414 const candidates = [415 product?.website,416 product?.websiteUrl,417 product?.externalUrl,418 product?.originalUrl,419 ];420 for (const c of candidates) {421 if (c && typeof c === 'string' && c.startsWith('http')) return c;422 }423 } catch {}424 return null;425 }) as string | null;426
427 if (nextDataUrl && !nextDataUrl.includes('producthunt.com')) {428 realPageUrl = nextDataUrl.split('?')[0].replace(/\/$/, '');429 log.info(` 📦 ${cleanRedirectUrl} → ${realPageUrl} (__NEXT_DATA__)`);430 } else {431 // We now have PH session cookies — try the redirect URL432 const redirectTarget = (nextDataUrl ?? cleanRedirectUrl).split('?')[0];433 let capturedLoc: string | null = null;434 page.on('response', (r) => {435 if (r.status() >= 300 && r.status() < 400) {436 const loc = r.headers()['location'];437 if (loc && !loc.includes('producthunt.com') && !capturedLoc) {438 try {439 const abs = loc.startsWith('http') ? loc : new URL(loc, redirectTarget).href;440 capturedLoc = abs.split('?')[0].replace(/\/$/, '');441 } catch {}442 }443 }444 });445 await page.goto(redirectTarget, { waitUntil: 'networkidle', timeout: 20000 });446 const navUrl = page.url().split('?')[0].replace(/\/$/, '');447 const resolved = capturedLoc || (navUrl && !navUrl.includes('producthunt.com') ? navUrl : null);448 if (resolved) {449 realPageUrl = resolved;450 log.info(` 🍪 ${cleanRedirectUrl} → ${realPageUrl} (cookie+redirect)`);451 } else {452 log.info(` ⚠️ Could not resolve ${cleanRedirectUrl} — staying on PH`);453 }454 }455 } catch (e: any) {456 log.debug(` URL resolution via PH page failed: ${e.message?.slice(0,80)}`);457 }458 }459
460 resolvedUrl = realPageUrl;461
462 // If we couldn't resolve past PH, skip email enrichment463 if (!realPageUrl || realPageUrl.includes('producthunt.com')) {464 return { emails: [], resolvedUrl, emailSource: 'none' };465 }466
467 let parsedReal: URL;468 try { parsedReal = new URL(realPageUrl); }469 catch { return { emails: [], resolvedUrl, emailSource: 'none' }; }470 origin = parsedReal.origin;471 baseDomain = parsedReal.hostname.replace(/^www\./, '');472
473 // Skip known junk domains (app stores, social media, etc.) — now against REAL domain474 const rootDomain = baseDomain.split('.').slice(-2).join('.');475 if (SKIP_DOMAINS.has(rootDomain) || SKIP_DOMAINS.has(baseDomain)) {476 log.debug(` [skip-domain] ${baseDomain} is in skip list — no email search`);477 return { emails: [], resolvedUrl, emailSource: 'none' };478 }479
480 // ── Step 2: Hunter.io fast-path (if API key configured) ──────────────────481 const { emails: hunterEmails } = await tryHunterIo(baseDomain);482 if (hunterEmails.length > 0) {483 log.info(` 🎯 Hunter.io → ${hunterEmails.length} email(s) for ${baseDomain}`);484 return { emails: hunterEmails, resolvedUrl, emailSource: 'hunter_io' };485 }486
487 // ── Navigate to the real website for email scraping ───────────────────────488 // After URL resolution, page may be at PH or unnavigated — go to real site.489 const currentPageUrl = page.url().split('?')[0].replace(/\/$/, '');490 const alreadyAtRealSite = currentPageUrl.startsWith(origin);491 if (!alreadyAtRealSite) {492 try {493 await page.goto(realPageUrl, { waitUntil: 'domcontentloaded', timeout: 15000 });494 await page.waitForTimeout(1500);495 } catch (e: any) {496 const tag = e.message?.includes('Timeout') ? '[timeout]' : '[network]';497 log.debug(` ${tag} Could not load ${realPageUrl}: ${e.message?.slice(0,80)}`);498 return { emails: [], resolvedUrl, emailSource: 'none' };499 }500 }501
502 pagesVisited = 1; // landing page loaded503
504 // ── Step 3: Extract emails from already-loaded landing page ───────────────505 const landingEmails = await page.evaluate(browserExtractEmails, baseDomain);506 if (landingEmails.length > 0) return { emails: landingEmails, resolvedUrl, emailSource: 'page_scrape' };507
508 // ── Step 4: Harvest internal links from landing page ─────────────────────509 type QueueEntry = { url: string; priority: number };510 const visited = new Set<string>([realPageUrl, rawWebsiteUrl]);511 const queue: QueueEntry[] = [];512
513 const landingLinks = await page.evaluate(browserExtractInternalLinks, origin);514 for (const link of landingLinks) {515 if (!visited.has(link)) {516 const path = link.replace(origin, '') || '/';517 queue.push({ url: link, priority: urlPriority(path) });518 }519 }520
521 // Also inject seed paths that weren't discovered naturally522 for (const path of SEED_PATHS) {523 const url = `${origin}${path}`;524 if (!visited.has(url) && !queue.some(q => q.url === url)) {525 queue.push({ url, priority: urlPriority(path) });526 }527 }528
529 // Sort descending — highest priority visited first530 queue.sort((a, b) => b.priority - a.priority);531
532 // ── Step 5: Crawl up to MAX_PAGES_PER_DOMAIN pages ───────────────────────533 while (queue.length > 0 && pagesVisited < MAX_PAGES_PER_DOMAIN) {534 const entry = queue.shift()!;535 if (visited.has(entry.url)) continue;536 visited.add(entry.url);537 pagesVisited++;538
539 try {540 const response = await page.goto(entry.url, {541 waitUntil: 'domcontentloaded',542 timeout: 14000,543 });544
545 const ct = response?.headers()['content-type'] ?? '';546 if (!ct.includes('text/') && !ct.includes('html')) continue;547
548 // Let React / Vue / Svelte hydrate549 await page.waitForTimeout(1500);550
551 const found = await page.evaluate(browserExtractEmails, baseDomain);552 if (found.length > 0) {553 emails = found;554 break; // stop crawling — email found, mark source below555 }556
557 // Expand queue with newly discovered internal links558 if (pagesVisited < MAX_PAGES_PER_DOMAIN) {559 const links = await page.evaluate(browserExtractInternalLinks, origin);560 for (const link of links) {561 if (!visited.has(link) && !queue.some(q => q.url === link)) {562 const path = link.replace(origin, '') || '/';563 queue.push({ url: link, priority: urlPriority(path) });564 }565 }566 queue.sort((a, b) => b.priority - a.priority);567 }568 } catch { continue; }569 }570 } catch (err: any) {571 const tag = err.message?.includes('Timeout') ? '[timeout]'572 : err.message?.includes('net::ERR') ? '[network]'573 : '[error]';574 log.debug(` ${tag} Navigation failed for ${rawWebsiteUrl}: ${err.message?.slice(0, 100)}`);575 } finally {576 await context.close();577 }578
579 const emailSource: EmailSource = emails.length > 0 ? 'page_scrape' : 'none';580 log.debug(` crawled ${pagesVisited} page(s) for ${baseDomain || rawWebsiteUrl} → ${emailSource}`);581 return { emails, resolvedUrl, emailSource };582}583
584// ─── TYPE HELPERS ─────────────────────────────────────────────────────────────585function toInt(v: unknown): number | null {586 if (v === null || v === undefined || v === '') return null;587 const n = parseInt(String(v), 10);588 return isNaN(n) ? null : n;589}590function toFloat(v: unknown): number | null {591 if (v === null || v === undefined || v === '') return null;592 const n = parseFloat(String(v));593 return isNaN(n) ? null : n;594}595
596// ─── NODE → RECORD ────────────────────────────────────────────────────────────597function nodeToRecord(node: any, rank: number): Record<string, unknown> {598 const makers = (node.makers ?? []).map((m: any) => ({599 maker_name: m.name ?? null,600 maker_id: m.username ?? null,601 maker_ph_id: m.id ?? null,602 twitter_url: m.twitterUsername ? `https://twitter.com/${m.twitterUsername}` : null,603 maker_url: m.url ?? null,604 }));605 const topics = (node.topics?.edges ?? []).map((e: any) => e?.node?.name).filter(Boolean);606 return {607 product_name: node.name ? node.name.split(/[-:—]/)[0].trim() : null,608 tagline: node.tagline ?? null,609 description: node.description ?? null,610 upvote_count: toInt(node.votesCount),611 comment_count: toInt(node.commentsCount),612 reviews_count: toInt(node.reviewsCount),613 reviews_rating: toFloat(node.reviewsRating),614 daily_rank: toInt(node.dailyRank) ?? rank,615 weekly_rank: toInt(node.weeklyRank),616 monthly_rank: toInt(node.monthlyRank),617 yearly_rank: toInt(node.yearlyRank),618 launch_date: node.featuredAt ? new Date(node.featuredAt).toISOString().split('T')[0]619 : node.createdAt ? new Date(node.createdAt).toISOString().split('T')[0] : null,620 product_hunt_url: node.url ?? (node.slug ? `https://www.producthunt.com/posts/${node.slug}` : null),621 website_url: node.website ?? null, // resolved to real domain in Phase 2622 topics,623 thumbnail_url: node.thumbnail?.url ?? null,624 makers,625 featured: !!node.featuredAt,626 emails: [] as string[],627 email_source: 'none' as string, // updated in processNode after enrichment628 scraped_at: new Date().toISOString(),629 };630}631
632// ─── OUTPUT MODE FILTER ───────────────────────────────────────────────────────633// lean — minimal fields for fast browsing / LLM context windows634// leads — full fields, but only products where emails were found635// full — everything (default)636const LEAN_FIELDS = new Set([637 'product_name', 'tagline', 'website_url', 'upvote_count', 'daily_rank',638 'emails', 'email_source', 'topics', 'launch_date', 'product_hunt_url',639 'featured', 'scraped_at',640]);641
642function applyOutputMode(643 record: Record<string, unknown>,644 mode: 'full' | 'lean' | 'leads',645): Record<string, unknown> | null {646 if (mode === 'leads' && (record.emails as string[]).length === 0) return null;647 if (mode === 'lean') {648 const out: Record<string, unknown> = {};649 for (const k of LEAN_FIELDS) out[k] = record[k] ?? null;650 return out;651 }652 return record;653}654
655// ─── QUERY VAR BUILDER ────────────────────────────────────────────────────────656const dateRange = buildDateRange();657
658function buildVars(after: string | null, remaining: number) {659 const pageSize = Math.min(20, remaining);660 if (mode === 'search') return { query: searchQuery, first: pageSize, after };661 const vars: Record<string, unknown> = { first: pageSize, after, order: 'VOTES' };662 if (!allProducts) vars.featured = true;663 if (dateRange) { vars.postedAfter = dateRange.postedAfter; vars.postedBefore = dateRange.postedBefore; }664 if (mode === 'topic') vars.topic = topic;665 return vars;666}667
668// ─── BROWSER (lazy — only launched when first product needs enrichment) ───────669let browser: Browser | null = null;670async function ensureBrowser(): Promise<Browser> {671 if (!browser) {672 browser = await chromium.launch({673 headless: true,674 args: [675 '--no-sandbox',676 '--disable-setuid-sandbox',677 '--disable-dev-shm-usage',678 '--disable-accelerated-2d-canvas',679 '--disable-gpu',680 '--no-zygote',681 '--disable-extensions',682 '--disable-background-networking',683 '--disable-default-apps',684 '--disable-blink-features=AutomationControlled',685 ],686 });687 log.info('Playwright browser launched.');688 }689 return browser;690}691
692// ─── PROCESS ONE NODE → pushData ─────────────────────────────────────────────693async function processNode(node: any, rank: number): Promise<void> {694 const record = nodeToRecord(node, rank);695
696 const rawUrl = typeof record.website_url === 'string' ? record.website_url : '';697 const phProductUrl = typeof record.product_hunt_url === 'string'698 ? record.product_hunt_url.split('?')[0]699 : undefined;700
701 if (enrichEmails && rawUrl.startsWith('http')) {702 const br = await ensureBrowser();703 try {704 const { emails, resolvedUrl, emailSource } = await findEmails(rawUrl, br, phProductUrl);705 record.website_url = resolvedUrl;706 record.emails = emails;707 record.email_source = emailSource;708 if (emails.length > 0) {709 log.info(` ✉ ${record.product_name}: ${emails.length} email(s) [${emailSource}] → ${emails.join(', ')}`);710 }711 } catch (err: any) {712 const msg = err.message ?? '';713 const tag = msg.includes('Timeout') || msg.includes('timeout') ? '[timeout]'714 : msg.includes('net::ERR') || msg.includes('ECONNREFUSED') ? '[network]'715 : msg.includes('Navigation') ? '[navigation]'716 : '[error]';717 log.warning(` Email enrichment failed ${tag} for ${record.product_name}: ${msg.slice(0, 100)}`);718 record.website_url = await resolveToRealUrl(rawUrl);719 }720 } else if (rawUrl.startsWith('http')) {721 record.website_url = await resolveToRealUrl(rawUrl);722 }723
724 const outputRecord = applyOutputMode(record, outputMode);725 if (outputRecord !== null) {726 await Actor.pushData(outputRecord);727 await Actor.charge({ eventName: 'product-scraped', count: 1 });728 }729}730
731// ─── STREAMING FETCH + PROCESS ────────────────────────────────────────────────732// Each page of products is fetched from the PH API and immediately processed733// (email enrichment + pushData) before fetching the next page.734// This guarantees that every product collected so far is in the dataset even735// if the run is aborted mid-way — no data is ever held only in memory.736log.info('Fetching products from ProductHunt API (streaming)...', {737 dateRange: dateRange ?? 'no date filter', allProducts, maxResults,738});739
740const seenIds = new Set<string>();741let cursor: string | null = null;742let pageNum = 1;743let totalCollected = 0;744let totalProcessed = 0;745const batchSize = enrichEmails ? Math.min(maxConcurrency, 3) : maxConcurrency;746
747while (totalCollected < maxResults) {748 log.info(`Page ${pageNum} (collected ${totalCollected}/${maxResults})...`);749 let data: any;750 try {751 data = await phQuery(752 mode === 'search' ? SEARCH_QUERY : POSTS_QUERY,753 buildVars(cursor, maxResults - totalCollected),754 );755 } catch (err: any) {756 log.error(`PH API error on page ${pageNum}: ${err.message}`);757 break;758 }759
760 const edges = data?.posts?.edges ?? [];761 const pageInfo = data?.posts?.pageInfo ?? {};762 if (!edges.length) { log.info('No more results from PH API.'); break; }763
764 // Collect unique nodes from this page765 const pageNodes: any[] = [];766 for (const { node } of edges) {767 if (totalCollected >= maxResults) break;768 if (!node?.id || seenIds.has(node.id)) continue;769 seenIds.add(node.id);770 pageNodes.push(node);771 totalCollected++;772 }773 log.info(`Page ${pageNum}: +${pageNodes.length} products (total: ${totalCollected})`);774
775 // ── Immediately process + push this page's products ────────────────────────776 for (let i = 0; i < pageNodes.length; i += batchSize) {777 const batch = pageNodes.slice(i, i + batchSize);778 await Promise.all(batch.map((node, j) => processNode(node, totalProcessed + i + j + 1)));779 }780 totalProcessed += pageNodes.length;781 if (totalProcessed % 50 === 0 || !pageInfo.hasNextPage) {782 log.info(`✓ Pushed ${totalProcessed} products to dataset so far`);783 }784
785 if (!pageInfo.hasNextPage || !pageInfo.endCursor) { log.info('No more pages.'); break; }786 cursor = pageInfo.endCursor;787 pageNum++;788}789
790// ─── CLEANUP ──────────────────────────────────────────────────────────────────791if (browser) {792 await browser.close();793 log.info('Playwright browser closed.');794}795
796log.info(`Done! ${totalProcessed} products scraped and pushed.`);797await Actor.setStatusMessage(`Run complete. If this saved you time, please leave a 5★ review at apify.com/khadinakbar/producthunt-scraper-pro/issues — your feedback keeps this actor maintained and free of bugs.`);798await Actor.exit();