1import { ControlledActor as Actor, managedEnvironment, managedFetch as fetch } from './cost-control.mjs';
2
3import { log } from 'crawlee';
4import { load } from 'cheerio';
5import { lookup } from 'node:dns/promises';
6import { isIP } from 'node:net';
7import { chromium, type Browser } from 'playwright';
8import { getDomain } from 'tldts';
9import { clampToFreePlan, freePlanLimitHit, freePlanLimitSummary, freePlanStatus, resolveFreePlanCap } from './free-plan-limit.js';
10import {
11 buildWebsitePagePlan,
12 canonicalWebsiteUrl,
13 clampInteger,
14 decodeCloudflareEmail,
15 DEFAULT_MAX_EMAILS_PER_PRODUCT,
16 DEFAULT_MAX_WEBSITE_PAGES,
17 emailCandidateLimit,
18 extractEmailCandidatesFromText,
19 hasUnexcludedProductHuntLaunch,
20 hasReachedUsableEmailTarget,
21 isRetryableEmailListVerifyHttpStatus,
22 isProductHuntHost,
23 isPublicHttpUrl,
24 MAX_EMAILS_PER_PRODUCT,
25 MAX_WEBSITE_PAGES,
26 normalizeEmailListVerifyStatus,
27 parseProductHuntSeedUrl,
28 productHuntLaunchKey,
29 productHuntRateLimitDelaySeconds,
30 productHuntStreamOrder,
31 shouldUseLightweightProductHuntIndex,
32 rankEmails,
33 resumableCheckpointCursor,
34 type EmailListVerifyStatus,
35 type WebsitePage,
36} from './enrichment-utils.js';
37import {
38 buildLeadProfile,
39 buildDomainHypothesisCandidates,
40 buildProtectedProductCandidates,
41 buildProductIdentityNames,
42 buildWebsiteSearchQueries,
43 evaluateWebsiteIdentity,
44 hasDirectOwnershipSignal,
45 hasProtectedProductVendorMatch,
46 isPrivateNetworkAddress,
47 isProtectedProductHostAllowed,
48 isRetryableDataForSeoStatus,
49 isUsableEmailStatus,
50 matchesProductSearch,
51 normalizeContactFinderResults,
52 parseDataForSeoCandidates,
53 parseSerpApiCandidates,
54 parseScrappaProductHuntCandidates,
55 rankMakerWebsiteLinkCandidates,
56 scoreSearchCandidate,
57 selectMakerFallbackEmails,
58 splitMakerName,
59 type ContactFinderResult,
60 type MakerWebsiteLink,
61 type SearchCandidate,
62} from './lead-engine.js';
63
64await Actor.init();
65
66type Mode = 'leaderboard' | 'search' | 'topic' | 'urls';
67type LeaderboardPeriod = 'daily' | 'weekly' | 'monthly' | 'yearly';
68type RunOutcome = 'COMPLETE' | 'PARTIAL' | 'VALID_EMPTY' | 'INVALID_INPUT' | 'UPSTREAM_FAILED' | 'CONFIG_ERROR';
69type EmailSource = 'page_scrape' | 'maker_website' | 'emaillistverify_contact_finder' | 'mixed' | 'none';
70type EmailResult = 'emails_found' | 'no_public_email_found' | 'not_assessed';
71type Coverage = 'complete' | 'partial' | 'none';
72type WebsiteResolutionStatus = 'verified_external' | 'verified_search_match' | 'unresolved' | 'not_available';
73type WebsiteResolutionSource = 'api_website_product_link' | 'input_override' | 'product_hunt_redirect' | 'maker_website' | 'maker_website_link' | 'protected_vendor_domain' | 'dataforseo_organic' | 'direct_domain_hypothesis' | 'serpapi_organic' | 'scrappa_product' | 'scrappa_organic' | 'none';
74type EmailVerification = {
75 email: string;
76 provider: 'emaillistverify';
77 status: EmailListVerifyStatus;
78 checkedAt: string | null;
79 source: 'public_website' | 'maker_website' | 'emaillistverify_contact_finder';
80 confidence: 'high' | 'medium' | 'low' | 'unknown' | null;
81};
82
83type DateRange = { postedAfter: string; postedBefore: string };
84type WebsiteOverride = { keys: string[]; websiteUrl: string };
85type EnrichmentResult = {
86 emails: string[];
87 emailSource: EmailSource;
88 result: EmailResult;
89 coverage: Coverage;
90 terminalReason: string;
91 pagesVisited: number;
92 pageUrls: string[];
93 warnings: string[];
94};
95
96type WebsiteResolution = {
97 websiteUrl: string | null;
98 source: WebsiteResolutionSource;
99 status: WebsiteResolutionStatus;
100 confidence: number;
101 signals: string[];
102 providerCostUsd: number;
103 providerAttempts: number;
104 warnings: string[];
105};
106
107type ContactDiscovery = {
108 contacts: ContactFinderResult[];
109 attempts: number;
110 credits: number;
111 warnings: string[];
112};
113
114const PH_API = 'https://api.producthunt.com/v2/api/graphql';
115const RESULT_EVENT = 'apify-default-dataset-item';
116const EMAIL_EVENT = 'email-found';
117const RESULT_PRICE_USD = 0.002;
118const EMAIL_PRICE_USD = 0.04;
119const EMAIL_LIST_VERIFY_API = 'https://api.emaillistverify.com/api/verifyEmail';
120const EMAIL_LIST_VERIFY_CONTACT_API = 'https://api.emaillistverify.com/api/findContact';
121const DATAFORSEO_SEARCH_API = 'https://api.dataforseo.com/v3/serp/google/organic/live/advanced';
122const SERPAPI_SEARCH_API = 'https://serpapi.com/search.json';
123const SCRAPPA_API = 'https://scrappa.co/api';
124const EMAIL_LIST_VERIFY_MIN_INTERVAL_MS = 225;
125const SEARCH_PAGES_WITHOUT_MATCH_LIMIT = 50;
126const FAILED_OUTCOMES = new Set<RunOutcome>(['UPSTREAM_FAILED', 'CONFIG_ERROR']);
127
128const SKIP_DOMAINS = new Set([
129 'apps.apple.com', 'play.google.com', 'github.com', 'youtube.com',
130 'twitter.com', 'x.com', 'linkedin.com', 'facebook.com', 'instagram.com',
131 'reddit.com', 'discord.com', 'discord.gg', 'slack.com', 'medium.com',
132 'substack.com', 'producthunt.com', 'amazon.com', 'chrome.google.com',
133 'marketplace.visualstudio.com', 'notion.so', 'figma.com', 'trello.com',
134 'airtable.com',
135]);
136
137const PRODUCT_HUNT_USER_AGENT = 'ProductHunt Lead Generator/1.4 (authorized API client)';
138const WEBSITE_USER_AGENT = 'Mozilla/5.0 (compatible; ProductHuntLeadGenerator/1.4; +https://apify.com/khadinakbar/producthunt-scraper-pro)';
139
140let nextEmailListVerifyRequestAt = 0;
141let emailListVerifyCreditsExhausted = false;
142const publicHostCache = new Map<string, Promise<boolean>>();
143
144
145const MAX_BROWSER_CONTEXTS = 2;
146let activeBrowserContexts = 0;
147const browserContextWaiters: Array<() => void> = [];
148
149async function withBrowserContextSlot<T>(task: () => Promise<T>): Promise<T> {
150 if (activeBrowserContexts >= MAX_BROWSER_CONTEXTS) {
151 await new Promise<void>((resolve) => browserContextWaiters.push(resolve));
152 } else {
153 activeBrowserContexts += 1;
154 }
155 try {
156 return await task();
157 } finally {
158 const next = browserContextWaiters.shift();
159 if (next) next();
160 else activeBrowserContexts -= 1;
161 }
162}
163
164function isoDate(date: Date): string {
165 return date.toISOString().slice(0, 10);
166}
167
168function getWeekBounds(date: Date): { start: string; end: string } {
169 const start = new Date(date);
170
171 const weekday = date.getUTCDay() || 7;
172 start.setUTCDate(date.getUTCDate() - weekday + 1);
173 const end = new Date(start);
174 end.setUTCDate(start.getUTCDate() + 6);
175 return { start: isoDate(start), end: isoDate(end) };
176}
177
178function getMonthBounds(date: Date): { start: string; end: string } {
179 const year = date.getUTCFullYear();
180 const month = date.getUTCMonth();
181 return {
182 start: isoDate(new Date(Date.UTC(year, month, 1))),
183 end: isoDate(new Date(Date.UTC(year, month + 1, 0))),
184 };
185}
186
187function getYearBounds(date: Date): { start: string; end: string } {
188 const year = date.getUTCFullYear();
189 return { start: `${year}-01-01`, end: `${year}-12-31` };
190}
191
192function validIsoDate(value: string): boolean {
193 if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
194 const parsed = new Date(`${value}T12:00:00Z`);
195 return Number.isFinite(parsed.getTime()) && isoDate(parsed) === value;
196}
197
198function dateRangeForPeriod(period: LeaderboardPeriod, anchorDate: string, endDate = ''): DateRange {
199 const base = new Date(`${anchorDate}T12:00:00Z`);
200 if (period === 'daily') {
201 return {
202 postedAfter: `${anchorDate}T00:00:00+00:00`,
203 postedBefore: `${endDate || anchorDate}T23:59:59+00:00`,
204 };
205 }
206 const bounds = period === 'weekly'
207 ? getWeekBounds(base)
208 : period === 'monthly'
209 ? getMonthBounds(base)
210 : getYearBounds(base);
211 return {
212 postedAfter: `${bounds.start}T00:00:00+00:00`,
213 postedBefore: `${bounds.end}T23:59:59+00:00`,
214 };
215}
216
217function normalizeKey(value: unknown): string | null {
218 if (typeof value !== 'string' || !value.trim()) return null;
219 return value.trim().toLowerCase().replace(/[?#].*$/, '').replace(/\/$/, '');
220}
221
222function parseWebsiteOverrides(value: unknown): WebsiteOverride[] {
223 const overrides: WebsiteOverride[] = [];
224 const add = (keys: unknown[], websiteUrl: unknown): void => {
225 if (typeof websiteUrl !== 'string' || !isPublicHttpUrl(websiteUrl)) return;
226 const canonical = canonicalWebsiteUrl(websiteUrl);
227 if (!canonical || isProductHuntHost(new URL(canonical).hostname)) return;
228 const usableKeys = keys.map(normalizeKey).filter((key): key is string => Boolean(key));
229 if (usableKeys.length > 0) overrides.push({ keys: usableKeys, websiteUrl: canonical });
230 };
231
232 if (Array.isArray(value)) {
233 for (const entry of value) {
234 if (!entry || typeof entry !== 'object') continue;
235 const row = entry as Record<string, unknown>;
236 add([row.productHuntUrl, row.productSlug, row.productName, row.productId], row.websiteUrl);
237 }
238 } else if (value && typeof value === 'object') {
239 for (const [key, websiteUrl] of Object.entries(value as Record<string, unknown>)) add([key], websiteUrl);
240 }
241 return overrides;
242}
243
244function lookupWebsiteOverride(node: Record<string, unknown>, overrides: WebsiteOverride[]): string | null {
245 const candidates = [node.url, node.slug, node.id, node.name]
246 .map(normalizeKey)
247 .filter((value): value is string => Boolean(value));
248 return overrides.find((override) => override.keys.some((key) => candidates.includes(key)))?.websiteUrl ?? null;
249}
250
251function rootDomain(hostname: string): string {
252 const labels = hostname.replace(/^www\./i, '').toLowerCase().split('.');
253 return labels.slice(-2).join('.');
254}
255
256function isAllowedExternalWebsite(url: string): boolean {
257 if (!isPublicHttpUrl(url)) return false;
258 try {
259 const host = new URL(url).hostname.toLowerCase();
260 return !isProductHuntHost(host) && !SKIP_DOMAINS.has(host) && !SKIP_DOMAINS.has(rootDomain(host));
261 } catch {
262 return false;
263 }
264}
265
266async function isSafePublicNetworkUrl(rawUrl: string): Promise<boolean> {
267 if (!isPublicHttpUrl(rawUrl)) return false;
268 let hostname: string;
269 try {
270 hostname = new URL(rawUrl).hostname.toLowerCase().replace(/^\[|\]$/g, '');
271 } catch {
272 return false;
273 }
274 if (isIP(hostname)) return !isPrivateNetworkAddress(hostname);
275
276 let assessment = publicHostCache.get(hostname);
277 if (!assessment) {
278 assessment = lookup(hostname, { all: true, verbatim: true })
279 .then((records) => records.length > 0 && records.every(({ address }) => !isPrivateNetworkAddress(address)))
280 .catch(() => false);
281 publicHostCache.set(hostname, assessment);
282 }
283 return assessment;
284}
285
286
287
288function browserExtractEmails(): string[] {
289 const emailRe = /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g;
290 const obfuscatedRe = /\b[a-zA-Z0-9._%+\-]+\s*(?:\[at\]|\(at\)|\sat\s)\s*[a-zA-Z0-9.\-]+\s*(?:\[dot\]|\(dot\)|\sdot\s)\s*[a-zA-Z]{2,}\b/gi;
291 const collected: string[] = [];
292 const add = (value: unknown): void => {
293 if (typeof value !== 'string') return;
294 let decoded = value;
295 try { decoded = decodeURIComponent(value); } catch { }
296 const email = decoded.trim().toLowerCase().replace(/^mailto:/, '').split('?')[0];
297 if (email.includes('@')) collected.push(email);
298 };
299 const walkJson = (value: unknown, depth = 0): void => {
300 if (depth > 8 || value === null || value === undefined) return;
301 if (Array.isArray(value)) {
302 value.forEach((item) => walkJson(item, depth + 1));
303 return;
304 }
305 if (typeof value !== 'object') return;
306 for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
307 if (key.toLowerCase() === 'email') add(child);
308 walkJson(child, depth + 1);
309 }
310 };
311 const decodeCfEmail = (encoded: string): string | null => {
312 if (!/^[0-9a-f]+$/i.test(encoded) || encoded.length < 4) return null;
313 const key = Number.parseInt(encoded.slice(0, 2), 16);
314 let decoded = '';
315 for (let index = 2; index < encoded.length; index += 2) {
316 decoded += String.fromCharCode(Number.parseInt(encoded.slice(index, index + 2), 16) ^ key);
317 }
318 return decoded;
319 };
320
321 document.querySelectorAll('a[href^="mailto:"]').forEach((element) => add((element as HTMLAnchorElement).href));
322 document.querySelectorAll('[data-email]').forEach((element) => add(element.getAttribute('data-email')));
323 document.querySelectorAll('[data-cfemail]').forEach((element) => {
324 const decoded = decodeCfEmail(element.getAttribute('data-cfemail') ?? '');
325 if (decoded) add(decoded);
326 });
327 document.querySelectorAll('script[type="application/ld+json"]').forEach((element) => {
328 try { walkJson(JSON.parse(element.textContent ?? '')); } catch { }
329 });
330
331 const combined = `${document.body?.innerText ?? ''}\n${document.documentElement.innerHTML}`
332 .replace(/\\(?:x40|u0040)/gi, '@')
333 .replace(/\\(?:x2e|u002e)/gi, '.')
334 .replace(/&#(?:64|x40);|@/gi, '@')
335 .replace(/&#(?:46|x2e);|./gi, '.');
336 (combined.match(emailRe) ?? []).forEach(add);
337 for (const match of combined.match(obfuscatedRe) ?? []) {
338 add(match.replace(/\s*(?:\[at\]|\(at\)|\sat\s)\s*/i, '@').replace(/\s*(?:\[dot\]|\(dot\)|\sdot\s)\s*/ig, '.'));
339 }
340 return collected;
341}
342
343
344
345function browserExtractLinkedFallbackPages(origin: string): WebsitePage[] {
346 const matchers: Array<{ kind: Exclude<WebsitePage['kind'], 'homepage'>; priority: number; pattern: RegExp }> = [
347 { kind: 'contact', priority: 0, pattern: /(contact|contact-us|get-in-touch|reach-us)/i },
348 { kind: 'about', priority: 1, pattern: /(about|about-us|company|our-story)/i },
349 { kind: 'team', priority: 2, pattern: /(team|our-team|meet-the-team|people|founders)/i },
350 { kind: 'privacy', priority: 3, pattern: /(privacy|privacy-policy)/i },
351 { kind: 'terms', priority: 4, pattern: /(terms|terms-of-service|terms-and-conditions|conditions)/i },
352 ];
353 const links = new Map<string, WebsitePage>();
354 document.querySelectorAll('a[href]').forEach((element) => {
355 const anchor = element as HTMLAnchorElement;
356 try {
357 const url = new URL(anchor.href);
358 if (url.origin !== origin || !['http:', 'https:'].includes(url.protocol) || url.username || url.password) return;
359 const label = (anchor.innerText || anchor.textContent || '').trim();
360 const matched = matchers.find((candidate) => candidate.pattern.test(`${url.pathname} ${label}`));
361 if (!matched) return;
362 url.hash = '';
363 const clean = url.toString();
364 if (clean === `${origin}/` || clean === origin || links.has(clean)) return;
365 links.set(clean, { url: clean, kind: matched.kind, priority: matched.priority });
366 } catch { }
367 });
368 return [...links.values()].sort((left, right) => left.priority - right.priority);
369}
370
371async function waitForEmailListVerifySlot(): Promise<void> {
372 const scheduledAt = Math.max(Date.now(), nextEmailListVerifyRequestAt);
373 nextEmailListVerifyRequestAt = scheduledAt + EMAIL_LIST_VERIFY_MIN_INTERVAL_MS;
374 const delayMs = scheduledAt - Date.now();
375 if (delayMs > 0) await new Promise((resolve) => setTimeout(resolve, delayMs));
376}
377
378async function verifyEmailsWithEmailListVerify(
379 emails: string[],
380 requested: boolean,
381 apiKey: string,
382 source: EmailVerification['source'] = 'public_website',
383 maxUsableEmails = emails.length,
384): Promise<{ verifications: EmailVerification[]; attempts: number; warnings: string[] }> {
385 if (emails.length === 0) return { verifications: [], attempts: 0, warnings: [] };
386 if (!requested) {
387 return {
388 verifications: emails.map((email) => ({
389 email, provider: 'emaillistverify', status: 'not_requested', checkedAt: null,
390 source, confidence: null,
391 })),
392 attempts: 0,
393 warnings: [],
394 };
395 }
396 if (!apiKey) {
397 return {
398 verifications: emails.map((email) => ({
399 email, provider: 'emaillistverify', status: 'not_configured', checkedAt: null,
400 source, confidence: null,
401 })),
402 attempts: 0,
403 warnings: ['EmailListVerify was requested but no owner-managed API key was configured; public emails were kept unverified.'],
404 };
405 }
406 if (emailListVerifyCreditsExhausted) {
407 return {
408 verifications: emails.map((email) => ({
409 email, provider: 'emaillistverify', status: 'error_credit', checkedAt: null,
410 source, confidence: null,
411 })),
412 attempts: 0,
413 warnings: ['EmailListVerify credits were exhausted earlier in this run; no additional verification requests were sent.'],
414 };
415 }
416
417 const warnings = new Set<string>();
418 const verifications: EmailVerification[] = [];
419 let attempts = 0;
420 for (const email of emails) {
421 if (hasReachedUsableEmailTarget(verifications.map(({ status }) => status), maxUsableEmails)) break;
422 for (let requestAttempt = 0; requestAttempt < 2; requestAttempt += 1) {
423 await waitForEmailListVerifySlot();
424 attempts += 1;
425 const checkedAt = new Date().toISOString();
426 try {
427 const requestUrl = new URL(EMAIL_LIST_VERIFY_API);
428 requestUrl.searchParams.set('email', email);
429 const response = await fetch(requestUrl, {
430 headers: { 'x-api-key': apiKey, Accept: 'text/plain, application/json;q=0.9' },
431 signal: AbortSignal.timeout(12_000),
432 });
433 if (!response.ok) {
434 if (requestAttempt === 0 && isRetryableEmailListVerifyHttpStatus(response.status)) {
435 await new Promise((resolve) => setTimeout(resolve, 500));
436 continue;
437 }
438 warnings.add(`EmailListVerify returned HTTP ${response.status}; affected public emails were retained with provider_error status.`);
439 if (response.status === 402) emailListVerifyCreditsExhausted = true;
440 verifications.push({
441 email, provider: 'emaillistverify', status: response.status === 402 ? 'error_credit' : 'provider_error', checkedAt,
442 source, confidence: null,
443 });
444 break;
445 }
446 const status = normalizeEmailListVerifyStatus(await response.text());
447 if (status === 'provider_error') warnings.add('EmailListVerify returned an undocumented response; affected public emails were retained with provider_error status.');
448 if (status === 'error_credit') {
449 emailListVerifyCreditsExhausted = true;
450 warnings.add('EmailListVerify reported insufficient verification credits; remaining public emails were retained with error_credit status.');
451 }
452 verifications.push({ email, provider: 'emaillistverify', status, checkedAt, source, confidence: null });
453 break;
454 } catch {
455 if (requestAttempt === 0) {
456 await new Promise((resolve) => setTimeout(resolve, 500));
457 continue;
458 }
459 warnings.add('EmailListVerify request failed twice; affected public emails were retained with provider_error status.');
460 verifications.push({ email, provider: 'emaillistverify', status: 'provider_error', checkedAt, source, confidence: null });
461 }
462 }
463 if (emailListVerifyCreditsExhausted) break;
464 }
465 return { verifications, attempts, warnings: [...warnings] };
466}
467
468let browser: Browser | null = null;
469
470async function ensureBrowser(): Promise<Browser> {
471 if (!browser) {
472 browser = await chromium.launch({
473 headless: true,
474 args: [
475 '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage',
476 '--disable-gpu', '--no-zygote', '--disable-extensions',
477 ],
478 });
479 log.info('Browser launched for verified external website enrichment.');
480 }
481 return browser;
482}
483
484async function closeBrowser(): Promise<void> {
485 if (!browser) return;
486 await browser.close().catch(() => undefined);
487 browser = null;
488}
489
490function extractStaticEmails(html: string): string[] {
491 const $ = load(html);
492 const collected: string[] = [];
493 const add = (value: unknown): void => {
494 if (typeof value !== 'string') return;
495 let decoded = value;
496 try { decoded = decodeURIComponent(value); } catch { }
497 const email = decoded.trim().toLowerCase().replace(/^mailto:/, '').split('?')[0];
498 if (email.includes('@')) collected.push(email);
499 };
500 const walkJson = (value: unknown, depth = 0): void => {
501 if (depth > 8 || value === null || value === undefined) return;
502 if (Array.isArray(value)) return value.forEach((item) => walkJson(item, depth + 1));
503 if (typeof value !== 'object') return;
504 for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
505 if (key.toLowerCase() === 'email') add(child);
506 walkJson(child, depth + 1);
507 }
508 };
509 $('a[href^="mailto:"]').each((_, element) => add($(element).attr('href')));
510 $('[data-email]').each((_, element) => add($(element).attr('data-email')));
511 $('[data-cfemail]').each((_, element) => add(decodeCloudflareEmail($(element).attr('data-cfemail') ?? '')));
512 $('script[type="application/ld+json"]').each((_, element) => {
513 try { walkJson(JSON.parse($(element).text())); } catch { }
514 });
515 const combined = `${$.text()}\n${html}`;
516 extractEmailCandidatesFromText(combined).forEach(add);
517 return collected;
518}
519
520function extractStaticFallbackPages(html: string, origin: string): WebsitePage[] {
521 const $ = load(html);
522 const matchers: Array<{ kind: Exclude<WebsitePage['kind'], 'homepage'>; priority: number; pattern: RegExp }> = [
523 { kind: 'contact', priority: 0, pattern: /(contact|contact-us|get-in-touch|reach-us)/i },
524 { kind: 'about', priority: 1, pattern: /(about|about-us|company|our-story)/i },
525 { kind: 'team', priority: 2, pattern: /(team|our-team|meet-the-team|people|founders)/i },
526 { kind: 'privacy', priority: 3, pattern: /(privacy|privacy-policy)/i },
527 { kind: 'terms', priority: 4, pattern: /(terms|terms-of-service|terms-and-conditions|conditions)/i },
528 ];
529 const candidates = new Map<string, WebsitePage>();
530 $('a[href]').each((_, element) => {
531 try {
532 const url = new URL($(element).attr('href') ?? '', origin);
533 if (url.origin !== origin || !['http:', 'https:'].includes(url.protocol) || url.username || url.password) return;
534 const label = $(element).text().trim();
535 const matched = matchers.find((candidate) => candidate.pattern.test(`${url.pathname} ${label}`));
536 if (!matched) return;
537 url.hash = '';
538 const clean = url.toString();
539 if (clean === `${origin}/` || clean === origin || candidates.has(clean)) return;
540 candidates.set(clean, { url: clean, kind: matched.kind, priority: matched.priority });
541 } catch { }
542 });
543 return [...candidates.values()].sort((left, right) => left.priority - right.priority);
544}
545
546type StaticPage = { ok: true; url: string; html: string } | { ok: false; reason: string };
547
548async function fetchPublicHtml(rawUrl: string, timeoutMs: number): Promise<StaticPage> {
549 let current = canonicalWebsiteUrl(rawUrl);
550 if (!current || !isAllowedExternalWebsite(current)) return { ok: false, reason: 'invalid_or_private_target' };
551 try {
552 for (let hop = 0; hop <= 3; hop += 1) {
553 if (!await isSafePublicNetworkUrl(current)) return { ok: false, reason: 'private_or_unresolvable_target' };
554 const response = await fetch(current, {
555 method: 'GET',
556 redirect: 'manual',
557 headers: { Accept: 'text/html,application/xhtml+xml', 'User-Agent': WEBSITE_USER_AGENT },
558 signal: AbortSignal.timeout(timeoutMs),
559 });
560 if (response.status >= 300 && response.status < 400) {
561 const location = response.headers.get('location');
562 if (!location || hop === 3) return { ok: false, reason: 'redirect_hop_limit' };
563 const next = canonicalWebsiteUrl(new URL(location, current).toString());
564 if (!next || !isAllowedExternalWebsite(next)) return { ok: false, reason: 'redirect_to_private_or_unsupported_target' };
565 if (!await isSafePublicNetworkUrl(next)) return { ok: false, reason: 'redirect_to_private_or_unresolvable_target' };
566 current = next;
567 continue;
568 }
569 if (!response.ok) return { ok: false, reason: `http_${response.status}` };
570 const contentType = response.headers.get('content-type') ?? '';
571 if (!/(?:text\/html|application\/xhtml\+xml)/i.test(contentType)) return { ok: false, reason: 'non_html_response' };
572 const contentLength = Number(response.headers.get('content-length') ?? '0');
573 if (Number.isFinite(contentLength) && contentLength > 1_500_000) return { ok: false, reason: 'response_too_large' };
574 const html = (await response.text()).slice(0, 1_500_000);
575 return { ok: true, url: current, html };
576 }
577 return { ok: false, reason: 'redirect_hop_limit' };
578 } catch (error) {
579 const message = error instanceof Error ? error.message : '';
580 return { ok: false, reason: /timeout|abort/i.test(message) ? 'timeout' : 'network_error' };
581 }
582}
583
584function pageIdentity(html: string): { title: string; description: string; text: string } {
585 const $ = load(html);
586 $('script,style,noscript,svg').remove();
587 return {
588 title: $('title').first().text().trim(),
589 description: $('meta[name="description"]').attr('content')?.trim()
590 || $('meta[property="og:description"]').attr('content')?.trim()
591 || '',
592 text: $('body').text().replace(/\s+/g, ' ').trim().slice(0, 100_000),
593 };
594}
595
596async function resolveProductHuntRedirect(rawUrl: string): Promise<string | null> {
597 let current = canonicalWebsiteUrl(rawUrl);
598 if (!current) return null;
599 try {
600 if (!isProductHuntHost(new URL(current).hostname)) return null;
601 for (let hop = 0; hop < 4; hop += 1) {
602 if (!await isSafePublicNetworkUrl(current)) return null;
603 let response = await fetch(current, {
604 method: 'HEAD',
605 redirect: 'manual',
606 headers: { 'User-Agent': PRODUCT_HUNT_USER_AGENT, Accept: 'text/html,*/*;q=0.5' },
607 signal: AbortSignal.timeout(8_000),
608 });
609 if (response.status === 405) {
610 response = await fetch(current, {
611 method: 'GET',
612 redirect: 'manual',
613 headers: { 'User-Agent': PRODUCT_HUNT_USER_AGENT, Accept: 'text/html,*/*;q=0.5' },
614 signal: AbortSignal.timeout(8_000),
615 });
616 }
617 if (response.status < 300 || response.status >= 400) return null;
618 const location = response.headers.get('location');
619 if (!location) return null;
620 const next = canonicalWebsiteUrl(new URL(location, current).toString());
621 if (!next) return null;
622 if (!isProductHuntHost(new URL(next).hostname)) {
623 return isAllowedExternalWebsite(next) && await isSafePublicNetworkUrl(next) ? next : null;
624 }
625 current = next;
626 }
627 } catch {
628 return null;
629 }
630 return null;
631}
632
633async function verifyWebsiteCandidates(
634 candidates: SearchCandidate[],
635 productName: string,
636 tagline: string,
637 description = '',
638): Promise<{ websiteUrl: string; confidence: number; signals: string[] } | null> {
639 const identityNames = buildProductIdentityNames(productName);
640 const purpose = `${tagline} ${description}`.trim().slice(0, 1_200);
641 const ranked = [...candidates]
642 .map((candidate) => ({
643 candidate,
644 score: Math.max(...identityNames.map((identity) => scoreSearchCandidate(candidate, identity, purpose))),
645 }))
646 .filter(({ score }) => score >= 34)
647 .sort((left, right) => right.score - left.score || left.candidate.rank - right.candidate.rank)
648 .slice(0, 5);
649
650 for (const { candidate } of ranked) {
651 let candidateUrl: URL;
652 try {
653 candidateUrl = new URL(candidate.url);
654 } catch {
655 continue;
656 }
657 if (!isProtectedProductHostAllowed(productName, candidateUrl.toString())) continue;
658 const homepage = `${candidateUrl.origin}/`;
659 const verificationUrls = candidateUrl.pathname !== '/' || candidateUrl.search
660 ? [candidateUrl.toString(), homepage]
661 : [homepage];
662 for (const verificationUrl of [...new Set(verificationUrls)]) {
663 const fetched = await fetchPublicHtml(verificationUrl, 6_000);
664 if (!fetched.ok) continue;
665 const identity = pageIdentity(fetched.html);
666 const assessments = identityNames.map((identityName) => ({
667 identityName,
668 assessment: evaluateWebsiteIdentity({
669 candidate: { ...candidate, url: fetched.url },
670 productName: identityName,
671 tagline: purpose,
672 pageTitle: identity.title,
673 pageDescription: identity.description,
674 pageText: identity.text,
675 }),
676 })).filter(({ assessment }) => assessment.accepted)
677 .sort((left, right) => right.assessment.confidence - left.assessment.confidence);
678 if (assessments.length === 0) continue;
679 const { assessment, identityName } = assessments[0];
680 if (new URL(verificationUrl).pathname !== '/'
681 && !hasDirectOwnershipSignal(assessment.signals)
682 && !hasProtectedProductVendorMatch(productName, verificationUrl)) {
683 continue;
684 }
685 const finalUrl = new URL(fetched.url);
686 const signals = [...assessment.signals];
687 if (identityName !== identityNames[0]) signals.push('core_brand_alias_confirmed');
688 if (new URL(verificationUrl).pathname !== '/') signals.push('official_product_page_confirmed');
689 return {
690 websiteUrl: `${finalUrl.origin}/`,
691 confidence: assessment.confidence,
692 signals: [...new Set(signals)],
693 };
694 }
695 }
696 return null;
697}
698
699async function discoverMakerLinkedProductCandidates(
700 makerWebsiteUrls: string[],
701 productName: string,
702 tagline: string,
703): Promise<SearchCandidate[]> {
704 const collected = new Map<string, SearchCandidate>();
705 for (const makerWebsiteUrl of makerWebsiteUrls.filter((url) => isAllowedExternalWebsite(url)).slice(0, 2)) {
706 const fetched = await fetchPublicHtml(makerWebsiteUrl, 6_000);
707 if (!fetched.ok) continue;
708 const makerOrigin = new URL(fetched.url).origin;
709 const $ = load(fetched.html);
710 const links: MakerWebsiteLink[] = [];
711 $('a[href]').each((index, element) => {
712 try {
713 const target = new URL($(element).attr('href') ?? '', fetched.url);
714 if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) return;
715 if (target.origin === makerOrigin) return;
716 const text = $(element).text().replace(/\s+/g, ' ').trim();
717 const context = $(element).closest('p,li,article,section,div').first().text().replace(/\s+/g, ' ').trim();
718 links.push({ url: target.toString(), text, context, rank: index + 1 });
719 } catch { }
720 });
721 for (const candidate of rankMakerWebsiteLinkCandidates(links, fetched.url, productName, tagline)) {
722 let key: string;
723 try { key = new URL(candidate.url).origin; } catch { continue; }
724 if (!collected.has(key)) collected.set(key, candidate);
725 }
726 }
727 return [...collected.values()]
728 .sort((left, right) => scoreSearchCandidate(right, productName, tagline) - scoreSearchCandidate(left, productName, tagline))
729 .slice(0, 5);
730}
731
732class DataForSeoSearchError extends Error {
733 constructor(message: string, readonly retryable: boolean, readonly costUsd: number) {
734 super(message);
735 }
736}
737
738async function callDataForSeoWebsiteSearch(
739 query: string,
740 login: string,
741 password: string,
742): Promise<{ candidates: SearchCandidate[]; costUsd: number }> {
743 const response = await fetch(DATAFORSEO_SEARCH_API, {
744 method: 'POST',
745 headers: {
746 Authorization: `Basic ${Buffer.from(`${login}:${password}`).toString('base64')}`,
747 'Content-Type': 'application/json',
748 Accept: 'application/json',
749 },
750 body: JSON.stringify([{
751 keyword: query,
752 location_code: 2840,
753 language_code: 'en',
754 device: 'desktop',
755 os: 'windows',
756 depth: 10,
757 group_organic_results: true,
758 }]),
759 signal: AbortSignal.timeout(45_000),
760 });
761 if (!response.ok) {
762 throw new DataForSeoSearchError(
763 `Primary website search returned HTTP ${response.status}.`,
764 response.status === 429 || response.status >= 500,
765 0,
766 );
767 }
768 const body = await response.json() as Record<string, any>;
769 const parsed = parseDataForSeoCandidates(body);
770 if (body.status_code && body.status_code !== 20000) {
771 throw new DataForSeoSearchError(
772 `Primary website search failed with provider code ${body.status_code}: ${String(body.status_message ?? 'unknown error')}.`,
773 isRetryableDataForSeoStatus(Number(body.status_code)),
774 parsed.costUsd,
775 );
776 }
777 const task = Array.isArray(body.tasks) ? body.tasks[0] : null;
778 if (task?.status_code && task.status_code !== 20000) {
779 throw new DataForSeoSearchError(
780 `Primary website search task failed with provider code ${task.status_code}: ${String(task.status_message ?? 'unknown error')}.`,
781 isRetryableDataForSeoStatus(Number(task.status_code)),
782 parsed.costUsd,
783 );
784 }
785 return parsed;
786}
787
788async function callSerpApiWebsiteSearch(query: string, apiKey: string): Promise<SearchCandidate[]> {
789 const url = new URL(SERPAPI_SEARCH_API);
790 url.searchParams.set('engine', 'google');
791 url.searchParams.set('q', query);
792 url.searchParams.set('num', '10');
793 url.searchParams.set('gl', 'us');
794 url.searchParams.set('hl', 'en');
795 url.searchParams.set('api_key', apiKey);
796 const response = await fetch(url, { signal: AbortSignal.timeout(30_000), headers: { Accept: 'application/json' } });
797 if (!response.ok) throw new Error(`Backup website search returned HTTP ${response.status}.`);
798 const body = await response.json() as Record<string, unknown>;
799 if (body.error) throw new Error('Backup website search returned a provider error.');
800 return parseSerpApiCandidates(body);
801}
802
803
804
805let scrappaUnavailable = false;
806
807async function scrappaGet(path: string, params: Record<string, string>, apiKey: string): Promise<Record<string, unknown> | null> {
808 if (scrappaUnavailable) throw new Error('Scrappa website lookup skipped for this run.');
809 const url = new URL(`${SCRAPPA_API}${path}`);
810 for (const [key, value] of Object.entries(params)) url.searchParams.set(key, value);
811 const response = await fetch(url, {
812 headers: { 'x-api-key': apiKey, Accept: 'application/json' },
813 signal: AbortSignal.timeout(30_000),
814 });
815 if ([401, 402, 403, 429].includes(response.status)) {
816 scrappaUnavailable = true;
817 throw new Error(`Scrappa website lookup returned HTTP ${response.status}.`);
818 }
819 if (response.status === 404) return null;
820 if (!response.ok) throw new Error(`Scrappa website lookup returned HTTP ${response.status}.`);
821 return await response.json() as Record<string, unknown>;
822}
823
824async function callScrappaProductHuntWebsite(postSlug: string, apiKey: string): Promise<SearchCandidate[]> {
825 const post = await scrappaGet(`/producthunt/posts/${encodeURIComponent(postSlug)}`, {}, apiKey);
826 if (!post) return [];
827 const productSlug = (post.data as Record<string, any> | undefined)?.product?.slug;
828 const product = typeof productSlug === 'string' && productSlug
829 ? await scrappaGet(`/producthunt/products/${encodeURIComponent(productSlug)}`, {}, apiKey)
830 : null;
831 return parseScrappaProductHuntCandidates(post, product);
832}
833
834async function callScrappaWebsiteSearch(query: string, apiKey: string): Promise<SearchCandidate[]> {
835 const body = await scrappaGet('/search', { query, gl: 'us', hl: 'en' }, apiKey);
836 return body ? parseSerpApiCandidates(body) : [];
837}
838
839async function resolveWebsite(input: {
840 productName: string;
841 tagline: string;
842 description: string;
843 productHuntRedirect: string;
844 makerWebsiteUrls: string[];
845 dataForSeoLogin: string;
846 dataForSeoPassword: string;
847 serpApiKey: string;
848 scrappaKey: string;
849 productHuntSlug: string;
850}): Promise<WebsiteResolution> {
851 const warnings: string[] = [];
852 let providerCostUsd = 0;
853 let providerAttempts = 0;
854 const unresolved = (): WebsiteResolution => ({
855 websiteUrl: null,
856 source: 'none',
857 status: input.productHuntRedirect ? 'unresolved' : 'not_available',
858 confidence: 0,
859 signals: [],
860 providerCostUsd,
861 providerAttempts,
862 warnings,
863 });
864
865 if (input.productHuntRedirect) {
866 const destination = await resolveProductHuntRedirect(input.productHuntRedirect);
867 if (destination) {
868 const verified = await verifyWebsiteCandidates([{
869 url: destination, title: input.productName, snippet: input.tagline, rank: 1,
870 }], input.productName, input.tagline, input.description);
871 if (verified) {
872 return {
873 ...verified,
874 source: 'product_hunt_redirect',
875 status: 'verified_external',
876 providerCostUsd,
877 providerAttempts,
878 warnings,
879 };
880 }
881 }
882 }
883
884 const makerCandidates = input.makerWebsiteUrls
885 .filter((url) => isAllowedExternalWebsite(url))
886 .slice(0, 2)
887 .map((url, index): SearchCandidate => ({ url, title: '', snippet: '', rank: index + 1 }));
888 if (makerCandidates.length > 0) {
889 const verified = await verifyWebsiteCandidates(makerCandidates, input.productName, input.tagline, input.description);
890 if (verified) {
891 return {
892 ...verified,
893 source: 'maker_website',
894 status: 'verified_external',
895 providerCostUsd,
896 providerAttempts,
897 warnings,
898 };
899 }
900
901 const linkedCandidates = await discoverMakerLinkedProductCandidates(
902 input.makerWebsiteUrls,
903 input.productName,
904 input.tagline,
905 );
906 if (linkedCandidates.length > 0) {
907 const linked = await verifyWebsiteCandidates(linkedCandidates, input.productName, input.tagline, input.description);
908 if (linked) {
909 return {
910 ...linked,
911 signals: [...new Set([...linked.signals, 'product_linked_from_maker_website'])],
912 source: 'maker_website_link',
913 status: 'verified_external',
914 providerCostUsd,
915 providerAttempts,
916 warnings,
917 };
918 }
919 warnings.push('Maker websites exposed product-like links, but none passed product-identity verification.');
920 }
921 }
922
923 const vendorCandidate = await verifyWebsiteCandidates(
924 buildProtectedProductCandidates(input.productName).map((candidate) => ({
925 ...candidate,
926 snippet: input.tagline,
927 })),
928 input.productName,
929 input.tagline,
930 input.description,
931 );
932 if (vendorCandidate) {
933 return {
934 ...vendorCandidate,
935 signals: [...new Set([...vendorCandidate.signals, 'protected_vendor_domain_confirmed'])],
936 source: 'protected_vendor_domain',
937 status: 'verified_external',
938 providerCostUsd,
939 providerAttempts,
940 warnings,
941 };
942 }
943
944 const queries = buildWebsiteSearchQueries(input.productName, input.tagline);
945 if (input.scrappaKey && input.productHuntSlug) {
946 providerAttempts += 1;
947 try {
948 const candidates = await callScrappaProductHuntWebsite(input.productHuntSlug, input.scrappaKey);
949 const verified = await verifyWebsiteCandidates(candidates, input.productName, input.tagline, input.description);
950 if (verified) {
951 return {
952 ...verified,
953 source: 'scrappa_product',
954 status: 'verified_search_match',
955 providerCostUsd,
956 providerAttempts,
957 warnings,
958 };
959 }
960 warnings.push('Product Hunt product record returned no website that passed product-identity verification.');
961 } catch (error) {
962 warnings.push((error instanceof Error ? error.message : 'Product Hunt product lookup failed.').slice(0, 160));
963 }
964 }
965 if (input.scrappaKey && !scrappaUnavailable) {
966 for (const query of queries.slice(0, 2)) {
967 providerAttempts += 1;
968 try {
969 const candidates = await callScrappaWebsiteSearch(query, input.scrappaKey);
970 const verified = await verifyWebsiteCandidates(candidates, input.productName, input.tagline, input.description);
971 if (verified) {
972 return {
973 ...verified,
974 source: 'scrappa_organic',
975 status: 'verified_search_match',
976 providerCostUsd,
977 providerAttempts,
978 warnings,
979 };
980 }
981 warnings.push('Website search returned no website candidate that passed product-identity verification.');
982 } catch (error) {
983 warnings.push((error instanceof Error ? error.message : 'Website search failed.').slice(0, 160));
984 break;
985 }
986 }
987 }
988 if (input.dataForSeoLogin && input.dataForSeoPassword) {
989 let queryIndex = 0;
990 while (queryIndex < queries.length && providerAttempts < 2) {
991 const query = queries[queryIndex];
992 providerAttempts += 1;
993 try {
994 const search = await callDataForSeoWebsiteSearch(query, input.dataForSeoLogin, input.dataForSeoPassword);
995 providerCostUsd += search.costUsd;
996 const verified = await verifyWebsiteCandidates(
997 search.candidates,
998 input.productName,
999 input.tagline,
1000 input.description,
1001 );
1002 if (verified) {
1003 return {
1004 ...verified,
1005 source: 'dataforseo_organic',
1006 status: 'verified_search_match',
1007 providerCostUsd,
1008 providerAttempts,
1009 warnings,
1010 };
1011 }
1012 warnings.push(`Primary website search query ${queryIndex + 1} returned no website candidate that passed product-identity verification.`);
1013 queryIndex += 1;
1014 } catch (error) {
1015
1016
1017 if (error && typeof error === 'object' && 'policy' in error && error.policy === true) throw error;
1018 if (error instanceof DataForSeoSearchError) providerCostUsd += error.costUsd;
1019 const message = error instanceof Error ? error.message : 'Primary website search failed.';
1020 warnings.push(message.slice(0, 200));
1021 if (error instanceof DataForSeoSearchError && error.retryable && providerAttempts < 2) {
1022 continue;
1023 }
1024 queryIndex += 1;
1025 }
1026 }
1027 }
1028
1029 const directDomain = await verifyWebsiteCandidates(
1030 buildDomainHypothesisCandidates(input.productName),
1031 input.productName,
1032 input.tagline,
1033 input.description,
1034 );
1035 if (directDomain) {
1036 return {
1037 ...directDomain,
1038 signals: [...new Set([...directDomain.signals, 'direct_domain_hypothesis_confirmed'])],
1039 source: 'direct_domain_hypothesis',
1040 status: 'verified_external',
1041 providerCostUsd,
1042 providerAttempts,
1043 warnings,
1044 };
1045 }
1046
1047 if (input.serpApiKey) {
1048 providerAttempts += 1;
1049 try {
1050 const query = queries.at(-1) ?? buildWebsiteSearchQueries(input.productName, '')[0];
1051 const candidates = await callSerpApiWebsiteSearch(query, input.serpApiKey);
1052 const verified = await verifyWebsiteCandidates(candidates, input.productName, input.tagline, input.description);
1053 if (verified) {
1054 return {
1055 ...verified,
1056 source: 'serpapi_organic',
1057 status: 'verified_search_match',
1058 providerCostUsd,
1059 providerAttempts,
1060 warnings,
1061 };
1062 }
1063 warnings.push('Backup website search returned no website candidate that passed product-identity verification.');
1064 } catch (error) {
1065 const message = error instanceof Error ? error.message : 'Backup website search failed.';
1066 warnings.push(message.slice(0, 160));
1067 }
1068 }
1069
1070 if (!input.dataForSeoLogin && !input.serpApiKey && !input.scrappaKey) {
1071 warnings.push('No owner-managed website search provider was configured for unresolved Product Hunt links.');
1072 }
1073 return unresolved();
1074}
1075
1076function companyDomainFromWebsite(websiteUrl: string): string {
1077 try {
1078 const hostname = new URL(websiteUrl).hostname.toLowerCase().replace(/^www\./, '');
1079 return getDomain(hostname, { allowPrivateDomains: true }) ?? hostname;
1080 } catch {
1081 return '';
1082 }
1083}
1084
1085async function requestEmailListVerifyContacts(
1086 apiKey: string,
1087 body: { domain: string; firstName?: string; lastName?: string },
1088): Promise<unknown> {
1089 await waitForEmailListVerifySlot();
1090 const response = await fetch(EMAIL_LIST_VERIFY_CONTACT_API, {
1091 method: 'POST',
1092 headers: { 'x-api-key': apiKey, 'Content-Type': 'application/json', Accept: 'application/json' },
1093 body: JSON.stringify(body),
1094 signal: AbortSignal.timeout(30_000),
1095 });
1096 if (!response.ok) {
1097 if (response.status === 402) emailListVerifyCreditsExhausted = true;
1098 throw new Error(`EmailListVerify Contact Finder returned HTTP ${response.status}.`);
1099 }
1100 return response.json();
1101}
1102
1103async function discoverContacts(
1104 websiteUrl: string,
1105 makerNames: string[],
1106 maxEmails: number,
1107 apiKey: string,
1108): Promise<ContactDiscovery> {
1109 const domain = companyDomainFromWebsite(websiteUrl);
1110 if (!domain || maxEmails <= 0) return { contacts: [], attempts: 0, credits: 0, warnings: [] };
1111 if (!apiKey) {
1112 return {
1113 contacts: [], attempts: 0, credits: 0,
1114 warnings: ['Contact discovery was requested but the owner-managed EmailListVerify key is not configured.'],
1115 };
1116 }
1117 if (emailListVerifyCreditsExhausted) {
1118 return {
1119 contacts: [], attempts: 0, credits: 0,
1120 warnings: ['EmailListVerify credits were exhausted earlier in this run; contact discovery was skipped.'],
1121 };
1122 }
1123
1124 const warnings: string[] = [];
1125 let attempts = 0;
1126 let credits = 0;
1127 let contacts: ContactFinderResult[] = [];
1128 const parsedMakers = makerNames.map(splitMakerName).filter(({ firstName, lastName }) => firstName || lastName);
1129 const maker = parsedMakers.find(({ firstName, lastName }) => firstName && lastName) ?? parsedMakers[0];
1130 if (maker) {
1131 attempts += 1;
1132 credits += 5;
1133 try {
1134 const payload = await requestEmailListVerifyContacts(apiKey, { domain, ...maker });
1135 contacts = normalizeContactFinderResults(payload, domain, maxEmails);
1136 if (contacts.length >= maxEmails) return { contacts, attempts, credits, warnings };
1137 } catch (error) {
1138 const message = error instanceof Error ? error.message : 'EmailListVerify named contact discovery failed.';
1139 warnings.push(message.slice(0, 160));
1140 }
1141 }
1142
1143 if (!emailListVerifyCreditsExhausted) {
1144 attempts += 1;
1145 credits += 10;
1146 try {
1147 const payload = await requestEmailListVerifyContacts(apiKey, { domain });
1148 contacts = normalizeContactFinderResults([
1149 ...contacts,
1150 ...(Array.isArray(payload) ? payload : []),
1151 ], domain, maxEmails);
1152 return { contacts, attempts, credits, warnings };
1153 } catch (error) {
1154 const message = error instanceof Error ? error.message : 'EmailListVerify domain contact discovery failed.';
1155 warnings.push(message.slice(0, 160));
1156 }
1157 }
1158 return { contacts, attempts, credits, warnings };
1159}
1160
1161function selectBestEmails(
1162 publicVerifications: EmailVerification[],
1163 contacts: ContactFinderResult[],
1164 maxEmails: number,
1165): EmailVerification[] {
1166 const contactVerifications: EmailVerification[] = contacts.map((contact) => ({
1167 email: contact.email,
1168 provider: 'emaillistverify',
1169 status: contact.result as EmailListVerifyStatus,
1170 checkedAt: new Date().toISOString(),
1171 source: 'emaillistverify_contact_finder',
1172 confidence: contact.confidence,
1173 }));
1174 const statusScore: Record<string, number> = {
1175 ok: 100, ok_for_all: 70, unknown: 55, antispam_system: 50,
1176 not_requested: 48, not_configured: 45, provider_error: 40, error_credit: 38,
1177 smtp_protocol: 30, dead_server: 10, email_disabled: 5, invalid_mx: 0,
1178 invalid_syntax: 0, disposable: 0, spamtrap: 0,
1179 };
1180 const best = new Map<string, EmailVerification>();
1181 for (const verification of [...publicVerifications, ...contactVerifications]) {
1182 const current = best.get(verification.email);
1183 const sourceBonus = verification.source === 'public_website' ? 3 : verification.source === 'maker_website' ? 2 : 0;
1184 const score = (statusScore[verification.status] ?? 20)
1185 + sourceBonus
1186 + (verification.confidence === 'high' ? 2 : verification.confidence === 'medium' ? 1 : 0);
1187 const currentScore = current
1188 ? (statusScore[current.status] ?? 20) + (current.source === 'public_website' ? 3 : current.source === 'maker_website' ? 2 : 0)
1189 : -1;
1190 if (!current || score > currentScore) best.set(verification.email, verification);
1191 }
1192 return [...best.values()]
1193 .filter(({ status }) => isUsableEmailStatus(status))
1194 .sort((left, right) => {
1195 const leftScore = (statusScore[left.status] ?? 20) + (left.source === 'public_website' ? 3 : left.source === 'maker_website' ? 2 : 0);
1196 const rightScore = (statusScore[right.status] ?? 20) + (right.source === 'public_website' ? 3 : right.source === 'maker_website' ? 2 : 0);
1197 return rightScore - leftScore;
1198 })
1199 .slice(0, maxEmails);
1200}
1201
1202async function findEmails(
1203 rawWebsiteUrl: string,
1204 maxWebsitePages: number,
1205 maxEmailsPerProduct: number,
1206): Promise<EnrichmentResult> {
1207 const websiteUrl = canonicalWebsiteUrl(rawWebsiteUrl);
1208 if (!websiteUrl || !isAllowedExternalWebsite(websiteUrl)) {
1209 return {
1210 emails: [], emailSource: 'none', result: 'not_assessed', coverage: 'none',
1211 terminalReason: 'invalid_or_private_target', pagesVisited: 0, pageUrls: [],
1212 warnings: ['Website URL was not a permitted public external HTTP(S) target.'],
1213 };
1214 }
1215
1216 const initialUrl = new URL(websiteUrl);
1217 const maxEmailCandidates = emailCandidateLimit(maxEmailsPerProduct);
1218 const warnings: string[] = [];
1219 const pageUrls: string[] = [];
1220 const addPageUrl = (url: string): void => {
1221 const canonical = canonicalWebsiteUrl(url);
1222 if (canonical && !pageUrls.includes(canonical) && pageUrls.length < maxWebsitePages) pageUrls.push(canonical);
1223 };
1224 let canonicalOrigin = initialUrl.origin;
1225 let baseDomain = initialUrl.hostname.replace(/^www\./i, '');
1226 let emails: string[] = [];
1227 let coverage: Coverage = 'complete';
1228 let pagePlan: WebsitePage[] = [];
1229
1230 const staticHomepage = await fetchPublicHtml(`${initialUrl.origin}/`, 4_000);
1231 if (staticHomepage.ok) {
1232 const finalHomepage = new URL(staticHomepage.url);
1233 canonicalOrigin = finalHomepage.origin;
1234 baseDomain = finalHomepage.hostname.replace(/^www\./i, '');
1235 addPageUrl(staticHomepage.url);
1236 emails = rankEmails(extractStaticEmails(staticHomepage.html), baseDomain, maxEmailCandidates);
1237 pagePlan = buildWebsitePagePlan(`${canonicalOrigin}/`, extractStaticFallbackPages(staticHomepage.html, canonicalOrigin), maxWebsitePages);
1238 for (const target of pagePlan.slice(1)) {
1239 if (emails.length >= maxEmailCandidates) break;
1240 const staticPage = await fetchPublicHtml(target.url, 3_000);
1241 if (!staticPage.ok) {
1242 coverage = 'partial';
1243 warnings.push(`${target.kind} static assessment failed: ${staticPage.reason}.`);
1244 continue;
1245 }
1246 const finalPage = new URL(staticPage.url);
1247 if (finalPage.origin !== canonicalOrigin) {
1248 coverage = 'partial';
1249 warnings.push(`${target.kind} redirected off the verified homepage origin; it was not extracted.`);
1250 continue;
1251 }
1252 addPageUrl(staticPage.url);
1253 emails = rankEmails([...emails, ...extractStaticEmails(staticPage.html)], baseDomain, maxEmailCandidates);
1254 }
1255 } else {
1256 coverage = 'partial';
1257 warnings.push(`Homepage static assessment failed: ${staticHomepage.reason}.`);
1258 }
1259
1260 if (emails.length < maxEmailCandidates) await withBrowserContextSlot(async () => {
1261 const context = await (await ensureBrowser()).newContext({
1262 userAgent: WEBSITE_USER_AGENT,
1263 locale: 'en-US',
1264 viewport: { width: 1280, height: 800 },
1265 extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' },
1266 });
1267 await context.route('**/*', async (route) => {
1268 if (['image', 'media', 'font'].includes(route.request().resourceType())) return route.abort();
1269 const requestUrl = route.request().url();
1270 if (!/^(?:data|blob|about):/i.test(requestUrl) && !await isSafePublicNetworkUrl(requestUrl)) return route.abort();
1271 return route.continue();
1272 });
1273 try {
1274 const page = await context.newPage();
1275 const response = await page.goto(`${canonicalOrigin}/`, { waitUntil: 'domcontentloaded', timeout: 6_000 });
1276 const finalHomepage = new URL(page.url());
1277 if (!response || finalHomepage.origin !== canonicalOrigin) {
1278 coverage = 'partial';
1279 warnings.push('Homepage DOM assessment redirected off origin or did not return a response.');
1280 } else {
1281 addPageUrl(finalHomepage.toString());
1282 await page.locator('body').waitFor({ state: 'attached', timeout: 750 }).catch(() => undefined);
1283 emails = rankEmails([...emails, ...await page.evaluate(browserExtractEmails)], baseDomain, maxEmailCandidates);
1284 const browserFallbackPages = await page.evaluate(browserExtractLinkedFallbackPages, canonicalOrigin);
1285 pagePlan = buildWebsitePagePlan(
1286 `${canonicalOrigin}/`,
1287 [...pagePlan.slice(1), ...browserFallbackPages],
1288 maxWebsitePages,
1289 );
1290 for (const target of pagePlan.slice(1)) {
1291 if (emails.length >= maxEmailCandidates) break;
1292 try {
1293 const fallbackResponse = await page.goto(target.url, { waitUntil: 'domcontentloaded', timeout: 4_500 });
1294 const finalFallback = new URL(page.url());
1295 if (!fallbackResponse || finalFallback.origin !== canonicalOrigin) {
1296 coverage = 'partial';
1297 warnings.push(`${target.kind} DOM assessment redirected off origin.`);
1298 continue;
1299 }
1300 addPageUrl(finalFallback.toString());
1301 await page.locator('body').waitFor({ state: 'attached', timeout: 750 }).catch(() => undefined);
1302 emails = rankEmails([...emails, ...await page.evaluate(browserExtractEmails)], baseDomain, maxEmailCandidates);
1303 } catch (error) {
1304 coverage = 'partial';
1305 const message = error instanceof Error ? error.message : 'navigation error';
1306 warnings.push(`${target.kind} DOM assessment failed: ${message.slice(0, 90)}.`);
1307 }
1308 }
1309 }
1310 } catch (error) {
1311 coverage = 'partial';
1312 const message = error instanceof Error ? error.message : 'browser error';
1313 warnings.push(`Homepage DOM assessment failed: ${message.slice(0, 90)}.`);
1314 } finally {
1315 await context.close();
1316 }
1317 });
1318
1319 const result: EmailResult = emails.length > 0
1320 ? 'emails_found'
1321 : pageUrls.length > 0 && coverage === 'complete'
1322 ? 'no_public_email_found'
1323 : 'not_assessed';
1324 return {
1325 emails,
1326 emailSource: emails.length > 0 ? 'page_scrape' : 'none',
1327 result,
1328 coverage: pageUrls.length === 0 ? 'none' : coverage,
1329 terminalReason: emails.length >= maxEmailCandidates
1330 ? 'max_public_email_candidates_reached'
1331 : result === 'no_public_email_found'
1332 ? 'eligible_pages_exhausted'
1333 : 'eligible_pages_partially_assessed',
1334 pagesVisited: pageUrls.length,
1335 pageUrls,
1336 warnings,
1337 };
1338}
1339
1340async function findMakerHomepageEmails(
1341 makerWebsiteUrls: string[],
1342 productWebsiteEmails: string[],
1343 maxEmailsPerProduct: number,
1344 filledEmailSlots: number,
1345): Promise<{ emails: string[]; pageUrls: string[]; warnings: string[] }> {
1346 const candidates: string[] = [];
1347 const pageUrls: string[] = [];
1348 const warnings: string[] = [];
1349 const seenOrigins = new Set<string>();
1350
1351 for (const rawUrl of makerWebsiteUrls) {
1352 if (selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots).length
1353 >= Math.max(0, maxEmailsPerProduct - filledEmailSlots)) break;
1354 const websiteUrl = canonicalWebsiteUrl(rawUrl);
1355 if (!websiteUrl || !isAllowedExternalWebsite(websiteUrl)) continue;
1356 const origin = new URL(websiteUrl).origin;
1357 if (seenOrigins.has(origin) || seenOrigins.size >= 2) continue;
1358 seenOrigins.add(origin);
1359
1360 const staticPage = await fetchPublicHtml(`${origin}/`, 4_000);
1361 if (staticPage.ok) {
1362 pageUrls.push(staticPage.url);
1363 const emailDomain = new URL(staticPage.url).hostname.replace(/^www\./i, '');
1364 candidates.push(...rankEmails(extractStaticEmails(staticPage.html), emailDomain, maxEmailsPerProduct));
1365 if (selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots).length > 0) continue;
1366 } else {
1367 warnings.push(`Maker homepage static assessment failed: ${staticPage.reason}.`);
1368 }
1369
1370 await withBrowserContextSlot(async () => {
1371 const context = await (await ensureBrowser()).newContext({
1372 userAgent: WEBSITE_USER_AGENT,
1373 locale: 'en-US',
1374 viewport: { width: 1280, height: 800 },
1375 extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' },
1376 });
1377 await context.route('**/*', async (route) => {
1378 if (['image', 'media', 'font'].includes(route.request().resourceType())) return route.abort();
1379 const requestUrl = route.request().url();
1380 if (!/^(?:data|blob|about):/i.test(requestUrl) && !await isSafePublicNetworkUrl(requestUrl)) return route.abort();
1381 return route.continue();
1382 });
1383 try {
1384 const page = await context.newPage();
1385 const response = await page.goto(`${origin}/`, { waitUntil: 'domcontentloaded', timeout: 10_000 });
1386 if (!response || response.status() >= 400 || new URL(page.url()).origin !== origin) {
1387 warnings.push('Maker homepage browser assessment did not reach the approved origin.');
1388 return;
1389 }
1390 if (!pageUrls.includes(page.url())) pageUrls.push(page.url());
1391 const emailDomain = new URL(page.url()).hostname.replace(/^www\./i, '');
1392 candidates.push(...rankEmails(await page.evaluate(browserExtractEmails), emailDomain, maxEmailsPerProduct));
1393 } catch {
1394 warnings.push('Maker homepage browser assessment failed.');
1395 } finally {
1396 await context.close();
1397 }
1398 });
1399 }
1400
1401 return {
1402 emails: selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots),
1403 pageUrls,
1404 warnings,
1405 };
1406}
1407
1408const POST_FIELDS = `
1409 id name tagline description votesCount commentsCount url website slug
1410 featuredAt createdAt dailyRank weeklyRank monthlyRank yearlyRank
1411 reviewsCount reviewsRating
1412 thumbnail { url }
1413 productLinks { type url }
1414 topics(first: 10) { edges { node { name } } }
1415 makers { id name username twitterUsername url websiteUrl }
1416`;
1417
1418const POSTS_QUERY = `query GetPosts($first:Int $after:String $order:PostsOrder $featured:Boolean $postedAfter:DateTime $postedBefore:DateTime $topic:String) {
1419 posts(first:$first after:$after order:$order featured:$featured postedAfter:$postedAfter postedBefore:$postedBefore topic:$topic) {
1420 edges { node { ${POST_FIELDS} } }
1421 pageInfo { endCursor hasNextPage }
1422 }
1423}`;
1424
1425const POSTS_INDEX_QUERY = `query GetPostIndex($first:Int $after:String $order:PostsOrder $featured:Boolean $postedAfter:DateTime $postedBefore:DateTime) {
1426 posts(first:$first after:$after order:$order featured:$featured postedAfter:$postedAfter postedBefore:$postedBefore) {
1427 edges { node { id url slug } }
1428 pageInfo { endCursor hasNextPage }
1429 }
1430}`;
1431
1432const POST_BY_SLUG_QUERY = `query GetPost($slug:String!) { post(slug:$slug) { ${POST_FIELDS} } }`;
1433
1434function toInt(value: unknown): number | null {
1435 if (value === null || value === undefined || value === '') return null;
1436 const parsed = Number.parseInt(String(value), 10);
1437 return Number.isNaN(parsed) ? null : parsed;
1438}
1439
1440function toFloat(value: unknown): number | null {
1441 if (value === null || value === undefined || value === '') return null;
1442 const parsed = Number.parseFloat(String(value));
1443 return Number.isNaN(parsed) ? null : parsed;
1444}
1445
1446function nodeToRecord(node: Record<string, any>, rank: number): Record<string, unknown> {
1447 const makers = (node.makers ?? []).map((maker: Record<string, unknown>) => ({
1448 maker_name: maker.name ?? null,
1449 maker_id: maker.username ?? null,
1450 maker_ph_id: maker.id ?? null,
1451 twitter_url: maker.twitterUsername ? `https://twitter.com/${String(maker.twitterUsername)}` : null,
1452 maker_url: maker.url ?? null,
1453 website_url: maker.websiteUrl ?? null,
1454 }));
1455 const topics = (node.topics?.edges ?? []).map((edge: any) => edge?.node?.name).filter(Boolean);
1456 const productLinks = Array.isArray(node.productLinks) ? node.productLinks : [];
1457 const directWebsite = productLinks.find((link: Record<string, unknown>) => {
1458 const type = String(link.type ?? '').toLowerCase();
1459 return type === 'website' && typeof link.url === 'string' && isAllowedExternalWebsite(link.url);
1460 })?.url as string | undefined;
1461
1462 const websiteUrl = directWebsite ? canonicalWebsiteUrl(directWebsite) : null;
1463 return {
1464 product_name: node.name ? String(node.name).trim() : null,
1465 tagline: node.tagline ?? null,
1466 description: node.description ?? null,
1467 upvote_count: toInt(node.votesCount),
1468 comment_count: toInt(node.commentsCount),
1469 reviews_count: toInt(node.reviewsCount),
1470 reviews_rating: toFloat(node.reviewsRating),
1471 daily_rank: toInt(node.dailyRank) ?? rank,
1472 weekly_rank: toInt(node.weeklyRank),
1473 monthly_rank: toInt(node.monthlyRank),
1474 yearly_rank: toInt(node.yearlyRank),
1475 launch_date: node.featuredAt ? isoDate(new Date(node.featuredAt)) : node.createdAt ? isoDate(new Date(node.createdAt)) : null,
1476 product_hunt_url: node.url ?? (node.slug ? `https://www.producthunt.com/posts/${node.slug}` : null),
1477 product_hunt_website_url: node.website ?? null,
1478 website_url: websiteUrl,
1479 website_resolution_source: websiteUrl ? 'api_website_product_link' satisfies WebsiteResolutionSource : 'none' satisfies WebsiteResolutionSource,
1480 website_resolution_status: websiteUrl
1481 ? 'verified_external' satisfies WebsiteResolutionStatus
1482 : node.website
1483 ? 'unresolved' satisfies WebsiteResolutionStatus
1484 : 'not_available' satisfies WebsiteResolutionStatus,
1485 website_resolution_confidence: websiteUrl ? 100 : 0,
1486 website_resolution_signals: websiteUrl ? ['official_api_product_link'] : [],
1487 website_resolution_provider_cost_usd: 0,
1488 company_domain: websiteUrl ? companyDomainFromWebsite(websiteUrl) : null,
1489 topics,
1490 thumbnail_url: node.thumbnail?.url ?? null,
1491 makers,
1492 featured: Boolean(node.featuredAt),
1493 emails: [] as string[],
1494 verified_emails: [] as string[],
1495 risky_emails: [] as string[],
1496 rejected_emails: [] as string[],
1497 rejected_email_verifications: [] as EmailVerification[],
1498 email_verifications: [] as EmailVerification[],
1499 email_verification_requested: false,
1500 email_source: 'none' as EmailSource,
1501 email_result: 'not_assessed' as EmailResult,
1502 email_coverage: 'none' as Coverage,
1503 email_terminal_reason: 'website_not_available',
1504 email_pages_visited: 0,
1505 email_page_urls: [] as string[],
1506 maker_email_pages_visited: 0,
1507 maker_email_page_urls: [] as string[],
1508 contact_discovery_requested: false,
1509 contact_discovery_attempts: 0,
1510 contact_discovery_credits: 0,
1511 lead_status: 'unresolved_website',
1512 lead_score: 0,
1513 lead_tier: 'unqualified',
1514 lead_signals: [] as string[],
1515 enrichment_warnings: [] as string[],
1516 scraped_at: new Date().toISOString(),
1517 };
1518}
1519
1520const LEAN_FIELDS = new Set([
1521 'product_name', 'tagline', 'website_url', 'company_domain', 'product_hunt_website_url', 'website_resolution_status',
1522 'website_resolution_source', 'website_resolution_confidence', 'upvote_count', 'daily_rank', 'emails', 'verified_emails',
1523 'risky_emails', 'rejected_emails', 'email_verifications', 'rejected_email_verifications', 'email_verification_requested', 'email_source', 'email_result', 'email_coverage',
1524 'email_pages_visited', 'contact_discovery_attempts', 'lead_status', 'lead_score', 'lead_tier', 'lead_signals',
1525 'maker_email_pages_visited',
1526 'topics', 'launch_date', 'product_hunt_url', 'featured', 'scraped_at',
1527]);
1528
1529function applyOutputMode(record: Record<string, unknown>, outputMode: 'full' | 'lean' | 'leads'): Record<string, unknown> | null {
1530 if (outputMode === 'leads' && (!Array.isArray(record.emails) || record.emails.length === 0)) return null;
1531 if (outputMode !== 'lean') return record;
1532 const output: Record<string, unknown> = {};
1533 for (const field of LEAN_FIELDS) output[field] = record[field] ?? null;
1534 return output;
1535}
1536
1537function validateOutputRecord(record: Record<string, unknown>): string | null {
1538 if (typeof record.product_name !== 'string' || !record.product_name.trim()) return 'product_name is missing.';
1539 if (typeof record.product_hunt_url !== 'string' || !record.product_hunt_url.startsWith('https://')) return 'product_hunt_url is missing or invalid.';
1540 if (!Array.isArray(record.emails) || !record.emails.every((email) => typeof email === 'string')) return 'emails must be a string array.';
1541 if (typeof record.scraped_at !== 'string' || Number.isNaN(Date.parse(record.scraped_at))) return 'scraped_at is invalid.';
1542 return null;
1543}
1544
1545async function run(): Promise<void> {
1546 const rawInput = (await Actor.getInput() ?? {}) as Record<string, unknown>;
1547 const mode = (rawInput.mode ?? rawInput.scrapeMode ?? 'leaderboard') as Mode;
1548 const leaderboardPeriod = (rawInput.leaderboardPeriod ?? rawInput.period ?? 'daily') as LeaderboardPeriod;
1549 const startDate = String(rawInput.startDate ?? rawInput.date ?? '');
1550 const endDate = String(rawInput.endDate ?? '');
1551 const lookbackDays = clampInteger(rawInput.lookbackDays, 1, 1, 365);
1552 const outputMode = (rawInput.outputMode ?? 'full') as 'full' | 'lean' | 'leads';
1553 const searchQuery = typeof (rawInput.searchQuery ?? rawInput.query ?? rawInput.q) === 'string'
1554 ? String(rawInput.searchQuery ?? rawInput.query ?? rawInput.q).trim()
1555 : '';
1556 const topic = typeof (rawInput.topic ?? rawInput.topicSlug) === 'string'
1557 ? String(rawInput.topic ?? rawInput.topicSlug).trim()
1558 : '';
1559 const startUrls = Array.isArray(rawInput.startUrls) ? rawInput.startUrls as Array<{ url?: unknown } | string> : [];
1560 const maxResults = clampInteger(rawInput.maxResults ?? rawInput.maxItems ?? rawInput.limit, 100, 1, 20_000);
1561 const maxConcurrency = clampInteger(rawInput.maxConcurrency, 3, 1, 5);
1562 const maxWebsitePages = clampInteger(rawInput.maxWebsitePages, DEFAULT_MAX_WEBSITE_PAGES, 1, MAX_WEBSITE_PAGES);
1563 const maxEmailsPerProduct = clampInteger(rawInput.maxEmailsPerProduct, DEFAULT_MAX_EMAILS_PER_PRODUCT, 1, MAX_EMAILS_PER_PRODUCT);
1564 const includeAllProducts = Boolean(rawInput.includeAllProducts ?? rawInput.allProducts ?? rawInput.includeAll ?? true);
1565 const enrichEmails = Boolean(rawInput.enrichEmails ?? rawInput.findEmails ?? true);
1566 const resolveWebsites = Boolean(rawInput.resolveWebsites ?? true);
1567 const verifyEmails = Boolean(rawInput.verifyEmails ?? rawInput.verifyWithEmailListVerify ?? true);
1568 const findContacts = Boolean(rawInput.findContacts ?? true);
1569 const emailListVerifyApiKey = String(process.env.EMAILLISTVERIFY_API_KEY ?? '').trim();
1570 const phApiToken = String(process.env.PH_API_TOKEN ?? '');
1571 const dataForSeoLogin = String(managedEnvironment(process.env).DATAFORSEO_LOGIN ?? '').trim();
1572 const dataForSeoPassword = String(managedEnvironment(process.env).DATAFORSEO_PASSWORD ?? '').trim();
1573 const serpApiKey = String(process.env.SERPAPI_KEY ?? '').trim();
1574 const scrappaKey = String(process.env.SCRAPPA_API_KEY ?? '').trim();
1575 const websiteOverrides = parseWebsiteOverrides(rawInput.websiteUrlOverrides);
1576 const excludedProductHuntUrls = Array.isArray(rawInput.excludeProductHuntUrls)
1577 ? rawInput.excludeProductHuntUrls
1578 .slice(0, 20_000)
1579 .map((value) => productHuntLaunchKey(String(value ?? '')))
1580 .filter((value): value is string => Boolean(value))
1581 : [];
1582 const excludedProductHuntKeys = new Set(excludedProductHuntUrls);
1583 const productHuntResumeCursor = typeof rawInput.productHuntResumeCursor === 'string'
1584 ? rawInput.productHuntResumeCursor.trim().slice(0, 2_000)
1585 : '';
1586 const startedAt = new Date().toISOString();
1587 emailListVerifyCreditsExhausted = false;
1588
1589 const diagnostics = {
1590 itemsPushed: 0,
1591 itemsProcessed: 0,
1592 itemsFiltered: 0,
1593 itemsExcluded: 0,
1594 itemsFailed: 0,
1595 invalidRecords: 0,
1596 apiFailures: 0,
1597 productHuntIndexPages: 0,
1598 productHuntDetailLookups: 0,
1599 productHuntFullPageLookups: 0,
1600 nextProductHuntCursor: null as string | null,
1601 websiteResolved: 0,
1602 websiteUnresolved: 0,
1603 websiteResolutionProviderAttempts: 0,
1604 websiteResolutionProviderCostUsd: 0,
1605 emailsFound: 0,
1606 verifiedEmailsFound: 0,
1607 emailVerificationAttempts: 0,
1608 emailVerificationProviderErrors: 0,
1609 makerEmailPagesVisited: 0,
1610 makerEmailsFound: 0,
1611 contactDiscoveryAttempts: 0,
1612 contactDiscoveryCredits: 0,
1613 contactDiscoveryEmailsFound: 0,
1614 leadTiers: { A: 0, B: 0, C: 0, unqualified: 0 },
1615 resultEvents: 0,
1616 emailEvents: 0,
1617 chargeLimitReached: false,
1618 resultLimitReached: false,
1619 deadlineReached: false,
1620 itemsSkippedAtDeadline: 0,
1621 searchScanCapReached: false,
1622 emailVerificationCreditsExhausted: false,
1623 warnings: [] as string[],
1624 };
1625 let finalised = false;
1626
1627 const paidItemLimit = clampInteger(process.env.ACTOR_MAX_PAID_DATASET_ITEMS, maxResults, 1, maxResults);
1628
1629
1630 const freeCap = resolveFreePlanCap();
1631 log.info('Run plan limits', { freePlan: freeCap.isFreePlan, freePlanLimit: freeCap.isFreePlan ? freeCap.limit : null });
1632 const effectiveMaxResults = clampToFreePlan(Math.min(maxResults, paidItemLimit), freeCap);
1633 const freeCapBinding = freeCap.isFreePlan && freeCap.limit < Math.min(maxResults, paidItemLimit);
1634 let candidatesForFreePlan = 0;
1635
1636 const runDeadlineMs = Date.parse(process.env.ACTOR_TIMEOUT_AT ?? '');
1637 const hasRunDeadline = Number.isFinite(runDeadlineMs);
1638 const runBudgetMs = hasRunDeadline ? Math.max(0, runDeadlineMs - Date.now()) : Number.POSITIVE_INFINITY;
1639
1640 const newWorkReserveMs = hasRunDeadline ? Math.min(Math.max(runBudgetMs * 0.15, 45_000), 480_000) : 0;
1641 const stopNewWorkAtMs = hasRunDeadline ? runDeadlineMs - newWorkReserveMs : Number.POSITIVE_INFINITY;
1642 const hardStopAtMs = hasRunDeadline ? runDeadlineMs - 20_000 : Number.POSITIVE_INFINITY;
1643 let hardStopping = false;
1644 let inFlightWrites = 0;
1645 let currentPageStartCursor: string | null = null;
1646 let lastProgressAt = 0;
1647
1648 const addWarning = (message: string): void => {
1649 if (diagnostics.warnings.length < 100 && !diagnostics.warnings.includes(message)) diagnostics.warnings.push(message);
1650 };
1651
1652 const deadlineReached = (): boolean => {
1653 if (Date.now() < stopNewWorkAtMs) return false;
1654 if (!diagnostics.deadlineReached) {
1655 diagnostics.deadlineReached = true;
1656 addWarning('The run deadline approached, so no new products were started. Continue with productHuntResumeCursor from RUN_SUMMARY.nextProductHuntCursor (when present) and excludeProductHuntUrls from this dataset to avoid duplicate charges.');
1657 log.warning('Run deadline approaching; stopped starting new products and preserving persisted results.');
1658 }
1659 return true;
1660 };
1661
1662 const reportProgress = async (force = false): Promise<void> => {
1663 if (!force && Date.now() - lastProgressAt < 15_000) return;
1664 lastProgressAt = Date.now();
1665 await Actor.setStatusMessage(
1666 `Processed ${diagnostics.itemsProcessed}/${effectiveMaxResults} products, saved ${diagnostics.itemsPushed} rows, found ${diagnostics.emailsFound} emails.`
1667 + (emailListVerifyCreditsExhausted ? ' Email verification credits are exhausted; remaining emails are unverified.' : ''),
1668 ).catch(() => undefined);
1669 };
1670
1671 const resolveOutcome = (): RunOutcome => {
1672 diagnostics.emailVerificationCreditsExhausted = emailListVerifyCreditsExhausted;
1673 if (diagnostics.itemsPushed === 0) {
1674 const searchedWithoutMatch = mode === 'search' && (diagnostics.searchScanCapReached || diagnostics.deadlineReached);
1675 return diagnostics.apiFailures > 0 || diagnostics.itemsFailed > 0 || diagnostics.invalidRecords > 0 || (diagnostics.deadlineReached && !searchedWithoutMatch)
1676 ? 'UPSTREAM_FAILED'
1677 : 'VALID_EMPTY';
1678 }
1679 return diagnostics.apiFailures > 0 || diagnostics.itemsFailed > 0 || (resolveWebsites && diagnostics.websiteUnresolved > 0) || diagnostics.chargeLimitReached
1680 || diagnostics.resultLimitReached || diagnostics.deadlineReached || diagnostics.searchScanCapReached || (verifyEmails && emailListVerifyCreditsExhausted)
1681 ? 'PARTIAL'
1682 : 'COMPLETE';
1683 };
1684
1685 const outcomeMessage = (outcome: RunOutcome): string => {
1686 const eventCost = diagnostics.resultEvents * RESULT_PRICE_USD + diagnostics.emailEvents * EMAIL_PRICE_USD;
1687 const deadlineNote = diagnostics.deadlineReached ? ' Stopped before the run deadline; resume details are in RUN_SUMMARY.' : '';
1688 if (diagnostics.itemsPushed === 0 && mode === 'search' && (diagnostics.searchScanCapReached || diagnostics.deadlineReached)) {
1689 return diagnostics.searchScanCapReached
1690 ? `${outcome}: no Product Hunt launch matched "${searchQuery}" in the newest ${SEARCH_PAGES_WITHOUT_MATCH_LIMIT * 20} launches searched.`
1691 : `${outcome}: no Product Hunt launch matched "${searchQuery}" before the run deadline.`;
1692 }
1693 if (diagnostics.itemsPushed === 0 && diagnostics.deadlineReached) {
1694 return `${outcome}: the run deadline was reached before any Product Hunt product could be persisted. Increase the run timeout and re-run.`;
1695 }
1696 return `${outcome}: processed ${diagnostics.itemsProcessed} Product Hunt products, persisted ${diagnostics.itemsPushed} rows, and found ${diagnostics.emailsFound} public email(s). Event charges: $${eventCost.toFixed(3)} plus applicable platform usage.${deadlineNote}`;
1697 };
1698
1699 const finalizeRun = async (outcome: RunOutcome, message: string): Promise<never> => {
1700 if (finalised) throw new Error(message);
1701 finalised = true;
1702 const freePlanBase = freePlanLimitSummary(freeCap, maxResults, diagnostics.itemsPushed);
1703 const freePlanLimit = {
1704 ...freePlanBase,
1705 reached: freePlanBase.reached || (freeCapBinding && diagnostics.itemsPushed > 0 && candidatesForFreePlan >= freeCap.limit),
1706 };
1707 if (freePlanLimit.reached && !FAILED_OUTCOMES.has(outcome)) message = `${outcome}: ${freePlanStatus(diagnostics.itemsPushed, freeCap.limit)}`;
1708 if (!message.startsWith(outcome)) message = `${outcome}: ${message}`;
1709 const finishedAt = new Date().toISOString();
1710 const summary = {
1711 outcome,
1712 message,
1713 startedAt,
1714 finishedAt,
1715 buildNumber: process.env.ACTOR_BUILD_NUMBER ?? null,
1716 runId: process.env.ACTOR_RUN_ID ?? null,
1717 datasetId: process.env.ACTOR_DEFAULT_DATASET_ID ?? null,
1718 keyValueStoreId: process.env.ACTOR_DEFAULT_KEY_VALUE_STORE_ID ?? null,
1719 input: {
1720 mode, leaderboardPeriod, startDate: startDate || null, endDate: endDate || null,
1721 lookbackDays, maxResults, effectiveMaxResults, includeAllProducts, enrichEmails, resolveWebsites, verifyEmails, findContacts,
1722 excludedProductHuntUrls: excludedProductHuntKeys.size,
1723 productHuntResumeCursorUsed: Boolean(productHuntResumeCursor),
1724 emailListVerifyConfigured: Boolean(emailListVerifyApiKey), maxWebsitePages,
1725 websiteSearchConfigured: Boolean((dataForSeoLogin && dataForSeoPassword) || serpApiKey || scrappaKey),
1726 maxEmailsPerProduct, outputMode,
1727 },
1728 ...diagnostics,
1729 chargedEventCounts: {
1730 [RESULT_EVENT]: diagnostics.resultEvents,
1731 [EMAIL_EVENT]: diagnostics.emailEvents,
1732 },
1733 freePlanLimit,
1734 };
1735 const output = {
1736 outcome,
1737 message,
1738 itemsPushed: diagnostics.itemsPushed,
1739 itemsFailed: diagnostics.itemsFailed,
1740 invalidRecords: diagnostics.invalidRecords,
1741 deadlineReached: diagnostics.deadlineReached,
1742 nextProductHuntCursor: diagnostics.nextProductHuntCursor,
1743 emailVerificationCreditsExhausted: diagnostics.emailVerificationCreditsExhausted,
1744 chargedEventCounts: summary.chargedEventCounts,
1745 warnings: diagnostics.warnings,
1746 freePlanLimit,
1747 };
1748 await Actor.setValue('OUTPUT', output);
1749 await Actor.setValue('RUN_SUMMARY', summary);
1750 const failed = FAILED_OUTCOMES.has(outcome);
1751 await Actor.setStatusMessage(message, {
1752 isStatusMessageTerminal: true,
1753 level: failed ? 'ERROR' : outcome === 'COMPLETE' ? 'INFO' : 'WARNING',
1754 });
1755 if (failed) {
1756
1757 await Actor.fail(message);
1758 }
1759 await Actor.exit();
1760 throw new Error('Actor terminal call returned unexpectedly.');
1761 };
1762
1763 try {
1764 const modes: Mode[] = ['leaderboard', 'search', 'topic', 'urls'];
1765 const periods: LeaderboardPeriod[] = ['daily', 'weekly', 'monthly', 'yearly'];
1766 const outputModes = ['full', 'lean', 'leads'];
1767 if (!modes.includes(mode)) await finalizeRun('INVALID_INPUT', 'mode must be leaderboard, search, topic, or urls.');
1768 if (!periods.includes(leaderboardPeriod)) await finalizeRun('INVALID_INPUT', 'leaderboardPeriod must be daily, weekly, monthly, or yearly.');
1769 if (!outputModes.includes(outputMode)) await finalizeRun('INVALID_INPUT', 'outputMode must be full, lean, or leads.');
1770 if (!phApiToken) await finalizeRun('CONFIG_ERROR', 'Product Hunt API token is not configured. Set the owner-managed PH_API_TOKEN secret and re-run.');
1771 if (mode === 'search' && !searchQuery) await finalizeRun('INVALID_INPUT', 'searchQuery is required for search mode.');
1772 if (mode === 'topic' && !topic) await finalizeRun('INVALID_INPUT', 'topic is required for topic mode.');
1773 if (mode === 'urls' && startUrls.length === 0) await finalizeRun('INVALID_INPUT', 'startUrls is required for urls mode.');
1774 if (startDate && !validIsoDate(startDate)) await finalizeRun('INVALID_INPUT', 'startDate must be a valid YYYY-MM-DD date.');
1775 if (endDate && !validIsoDate(endDate)) await finalizeRun('INVALID_INPUT', 'endDate must be a valid YYYY-MM-DD date.');
1776 if (startDate && endDate && startDate > endDate) await finalizeRun('INVALID_INPUT', 'endDate must be on or after startDate.');
1777 if (leaderboardPeriod !== 'daily' && endDate) await finalizeRun('INVALID_INPUT', 'endDate is supported only with daily leaderboard ranges.');
1778
1779 const maxPotentialEventCost = effectiveMaxResults * (RESULT_PRICE_USD + (enrichEmails ? maxEmailsPerProduct * EMAIL_PRICE_USD : 0));
1780 const maxPotentialVerificationCredits = enrichEmails && emailListVerifyApiKey
1781 ? effectiveMaxResults * ((verifyEmails ? maxEmailsPerProduct * 3 * 2 : 0) + (findContacts ? 15 : 0))
1782 : 0;
1783 log.info('ProductHunt Scraper starting', {
1784 mode, leaderboardPeriod, startDate: startDate || '(default)', endDate: endDate || null,
1785 lookbackDays, effectiveMaxResults, includeAllProducts, enrichEmails, resolveWebsites, verifyEmails, findContacts,
1786 emailListVerifyConfigured: Boolean(emailListVerifyApiKey), maxWebsitePages, maxEmailsPerProduct,
1787 websiteSearchProviders: {
1788 dataForSeo: Boolean(dataForSeoLogin && dataForSeoPassword),
1789 scrappa: Boolean(scrappaKey),
1790 serpApiFallback: Boolean(serpApiKey),
1791 },
1792 eventCostCeilingUsd: Number(maxPotentialEventCost.toFixed(3)),
1793 excludedProductHuntUrls: excludedProductHuntKeys.size,
1794 emailListVerifyCreditCeiling: maxPotentialVerificationCredits,
1795 websiteSearchQueryCeiling: resolveWebsites
1796 ? effectiveMaxResults * ((dataForSeoLogin && dataForSeoPassword ? 2 : 0) + (serpApiKey ? 1 : 0) + (scrappaKey ? 3 : 0))
1797 : 0,
1798 platformUsage: 'Additional compute and proxy usage is billed by Apify separately where applicable.',
1799 });
1800 await Actor.setStatusMessage(`Starting: up to ${effectiveMaxResults} products, ${maxWebsitePages} public website pages/product, and ${maxEmailsPerProduct} public emails/product.`);
1801 lastProgressAt = Date.now();
1802
1803 if (hasRunDeadline) {
1804 const hardStopTimer = setInterval(() => {
1805 if (Date.now() < hardStopAtMs || hardStopping || finalised) return;
1806 hardStopping = true;
1807 clearInterval(hardStopTimer);
1808 void (async () => {
1809 deadlineReached();
1810 const waitUntil = Date.now() + 10_000;
1811 while (inFlightWrites > 0 && Date.now() < waitUntil) await new Promise((resolve) => setTimeout(resolve, 200));
1812 if (currentPageStartCursor && !diagnostics.nextProductHuntCursor) diagnostics.nextProductHuntCursor = currentPageStartCursor;
1813 await closeBrowser();
1814 const outcome = resolveOutcome();
1815 await finalizeRun(outcome, outcomeMessage(outcome)).catch(() => undefined);
1816 })();
1817 }, 5_000);
1818 hardStopTimer.unref();
1819 }
1820
1821 const phQuery = async (query: string, variables: Record<string, unknown>, attempt = 0): Promise<any> => {
1822 const response = await fetch(PH_API, {
1823 method: 'POST',
1824 headers: {
1825 'Content-Type': 'application/json',
1826 Accept: 'application/json',
1827 Authorization: `Bearer ${phApiToken}`,
1828 'User-Agent': PRODUCT_HUNT_USER_AGENT,
1829 },
1830 body: JSON.stringify({ query, variables }),
1831 signal: AbortSignal.timeout(30_000),
1832 });
1833 if (response.status === 429 && attempt < 3) {
1834 const waitSeconds = productHuntRateLimitDelaySeconds(
1835 response.headers.get('x-rate-limit-reset'),
1836 response.headers.get('retry-after'),
1837 );
1838 log.warning(`Product Hunt API rate limited; waiting ${waitSeconds}s before retry ${attempt + 1}/3.`);
1839 await new Promise((resolve) => setTimeout(resolve, waitSeconds * 1000));
1840 return phQuery(query, variables, attempt + 1);
1841 }
1842 if (!response.ok) throw new Error(`Product Hunt API returned HTTP ${response.status}.`);
1843 const payload = await response.json() as { data?: unknown; errors?: Array<{ message?: string }> };
1844 if (payload.errors?.length) throw new Error(`Product Hunt GraphQL: ${payload.errors[0]?.message ?? 'unknown error'}`);
1845 return payload.data as any;
1846 };
1847
1848 const seenIds = new Set<string>();
1849 let collectedCandidates = 0;
1850
1851 const pushNode = async (node: Record<string, any>, rank: number): Promise<void> => {
1852 const record = nodeToRecord(node, rank);
1853 record.email_verification_requested = verifyEmails;
1854 const override = lookupWebsiteOverride(node, websiteOverrides);
1855 if (override) {
1856 record.website_url = override;
1857 record.website_resolution_source = 'input_override' satisfies WebsiteResolutionSource;
1858 record.website_resolution_status = 'verified_external' satisfies WebsiteResolutionStatus;
1859 record.website_resolution_confidence = 100;
1860 record.website_resolution_signals = ['approved_input_override'];
1861 }
1862
1863 let websiteUrl = typeof record.website_url === 'string' ? record.website_url : '';
1864 if (!websiteUrl && resolveWebsites) {
1865 const resolution = await resolveWebsite({
1866 productName: String(record.product_name ?? ''),
1867 tagline: String(record.tagline ?? ''),
1868 description: String(record.description ?? ''),
1869 productHuntRedirect: typeof record.product_hunt_website_url === 'string' ? record.product_hunt_website_url : '',
1870 makerWebsiteUrls: (node.makers ?? [])
1871 .map((maker: Record<string, unknown>) => typeof maker.websiteUrl === 'string' ? maker.websiteUrl : '')
1872 .filter(Boolean),
1873 dataForSeoLogin,
1874 dataForSeoPassword,
1875 serpApiKey,
1876 scrappaKey,
1877 productHuntSlug: typeof node.slug === 'string' ? node.slug : '',
1878 });
1879 diagnostics.websiteResolutionProviderAttempts += resolution.providerAttempts;
1880 diagnostics.websiteResolutionProviderCostUsd += resolution.providerCostUsd;
1881 resolution.warnings.forEach(addWarning);
1882 record.website_resolution_provider_cost_usd = Number(resolution.providerCostUsd.toFixed(6));
1883 if (resolution.websiteUrl) {
1884 websiteUrl = resolution.websiteUrl;
1885 record.website_url = resolution.websiteUrl;
1886 record.website_resolution_source = resolution.source;
1887 record.website_resolution_status = resolution.status;
1888 record.website_resolution_confidence = resolution.confidence;
1889 record.website_resolution_signals = resolution.signals;
1890 }
1891 }
1892
1893 if (websiteUrl) {
1894 diagnostics.websiteResolved += 1;
1895 record.company_domain = companyDomainFromWebsite(websiteUrl);
1896 } else {
1897 diagnostics.websiteUnresolved += 1;
1898 record.website_resolution_status = record.product_hunt_website_url ? 'unresolved' : 'not_available';
1899 }
1900
1901 let publicEmailCount = 0;
1902 let publicVerifications: EmailVerification[] = [];
1903 if (enrichEmails && websiteUrl) {
1904 const enriched = await findEmails(websiteUrl, maxWebsitePages, maxEmailsPerProduct);
1905 record.emails = enriched.emails;
1906 record.email_source = enriched.emailSource;
1907 record.email_result = enriched.result;
1908 record.email_coverage = enriched.coverage;
1909 record.email_terminal_reason = enriched.terminalReason;
1910 record.email_pages_visited = enriched.pagesVisited;
1911 record.email_page_urls = enriched.pageUrls;
1912 record.enrichment_warnings = enriched.warnings;
1913 publicEmailCount = enriched.emails.length;
1914 enriched.warnings.forEach(addWarning);
1915 log.info('Website enrichment complete', {
1916 product: record.product_name,
1917 emailsFound: enriched.emails.length,
1918 source: enriched.emailSource,
1919 pagesVisited: enriched.pagesVisited,
1920 coverage: enriched.coverage,
1921 terminalReason: enriched.terminalReason,
1922 });
1923 }
1924
1925 const selectedEmails = Array.isArray(record.emails) ? record.emails.filter((email): email is string => typeof email === 'string') : [];
1926 if (selectedEmails.length > 0) {
1927 const verification = await verifyEmailsWithEmailListVerify(
1928 selectedEmails,
1929 verifyEmails,
1930 emailListVerifyApiKey,
1931 'public_website',
1932 maxEmailsPerProduct,
1933 );
1934 publicVerifications = verification.verifications;
1935 diagnostics.emailVerificationAttempts += verification.attempts;
1936 diagnostics.emailVerificationProviderErrors += verification.verifications.filter(({ status }) => status === 'provider_error' || status === 'error_credit').length;
1937 verification.warnings.forEach(addWarning);
1938 log.info('Public email verification complete', {
1939 product: record.product_name,
1940 emailsAssessed: selectedEmails.length,
1941 requested: verifyEmails,
1942 attempts: verification.attempts,
1943 statuses: verification.verifications.map(({ status }) => status),
1944 });
1945 }
1946
1947 const makerWebsiteUrls = (node.makers ?? [])
1948 .map((maker: Record<string, unknown>) => typeof maker.websiteUrl === 'string' ? maker.websiteUrl : '')
1949 .filter(Boolean);
1950 const productDeliverableCount = verifyEmails
1951 ? publicVerifications.filter(({ status }) => status === 'ok').length
1952 : publicVerifications.filter(({ status }) => isUsableEmailStatus(status)).length;
1953 if (enrichEmails && productDeliverableCount < maxEmailsPerProduct && makerWebsiteUrls.length > 0) {
1954 const makerEnrichment = await findMakerHomepageEmails(
1955 makerWebsiteUrls,
1956 selectedEmails,
1957 maxEmailsPerProduct,
1958 productDeliverableCount,
1959 );
1960 record.maker_email_pages_visited = makerEnrichment.pageUrls.length;
1961 record.maker_email_page_urls = makerEnrichment.pageUrls;
1962 diagnostics.makerEmailPagesVisited += makerEnrichment.pageUrls.length;
1963 makerEnrichment.warnings.forEach(addWarning);
1964 record.enrichment_warnings = [
1965 ...new Set([
1966 ...(Array.isArray(record.enrichment_warnings) ? record.enrichment_warnings as string[] : []),
1967 ...makerEnrichment.warnings,
1968 ]),
1969 ];
1970 if (makerEnrichment.emails.length > 0) {
1971 const makerVerification = await verifyEmailsWithEmailListVerify(
1972 makerEnrichment.emails,
1973 verifyEmails,
1974 emailListVerifyApiKey,
1975 'maker_website',
1976 );
1977 publicVerifications.push(...makerVerification.verifications);
1978 publicEmailCount += makerEnrichment.emails.length;
1979 diagnostics.makerEmailsFound += makerEnrichment.emails.length;
1980 diagnostics.emailVerificationAttempts += makerVerification.attempts;
1981 diagnostics.emailVerificationProviderErrors += makerVerification.verifications
1982 .filter(({ status }) => status === 'provider_error' || status === 'error_credit').length;
1983 makerVerification.warnings.forEach(addWarning);
1984 log.info('Maker homepage email fallback complete', {
1985 product: record.product_name,
1986 emailsFound: makerEnrichment.emails.length,
1987 pagesVisited: makerEnrichment.pageUrls.length,
1988 statuses: makerVerification.verifications.map(({ status }) => status),
1989 });
1990 }
1991 }
1992
1993 let discoveredContacts: ContactFinderResult[] = [];
1994 const deliverablePublicCount = verifyEmails
1995 ? publicVerifications.filter(({ status }) => status === 'ok').length
1996 : publicVerifications.filter(({ status }) => isUsableEmailStatus(status)).length;
1997 if (enrichEmails && findContacts && websiteUrl && deliverablePublicCount < maxEmailsPerProduct) {
1998 record.contact_discovery_requested = true;
1999 const discovery = await discoverContacts(
2000 websiteUrl,
2001 (node.makers ?? []).map((maker: Record<string, unknown>) => String(maker.name ?? '')).filter(Boolean),
2002 Math.max(1, maxEmailsPerProduct - deliverablePublicCount),
2003 emailListVerifyApiKey,
2004 );
2005 discoveredContacts = discovery.contacts;
2006 record.contact_discovery_attempts = discovery.attempts;
2007 record.contact_discovery_credits = discovery.credits;
2008 diagnostics.contactDiscoveryAttempts += discovery.attempts;
2009 diagnostics.contactDiscoveryCredits += discovery.credits;
2010 diagnostics.contactDiscoveryEmailsFound += discovery.contacts.length;
2011 discovery.warnings.forEach(addWarning);
2012 }
2013
2014 const finalVerifications = selectBestEmails(publicVerifications, discoveredContacts, maxEmailsPerProduct);
2015 const rejectedVerifications = publicVerifications.filter(({ status }) => !isUsableEmailStatus(status));
2016 const finalEmails = finalVerifications.map(({ email }) => email);
2017 const publicSelected = finalVerifications.filter(({ source }) => source === 'public_website').length;
2018 const makerSelected = finalVerifications.filter(({ source }) => source === 'maker_website').length;
2019 const finderSelected = finalVerifications.filter(({ source }) => source === 'emaillistverify_contact_finder').length;
2020 record.emails = finalEmails;
2021 record.verified_emails = finalVerifications.filter(({ status }) => status === 'ok').map(({ email }) => email);
2022 record.risky_emails = finalVerifications.filter(({ status }) => status !== 'ok').map(({ email }) => email);
2023 record.rejected_emails = rejectedVerifications.map(({ email }) => email);
2024 record.email_verifications = finalVerifications;
2025 record.rejected_email_verifications = rejectedVerifications;
2026 const selectedSourceCount = [publicSelected, makerSelected, finderSelected].filter((count) => count > 0).length;
2027 record.email_source = selectedSourceCount > 1
2028 ? 'mixed'
2029 : publicSelected > 0
2030 ? 'page_scrape'
2031 : makerSelected > 0
2032 ? 'maker_website'
2033 : finderSelected > 0
2034 ? 'emaillistverify_contact_finder'
2035 : 'none';
2036 record.email_result = finalEmails.length > 0 ? 'emails_found' : record.email_result;
2037 if (finderSelected > 0) record.email_terminal_reason = 'contact_finder_enriched';
2038 else if (makerSelected > 0) record.email_terminal_reason = 'maker_homepage_enriched';
2039 diagnostics.emailsFound += finalEmails.length;
2040 diagnostics.verifiedEmailsFound += (record.verified_emails as string[]).length;
2041
2042 const lead = buildLeadProfile({
2043 websiteResolved: Boolean(websiteUrl),
2044 websiteConfidence: Number(record.website_resolution_confidence ?? 0),
2045 emailStatuses: finalVerifications.map(({ status }) => status),
2046 publicEmailCount,
2047 upvotes: Number(record.upvote_count ?? 0),
2048 pagesVisited: Number(record.email_pages_visited ?? 0),
2049 });
2050 record.lead_status = lead.leadStatus;
2051 record.lead_score = lead.leadScore;
2052 record.lead_tier = lead.leadTier;
2053 record.lead_signals = lead.signals;
2054 diagnostics.leadTiers[lead.leadTier] += 1;
2055
2056 log.info('Lead enrichment complete', {
2057 product: record.product_name,
2058 websiteResolutionSource: record.website_resolution_source,
2059 websiteResolutionConfidence: record.website_resolution_confidence,
2060 publicEmailsFound: publicEmailCount,
2061 contactFinderEmailsFound: discoveredContacts.length,
2062 finalEmails: finalEmails.length,
2063 verifiedEmails: (record.verified_emails as string[]).length,
2064 leadTier: lead.leadTier,
2065 });
2066
2067 const outputRecord = applyOutputMode(record, outputMode);
2068 if (!outputRecord) {
2069 diagnostics.itemsFiltered += 1;
2070 return;
2071 }
2072 const validationError = validateOutputRecord(outputRecord);
2073 if (validationError) {
2074 diagnostics.invalidRecords += 1;
2075 diagnostics.itemsFailed += 1;
2076 addWarning(`A Product Hunt record was not persisted: ${validationError}`);
2077 return;
2078 }
2079
2080 if (hardStopping) {
2081 diagnostics.itemsSkippedAtDeadline += 1;
2082 return;
2083 }
2084
2085 if (freePlanLimitHit(diagnostics.itemsPushed + inFlightWrites, freeCap)) return;
2086 inFlightWrites += 1;
2087 try {
2088
2089
2090
2091 const primaryResult = await Actor.pushData(outputRecord) as unknown as { chargedCount?: number; eventChargeLimitReached?: boolean };
2092 if (primaryResult.eventChargeLimitReached && (primaryResult.chargedCount ?? 0) === 0) {
2093 diagnostics.chargeLimitReached = true;
2094 addWarning('The event-charge limit was reached before this validated row could be persisted.');
2095 return;
2096 }
2097 diagnostics.itemsPushed += 1;
2098 diagnostics.resultEvents += primaryResult.chargedCount ?? 1;
2099 diagnostics.chargeLimitReached ||= Boolean(primaryResult.eventChargeLimitReached);
2100
2101 const emails = outputRecord.emails as string[];
2102 if (emails.length > 0 && !diagnostics.chargeLimitReached) {
2103 const emailResult = await Actor.charge({ eventName: EMAIL_EVENT, count: emails.length }) as unknown as { chargedCount?: number; eventChargeLimitReached?: boolean };
2104 diagnostics.emailEvents += emailResult.chargedCount ?? emails.length;
2105 diagnostics.chargeLimitReached ||= Boolean(emailResult.eventChargeLimitReached);
2106 if (emailResult.eventChargeLimitReached) addWarning('The event-charge limit was reached after a persisted record; remaining work stopped at the configured cap.');
2107 }
2108 } catch (error) {
2109 diagnostics.itemsFailed += 1;
2110 const message = error instanceof Error ? error.message : 'Unknown dataset-write failure.';
2111 addWarning(`Validated dataset write failed: ${message.slice(0, 120)}`);
2112 log.warning(`Validated dataset write failed for ${String(record.product_name)}: ${message.slice(0, 120)}`);
2113 } finally {
2114 inFlightWrites -= 1;
2115 }
2116 };
2117
2118 const processNodes = async (nodes: Array<Record<string, any>>): Promise<void> => {
2119 const uniqueNodes: Array<Record<string, any>> = [];
2120 for (const node of nodes) {
2121 if (collectedCandidates >= effectiveMaxResults || diagnostics.chargeLimitReached) break;
2122 const id = String(node.id ?? '');
2123 if (!id || seenIds.has(id)) continue;
2124 seenIds.add(id);
2125 const launchKey = productHuntLaunchKey(String(node.url ?? ''));
2126 if (launchKey && excludedProductHuntKeys.has(launchKey)) {
2127 diagnostics.itemsExcluded += 1;
2128 continue;
2129 }
2130 uniqueNodes.push(node);
2131 collectedCandidates += 1;
2132 candidatesForFreePlan = collectedCandidates;
2133 }
2134 const rankBase = diagnostics.itemsProcessed;
2135 let nextIndex = 0;
2136 const worker = async (): Promise<void> => {
2137 while (nextIndex < uniqueNodes.length && !diagnostics.chargeLimitReached && !hardStopping
2138 && !freePlanLimitHit(diagnostics.itemsPushed, freeCap)) {
2139 if (deadlineReached()) break;
2140 const index = nextIndex;
2141 nextIndex += 1;
2142 try {
2143 await pushNode(uniqueNodes[index], rankBase + index + 1);
2144 } catch (error) {
2145 diagnostics.itemsFailed += 1;
2146 const message = error instanceof Error ? error.message : 'Unknown product enrichment failure.';
2147 addWarning(`A Product Hunt product could not be enriched: ${message.slice(0, 120)}`);
2148 log.warning(`Product enrichment failed for ${String(uniqueNodes[index]?.name ?? 'unknown')}: ${message.slice(0, 120)}`);
2149 }
2150 diagnostics.itemsProcessed += 1;
2151 await reportProgress();
2152 }
2153 };
2154 await Promise.all(Array.from({ length: Math.min(maxConcurrency, uniqueNodes.length) }, () => worker()));
2155 };
2156
2157 const streamPosts = async (range: DateRange | null, label: string): Promise<void> => {
2158 let cursor: string | null = productHuntResumeCursor || null;
2159 let pageNumber = 1;
2160 let pagesWithoutSearchMatch = 0;
2161 let useLightweightIndex = shouldUseLightweightProductHuntIndex(
2162 mode,
2163 excludedProductHuntKeys.size,
2164 productHuntResumeCursor,
2165 );
2166 while (collectedCandidates < effectiveMaxResults && !diagnostics.chargeLimitReached) {
2167 if (deadlineReached() || hardStopping) break;
2168 const pageStartCursor = cursor;
2169 currentPageStartCursor = pageStartCursor;
2170 const failuresBeforePage = diagnostics.itemsFailed;
2171 const remaining = effectiveMaxResults - collectedCandidates;
2172 const variables: Record<string, unknown> = {
2173 first: Math.min(20, remaining),
2174 after: cursor,
2175 order: productHuntStreamOrder(Boolean(range)),
2176 };
2177 if (!includeAllProducts) variables.featured = true;
2178 if (range) {
2179 variables.postedAfter = range.postedAfter;
2180 variables.postedBefore = range.postedBefore;
2181 }
2182 if (mode === 'topic') variables.topic = topic;
2183 try {
2184 let data: any = await phQuery(useLightweightIndex ? POSTS_INDEX_QUERY : POSTS_QUERY, variables);
2185 let edges = data?.posts?.edges ?? [];
2186 let pageInfo: { hasNextPage?: boolean; endCursor?: string | null } = data?.posts?.pageInfo ?? {};
2187 if (!edges.length) break;
2188 log.info(`Product Hunt ${label} page ${pageNumber}`, { received: edges.length, processed: diagnostics.itemsProcessed });
2189 let nodes = edges.map((edge: { node?: Record<string, any> }) => edge.node).filter(Boolean);
2190 if (useLightweightIndex) {
2191 diagnostics.productHuntIndexPages += 1;
2192 const hasUnseenLaunch = hasUnexcludedProductHuntLaunch(
2193 nodes.map((node: Record<string, any>) => String(node.url ?? '')),
2194 excludedProductHuntKeys,
2195 );
2196 if (hasUnseenLaunch) {
2197 data = await phQuery(POSTS_QUERY, variables);
2198 diagnostics.productHuntFullPageLookups += 1;
2199 edges = data?.posts?.edges ?? [];
2200 pageInfo = data?.posts?.pageInfo ?? pageInfo;
2201 nodes = edges.map((edge: { node?: Record<string, any> }) => edge.node).filter(Boolean);
2202 useLightweightIndex = false;
2203 } else {
2204 for (const node of nodes) {
2205 const id = String(node.id ?? '');
2206 if (!id || seenIds.has(id)) continue;
2207 const launchKey = productHuntLaunchKey(String(node.url ?? ''));
2208 if (launchKey && excludedProductHuntKeys.has(launchKey)) {
2209 seenIds.add(id);
2210 diagnostics.itemsExcluded += 1;
2211 }
2212 }
2213 nodes = [];
2214 }
2215 }
2216 const selectedNodes = mode === 'search'
2217 ? nodes.filter((node: Record<string, any>) => matchesProductSearch(searchQuery, {
2218 name: node.name,
2219 tagline: node.tagline,
2220 description: node.description,
2221 topics: (node.topics?.edges ?? []).map((edge: any) => edge?.node?.name).filter(Boolean),
2222 }))
2223 : nodes;
2224 if (mode === 'search') {
2225 pagesWithoutSearchMatch = selectedNodes.length > 0 ? 0 : pagesWithoutSearchMatch + 1;
2226 if (pagesWithoutSearchMatch >= SEARCH_PAGES_WITHOUT_MATCH_LIMIT && !diagnostics.searchScanCapReached) {
2227 diagnostics.searchScanCapReached = true;
2228 addWarning(`Search stopped after ${SEARCH_PAGES_WITHOUT_MATCH_LIMIT} consecutive Product Hunt pages without a match. Continue with productHuntResumeCursor from RUN_SUMMARY.nextProductHuntCursor to search older launches.`);
2229 }
2230 }
2231 await processNodes(selectedNodes);
2232 const pageInterrupted = diagnostics.deadlineReached || hardStopping;
2233 diagnostics.nextProductHuntCursor = pageInterrupted
2234 ? pageStartCursor
2235 : pageInfo.hasNextPage && pageInfo.endCursor
2236 ? pageInfo.endCursor
2237 : null;
2238 const checkpointCursor = resumableCheckpointCursor(
2239 pageStartCursor,
2240 Boolean(pageInfo.hasNextPage),
2241 pageInfo.endCursor,
2242 failuresBeforePage,
2243 diagnostics.itemsFailed,
2244 );
2245 if (!useLightweightIndex || pageNumber === 1 || pageNumber % 10 === 0 || !pageInfo.hasNextPage) {
2246 try {
2247 await Actor.setValue('RUN_CHECKPOINT', {
2248 status: 'RUNNING',
2249 updatedAt: new Date().toISOString(),
2250 runId: process.env.ACTOR_RUN_ID ?? null,
2251 buildNumber: process.env.ACTOR_BUILD_NUMBER ?? null,
2252 mode,
2253 label,
2254 range,
2255 productHuntResumeCursor: checkpointCursor,
2256 pageNumber,
2257 itemsProcessed: diagnostics.itemsProcessed,
2258 itemsPushed: diagnostics.itemsPushed,
2259 itemsExcluded: diagnostics.itemsExcluded,
2260 itemsFailed: diagnostics.itemsFailed,
2261 });
2262 } catch (error) {
2263 const message = error instanceof Error ? error.message : 'unknown checkpoint failure';
2264 addWarning(`Run checkpoint could not be persisted: ${message.slice(0, 120)}`);
2265 }
2266 }
2267 if (collectedCandidates >= effectiveMaxResults && pageInfo.hasNextPage) {
2268 diagnostics.resultLimitReached = true;
2269 addWarning('The result limit was reached while Product Hunt still had another page. Continue with excludeProductHuntUrls from this dataset to avoid duplicate enrichment charges.');
2270 }
2271 await reportProgress(true);
2272 if (pageInterrupted || diagnostics.searchScanCapReached) break;
2273 if (!pageInfo.hasNextPage || !pageInfo.endCursor) break;
2274 cursor = pageInfo.endCursor;
2275 pageNumber += 1;
2276 } catch (error) {
2277 diagnostics.apiFailures += 1;
2278 const message = error instanceof Error ? error.message : 'Unknown Product Hunt API error.';
2279 addWarning(`Product Hunt ${label} query failed: ${message}`);
2280 break;
2281 }
2282 }
2283 };
2284
2285 if (mode === 'urls') {
2286 const rawSeeds = startUrls
2287 .map((entry) => typeof entry === 'string' ? entry : typeof entry?.url === 'string' ? entry.url : '')
2288 .filter(Boolean);
2289 const seeds = rawSeeds.map(parseProductHuntSeedUrl);
2290 const validSeeds = seeds.filter((seed) => seed.kind !== 'invalid');
2291 seeds.filter((seed) => seed.kind === 'invalid').forEach((seed) => addWarning(`Ignored start URL: ${seed.reason}`));
2292 if (validSeeds.length === 0) await finalizeRun('INVALID_INPUT', 'No valid Product Hunt post, product launch, or dated leaderboard URLs were supplied.');
2293
2294 for (const seed of validSeeds) {
2295 if (collectedCandidates >= effectiveMaxResults || diagnostics.chargeLimitReached) break;
2296 if (deadlineReached() || hardStopping) break;
2297 if (seed.kind === 'post') {
2298 try {
2299 const data = await phQuery(POST_BY_SLUG_QUERY, { slug: seed.slug });
2300 if (!data?.post) {
2301 addWarning(`No Product Hunt post matched the supplied URL slug: ${seed.slug}.`);
2302 continue;
2303 }
2304 await processNodes([data.post]);
2305 } catch (error) {
2306 diagnostics.apiFailures += 1;
2307 const message = error instanceof Error ? error.message : 'Unknown Product Hunt API error.';
2308 addWarning(`Product Hunt URL lookup failed for ${seed.slug}: ${message}`);
2309 }
2310 } else if (seed.kind === 'leaderboard') {
2311 await streamPosts(dateRangeForPeriod(seed.period, seed.anchorDate), `${seed.period} leaderboard URL`);
2312 }
2313 }
2314 } else {
2315 let range: DateRange | null = null;
2316 if (mode === 'leaderboard') {
2317 if (startDate) {
2318 range = dateRangeForPeriod(leaderboardPeriod, startDate, endDate);
2319 } else if (lookbackDays > 1) {
2320 const end = new Date();
2321 const begin = new Date(end);
2322 begin.setUTCDate(end.getUTCDate() - lookbackDays + 1);
2323 range = { postedAfter: `${isoDate(begin)}T00:00:00+00:00`, postedBefore: `${isoDate(end)}T23:59:59+00:00` };
2324 } else {
2325 range = dateRangeForPeriod(leaderboardPeriod, isoDate(new Date()));
2326 }
2327 }
2328 await streamPosts(range, mode);
2329 }
2330
2331 if (hardStopping) return;
2332 await closeBrowser();
2333 const outcome = resolveOutcome();
2334 await finalizeRun(outcome, outcomeMessage(outcome));
2335 } catch (error) {
2336 if (finalised || hardStopping) return;
2337 await closeBrowser();
2338 const message = error instanceof Error ? error.message : 'Unknown actor failure.';
2339 diagnostics.itemsFailed += 1;
2340 addWarning(`Unhandled actor error: ${message.slice(0, 160)}`);
2341 await finalizeRun('UPSTREAM_FAILED', `The run stopped before completion: ${message.slice(0, 160)}`);
2342 }
2343}
2344
2345await run();