1import { Actor } from 'apify';
2import { log } from 'crawlee';
3import { load } from 'cheerio';
4import { lookup } from 'node:dns/promises';
5import { isIP } from 'node:net';
6import { chromium, type Browser } from 'playwright';
7import { getDomain } from 'tldts';
8import {
9 buildWebsitePagePlan,
10 canonicalWebsiteUrl,
11 clampInteger,
12 decodeCloudflareEmail,
13 DEFAULT_MAX_EMAILS_PER_PRODUCT,
14 DEFAULT_MAX_WEBSITE_PAGES,
15 emailCandidateLimit,
16 extractEmailCandidatesFromText,
17 hasUnexcludedProductHuntLaunch,
18 hasReachedUsableEmailTarget,
19 isRetryableEmailListVerifyHttpStatus,
20 isProductHuntHost,
21 isPublicHttpUrl,
22 MAX_EMAILS_PER_PRODUCT,
23 MAX_WEBSITE_PAGES,
24 normalizeEmailListVerifyStatus,
25 parseProductHuntSeedUrl,
26 productHuntLaunchKey,
27 productHuntRateLimitDelaySeconds,
28 productHuntStreamOrder,
29 shouldUseLightweightProductHuntIndex,
30 rankEmails,
31 resumableCheckpointCursor,
32 type EmailListVerifyStatus,
33 type WebsitePage,
34} from './enrichment-utils.js';
35import {
36 buildLeadProfile,
37 buildDomainHypothesisCandidates,
38 buildProtectedProductCandidates,
39 buildProductIdentityNames,
40 buildWebsiteSearchQueries,
41 evaluateWebsiteIdentity,
42 hasDirectOwnershipSignal,
43 hasProtectedProductVendorMatch,
44 isPrivateNetworkAddress,
45 isProtectedProductHostAllowed,
46 isRetryableDataForSeoStatus,
47 isUsableEmailStatus,
48 matchesProductSearch,
49 normalizeContactFinderResults,
50 parseDataForSeoCandidates,
51 parseSerpApiCandidates,
52 rankMakerWebsiteLinkCandidates,
53 scoreSearchCandidate,
54 selectMakerFallbackEmails,
55 splitMakerName,
56 type ContactFinderResult,
57 type MakerWebsiteLink,
58 type SearchCandidate,
59} from './lead-engine.js';
60
61await Actor.init();
62
63type Mode = 'leaderboard' | 'search' | 'topic' | 'urls';
64type LeaderboardPeriod = 'daily' | 'weekly' | 'monthly' | 'yearly';
65type RunOutcome = 'COMPLETE' | 'PARTIAL' | 'VALID_EMPTY' | 'INVALID_INPUT' | 'UPSTREAM_FAILED' | 'CONFIG_ERROR';
66type EmailSource = 'page_scrape' | 'maker_website' | 'emaillistverify_contact_finder' | 'mixed' | 'none';
67type EmailResult = 'emails_found' | 'no_public_email_found' | 'not_assessed';
68type Coverage = 'complete' | 'partial' | 'none';
69type WebsiteResolutionStatus = 'verified_external' | 'verified_search_match' | 'unresolved' | 'not_available';
70type WebsiteResolutionSource = 'api_website_product_link' | 'input_override' | 'product_hunt_redirect' | 'maker_website' | 'maker_website_link' | 'protected_vendor_domain' | 'dataforseo_organic' | 'direct_domain_hypothesis' | 'serpapi_organic' | 'none';
71type EmailVerification = {
72 email: string;
73 provider: 'emaillistverify';
74 status: EmailListVerifyStatus;
75 checkedAt: string | null;
76 source: 'public_website' | 'maker_website' | 'emaillistverify_contact_finder';
77 confidence: 'high' | 'medium' | 'low' | 'unknown' | null;
78};
79
80type DateRange = { postedAfter: string; postedBefore: string };
81type WebsiteOverride = { keys: string[]; websiteUrl: string };
82type EnrichmentResult = {
83 emails: string[];
84 emailSource: EmailSource;
85 result: EmailResult;
86 coverage: Coverage;
87 terminalReason: string;
88 pagesVisited: number;
89 pageUrls: string[];
90 warnings: string[];
91};
92
93type WebsiteResolution = {
94 websiteUrl: string | null;
95 source: WebsiteResolutionSource;
96 status: WebsiteResolutionStatus;
97 confidence: number;
98 signals: string[];
99 providerCostUsd: number;
100 providerAttempts: number;
101 warnings: string[];
102};
103
104type ContactDiscovery = {
105 contacts: ContactFinderResult[];
106 attempts: number;
107 credits: number;
108 warnings: string[];
109};
110
111const PH_API = 'https://api.producthunt.com/v2/api/graphql';
112const RESULT_EVENT = 'apify-default-dataset-item';
113const EMAIL_EVENT = 'email-found';
114const RESULT_PRICE_USD = 0.002;
115const EMAIL_PRICE_USD = 0.04;
116const EMAIL_LIST_VERIFY_API = 'https://api.emaillistverify.com/api/verifyEmail';
117const EMAIL_LIST_VERIFY_CONTACT_API = 'https://api.emaillistverify.com/api/findContact';
118const DATAFORSEO_SEARCH_API = 'https://api.dataforseo.com/v3/serp/google/organic/live/advanced';
119const SERPAPI_SEARCH_API = 'https://serpapi.com/search.json';
120const EMAIL_LIST_VERIFY_MIN_INTERVAL_MS = 225;
121const SEARCH_PAGES_WITHOUT_MATCH_LIMIT = 50;
122const FAILED_OUTCOMES = new Set<RunOutcome>(['UPSTREAM_FAILED', 'CONFIG_ERROR']);
123
124const SKIP_DOMAINS = new Set([
125 'apps.apple.com', 'play.google.com', 'github.com', 'youtube.com',
126 'twitter.com', 'x.com', 'linkedin.com', 'facebook.com', 'instagram.com',
127 'reddit.com', 'discord.com', 'discord.gg', 'slack.com', 'medium.com',
128 'substack.com', 'producthunt.com', 'amazon.com', 'chrome.google.com',
129 'marketplace.visualstudio.com', 'notion.so', 'figma.com', 'trello.com',
130 'airtable.com',
131]);
132
133const PRODUCT_HUNT_USER_AGENT = 'ProductHunt Lead Generator/1.4 (authorized API client)';
134const WEBSITE_USER_AGENT = 'Mozilla/5.0 (compatible; ProductHuntLeadGenerator/1.4; +https://apify.com/khadinakbar/producthunt-scraper-pro)';
135
136let nextEmailListVerifyRequestAt = 0;
137let emailListVerifyCreditsExhausted = false;
138const publicHostCache = new Map<string, Promise<boolean>>();
139
140
141const MAX_BROWSER_CONTEXTS = 2;
142let activeBrowserContexts = 0;
143const browserContextWaiters: Array<() => void> = [];
144
145async function withBrowserContextSlot<T>(task: () => Promise<T>): Promise<T> {
146 if (activeBrowserContexts >= MAX_BROWSER_CONTEXTS) {
147 await new Promise<void>((resolve) => browserContextWaiters.push(resolve));
148 } else {
149 activeBrowserContexts += 1;
150 }
151 try {
152 return await task();
153 } finally {
154 const next = browserContextWaiters.shift();
155 if (next) next();
156 else activeBrowserContexts -= 1;
157 }
158}
159
160function isoDate(date: Date): string {
161 return date.toISOString().slice(0, 10);
162}
163
164function getWeekBounds(date: Date): { start: string; end: string } {
165 const start = new Date(date);
166
167 const weekday = date.getUTCDay() || 7;
168 start.setUTCDate(date.getUTCDate() - weekday + 1);
169 const end = new Date(start);
170 end.setUTCDate(start.getUTCDate() + 6);
171 return { start: isoDate(start), end: isoDate(end) };
172}
173
174function getMonthBounds(date: Date): { start: string; end: string } {
175 const year = date.getUTCFullYear();
176 const month = date.getUTCMonth();
177 return {
178 start: isoDate(new Date(Date.UTC(year, month, 1))),
179 end: isoDate(new Date(Date.UTC(year, month + 1, 0))),
180 };
181}
182
183function getYearBounds(date: Date): { start: string; end: string } {
184 const year = date.getUTCFullYear();
185 return { start: `${year}-01-01`, end: `${year}-12-31` };
186}
187
188function validIsoDate(value: string): boolean {
189 if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
190 const parsed = new Date(`${value}T12:00:00Z`);
191 return Number.isFinite(parsed.getTime()) && isoDate(parsed) === value;
192}
193
194function dateRangeForPeriod(period: LeaderboardPeriod, anchorDate: string, endDate = ''): DateRange {
195 const base = new Date(`${anchorDate}T12:00:00Z`);
196 if (period === 'daily') {
197 return {
198 postedAfter: `${anchorDate}T00:00:00+00:00`,
199 postedBefore: `${endDate || anchorDate}T23:59:59+00:00`,
200 };
201 }
202 const bounds = period === 'weekly'
203 ? getWeekBounds(base)
204 : period === 'monthly'
205 ? getMonthBounds(base)
206 : getYearBounds(base);
207 return {
208 postedAfter: `${bounds.start}T00:00:00+00:00`,
209 postedBefore: `${bounds.end}T23:59:59+00:00`,
210 };
211}
212
213function normalizeKey(value: unknown): string | null {
214 if (typeof value !== 'string' || !value.trim()) return null;
215 return value.trim().toLowerCase().replace(/[?#].*$/, '').replace(/\/$/, '');
216}
217
218function parseWebsiteOverrides(value: unknown): WebsiteOverride[] {
219 const overrides: WebsiteOverride[] = [];
220 const add = (keys: unknown[], websiteUrl: unknown): void => {
221 if (typeof websiteUrl !== 'string' || !isPublicHttpUrl(websiteUrl)) return;
222 const canonical = canonicalWebsiteUrl(websiteUrl);
223 if (!canonical || isProductHuntHost(new URL(canonical).hostname)) return;
224 const usableKeys = keys.map(normalizeKey).filter((key): key is string => Boolean(key));
225 if (usableKeys.length > 0) overrides.push({ keys: usableKeys, websiteUrl: canonical });
226 };
227
228 if (Array.isArray(value)) {
229 for (const entry of value) {
230 if (!entry || typeof entry !== 'object') continue;
231 const row = entry as Record<string, unknown>;
232 add([row.productHuntUrl, row.productSlug, row.productName, row.productId], row.websiteUrl);
233 }
234 } else if (value && typeof value === 'object') {
235 for (const [key, websiteUrl] of Object.entries(value as Record<string, unknown>)) add([key], websiteUrl);
236 }
237 return overrides;
238}
239
240function lookupWebsiteOverride(node: Record<string, unknown>, overrides: WebsiteOverride[]): string | null {
241 const candidates = [node.url, node.slug, node.id, node.name]
242 .map(normalizeKey)
243 .filter((value): value is string => Boolean(value));
244 return overrides.find((override) => override.keys.some((key) => candidates.includes(key)))?.websiteUrl ?? null;
245}
246
247function rootDomain(hostname: string): string {
248 const labels = hostname.replace(/^www\./i, '').toLowerCase().split('.');
249 return labels.slice(-2).join('.');
250}
251
252function isAllowedExternalWebsite(url: string): boolean {
253 if (!isPublicHttpUrl(url)) return false;
254 try {
255 const host = new URL(url).hostname.toLowerCase();
256 return !isProductHuntHost(host) && !SKIP_DOMAINS.has(host) && !SKIP_DOMAINS.has(rootDomain(host));
257 } catch {
258 return false;
259 }
260}
261
262async function isSafePublicNetworkUrl(rawUrl: string): Promise<boolean> {
263 if (!isPublicHttpUrl(rawUrl)) return false;
264 let hostname: string;
265 try {
266 hostname = new URL(rawUrl).hostname.toLowerCase().replace(/^\[|\]$/g, '');
267 } catch {
268 return false;
269 }
270 if (isIP(hostname)) return !isPrivateNetworkAddress(hostname);
271
272 let assessment = publicHostCache.get(hostname);
273 if (!assessment) {
274 assessment = lookup(hostname, { all: true, verbatim: true })
275 .then((records) => records.length > 0 && records.every(({ address }) => !isPrivateNetworkAddress(address)))
276 .catch(() => false);
277 publicHostCache.set(hostname, assessment);
278 }
279 return assessment;
280}
281
282
283
284function browserExtractEmails(): string[] {
285 const emailRe = /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g;
286 const obfuscatedRe = /\b[a-zA-Z0-9._%+\-]+\s*(?:\[at\]|\(at\)|\sat\s)\s*[a-zA-Z0-9.\-]+\s*(?:\[dot\]|\(dot\)|\sdot\s)\s*[a-zA-Z]{2,}\b/gi;
287 const collected: string[] = [];
288 const add = (value: unknown): void => {
289 if (typeof value !== 'string') return;
290 let decoded = value;
291 try { decoded = decodeURIComponent(value); } catch { }
292 const email = decoded.trim().toLowerCase().replace(/^mailto:/, '').split('?')[0];
293 if (email.includes('@')) collected.push(email);
294 };
295 const walkJson = (value: unknown, depth = 0): void => {
296 if (depth > 8 || value === null || value === undefined) return;
297 if (Array.isArray(value)) {
298 value.forEach((item) => walkJson(item, depth + 1));
299 return;
300 }
301 if (typeof value !== 'object') return;
302 for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
303 if (key.toLowerCase() === 'email') add(child);
304 walkJson(child, depth + 1);
305 }
306 };
307 const decodeCfEmail = (encoded: string): string | null => {
308 if (!/^[0-9a-f]+$/i.test(encoded) || encoded.length < 4) return null;
309 const key = Number.parseInt(encoded.slice(0, 2), 16);
310 let decoded = '';
311 for (let index = 2; index < encoded.length; index += 2) {
312 decoded += String.fromCharCode(Number.parseInt(encoded.slice(index, index + 2), 16) ^ key);
313 }
314 return decoded;
315 };
316
317 document.querySelectorAll('a[href^="mailto:"]').forEach((element) => add((element as HTMLAnchorElement).href));
318 document.querySelectorAll('[data-email]').forEach((element) => add(element.getAttribute('data-email')));
319 document.querySelectorAll('[data-cfemail]').forEach((element) => {
320 const decoded = decodeCfEmail(element.getAttribute('data-cfemail') ?? '');
321 if (decoded) add(decoded);
322 });
323 document.querySelectorAll('script[type="application/ld+json"]').forEach((element) => {
324 try { walkJson(JSON.parse(element.textContent ?? '')); } catch { }
325 });
326
327 const combined = `${document.body?.innerText ?? ''}\n${document.documentElement.innerHTML}`
328 .replace(/\\(?:x40|u0040)/gi, '@')
329 .replace(/\\(?:x2e|u002e)/gi, '.')
330 .replace(/&#(?:64|x40);|@/gi, '@')
331 .replace(/&#(?:46|x2e);|./gi, '.');
332 (combined.match(emailRe) ?? []).forEach(add);
333 for (const match of combined.match(obfuscatedRe) ?? []) {
334 add(match.replace(/\s*(?:\[at\]|\(at\)|\sat\s)\s*/i, '@').replace(/\s*(?:\[dot\]|\(dot\)|\sdot\s)\s*/ig, '.'));
335 }
336 return collected;
337}
338
339
340
341function browserExtractLinkedFallbackPages(origin: string): WebsitePage[] {
342 const matchers: Array<{ kind: Exclude<WebsitePage['kind'], 'homepage'>; priority: number; pattern: RegExp }> = [
343 { kind: 'contact', priority: 0, pattern: /(contact|contact-us|get-in-touch|reach-us)/i },
344 { kind: 'about', priority: 1, pattern: /(about|about-us|company|our-story)/i },
345 { kind: 'team', priority: 2, pattern: /(team|our-team|meet-the-team|people|founders)/i },
346 { kind: 'privacy', priority: 3, pattern: /(privacy|privacy-policy)/i },
347 { kind: 'terms', priority: 4, pattern: /(terms|terms-of-service|terms-and-conditions|conditions)/i },
348 ];
349 const links = new Map<string, WebsitePage>();
350 document.querySelectorAll('a[href]').forEach((element) => {
351 const anchor = element as HTMLAnchorElement;
352 try {
353 const url = new URL(anchor.href);
354 if (url.origin !== origin || !['http:', 'https:'].includes(url.protocol) || url.username || url.password) return;
355 const label = (anchor.innerText || anchor.textContent || '').trim();
356 const matched = matchers.find((candidate) => candidate.pattern.test(`${url.pathname} ${label}`));
357 if (!matched) return;
358 url.hash = '';
359 const clean = url.toString();
360 if (clean === `${origin}/` || clean === origin || links.has(clean)) return;
361 links.set(clean, { url: clean, kind: matched.kind, priority: matched.priority });
362 } catch { }
363 });
364 return [...links.values()].sort((left, right) => left.priority - right.priority);
365}
366
367async function waitForEmailListVerifySlot(): Promise<void> {
368 const scheduledAt = Math.max(Date.now(), nextEmailListVerifyRequestAt);
369 nextEmailListVerifyRequestAt = scheduledAt + EMAIL_LIST_VERIFY_MIN_INTERVAL_MS;
370 const delayMs = scheduledAt - Date.now();
371 if (delayMs > 0) await new Promise((resolve) => setTimeout(resolve, delayMs));
372}
373
374async function verifyEmailsWithEmailListVerify(
375 emails: string[],
376 requested: boolean,
377 apiKey: string,
378 source: EmailVerification['source'] = 'public_website',
379 maxUsableEmails = emails.length,
380): Promise<{ verifications: EmailVerification[]; attempts: number; warnings: string[] }> {
381 if (emails.length === 0) return { verifications: [], attempts: 0, warnings: [] };
382 if (!requested) {
383 return {
384 verifications: emails.map((email) => ({
385 email, provider: 'emaillistverify', status: 'not_requested', checkedAt: null,
386 source, confidence: null,
387 })),
388 attempts: 0,
389 warnings: [],
390 };
391 }
392 if (!apiKey) {
393 return {
394 verifications: emails.map((email) => ({
395 email, provider: 'emaillistverify', status: 'not_configured', checkedAt: null,
396 source, confidence: null,
397 })),
398 attempts: 0,
399 warnings: ['EmailListVerify was requested but no owner-managed API key was configured; public emails were kept unverified.'],
400 };
401 }
402 if (emailListVerifyCreditsExhausted) {
403 return {
404 verifications: emails.map((email) => ({
405 email, provider: 'emaillistverify', status: 'error_credit', checkedAt: null,
406 source, confidence: null,
407 })),
408 attempts: 0,
409 warnings: ['EmailListVerify credits were exhausted earlier in this run; no additional verification requests were sent.'],
410 };
411 }
412
413 const warnings = new Set<string>();
414 const verifications: EmailVerification[] = [];
415 let attempts = 0;
416 for (const email of emails) {
417 if (hasReachedUsableEmailTarget(verifications.map(({ status }) => status), maxUsableEmails)) break;
418 for (let requestAttempt = 0; requestAttempt < 2; requestAttempt += 1) {
419 await waitForEmailListVerifySlot();
420 attempts += 1;
421 const checkedAt = new Date().toISOString();
422 try {
423 const requestUrl = new URL(EMAIL_LIST_VERIFY_API);
424 requestUrl.searchParams.set('email', email);
425 const response = await fetch(requestUrl, {
426 headers: { 'x-api-key': apiKey, Accept: 'text/plain, application/json;q=0.9' },
427 signal: AbortSignal.timeout(12_000),
428 });
429 if (!response.ok) {
430 if (requestAttempt === 0 && isRetryableEmailListVerifyHttpStatus(response.status)) {
431 await new Promise((resolve) => setTimeout(resolve, 500));
432 continue;
433 }
434 warnings.add(`EmailListVerify returned HTTP ${response.status}; affected public emails were retained with provider_error status.`);
435 if (response.status === 402) emailListVerifyCreditsExhausted = true;
436 verifications.push({
437 email, provider: 'emaillistverify', status: response.status === 402 ? 'error_credit' : 'provider_error', checkedAt,
438 source, confidence: null,
439 });
440 break;
441 }
442 const status = normalizeEmailListVerifyStatus(await response.text());
443 if (status === 'provider_error') warnings.add('EmailListVerify returned an undocumented response; affected public emails were retained with provider_error status.');
444 if (status === 'error_credit') {
445 emailListVerifyCreditsExhausted = true;
446 warnings.add('EmailListVerify reported insufficient verification credits; remaining public emails were retained with error_credit status.');
447 }
448 verifications.push({ email, provider: 'emaillistverify', status, checkedAt, source, confidence: null });
449 break;
450 } catch {
451 if (requestAttempt === 0) {
452 await new Promise((resolve) => setTimeout(resolve, 500));
453 continue;
454 }
455 warnings.add('EmailListVerify request failed twice; affected public emails were retained with provider_error status.');
456 verifications.push({ email, provider: 'emaillistverify', status: 'provider_error', checkedAt, source, confidence: null });
457 }
458 }
459 if (emailListVerifyCreditsExhausted) break;
460 }
461 return { verifications, attempts, warnings: [...warnings] };
462}
463
464let browser: Browser | null = null;
465
466async function ensureBrowser(): Promise<Browser> {
467 if (!browser) {
468 browser = await chromium.launch({
469 headless: true,
470 args: [
471 '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage',
472 '--disable-gpu', '--no-zygote', '--disable-extensions',
473 ],
474 });
475 log.info('Browser launched for verified external website enrichment.');
476 }
477 return browser;
478}
479
480async function closeBrowser(): Promise<void> {
481 if (!browser) return;
482 await browser.close().catch(() => undefined);
483 browser = null;
484}
485
486function extractStaticEmails(html: string): string[] {
487 const $ = load(html);
488 const collected: string[] = [];
489 const add = (value: unknown): void => {
490 if (typeof value !== 'string') return;
491 let decoded = value;
492 try { decoded = decodeURIComponent(value); } catch { }
493 const email = decoded.trim().toLowerCase().replace(/^mailto:/, '').split('?')[0];
494 if (email.includes('@')) collected.push(email);
495 };
496 const walkJson = (value: unknown, depth = 0): void => {
497 if (depth > 8 || value === null || value === undefined) return;
498 if (Array.isArray(value)) return value.forEach((item) => walkJson(item, depth + 1));
499 if (typeof value !== 'object') return;
500 for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
501 if (key.toLowerCase() === 'email') add(child);
502 walkJson(child, depth + 1);
503 }
504 };
505 $('a[href^="mailto:"]').each((_, element) => add($(element).attr('href')));
506 $('[data-email]').each((_, element) => add($(element).attr('data-email')));
507 $('[data-cfemail]').each((_, element) => add(decodeCloudflareEmail($(element).attr('data-cfemail') ?? '')));
508 $('script[type="application/ld+json"]').each((_, element) => {
509 try { walkJson(JSON.parse($(element).text())); } catch { }
510 });
511 const combined = `${$.text()}\n${html}`;
512 extractEmailCandidatesFromText(combined).forEach(add);
513 return collected;
514}
515
516function extractStaticFallbackPages(html: string, origin: string): WebsitePage[] {
517 const $ = load(html);
518 const matchers: Array<{ kind: Exclude<WebsitePage['kind'], 'homepage'>; priority: number; pattern: RegExp }> = [
519 { kind: 'contact', priority: 0, pattern: /(contact|contact-us|get-in-touch|reach-us)/i },
520 { kind: 'about', priority: 1, pattern: /(about|about-us|company|our-story)/i },
521 { kind: 'team', priority: 2, pattern: /(team|our-team|meet-the-team|people|founders)/i },
522 { kind: 'privacy', priority: 3, pattern: /(privacy|privacy-policy)/i },
523 { kind: 'terms', priority: 4, pattern: /(terms|terms-of-service|terms-and-conditions|conditions)/i },
524 ];
525 const candidates = new Map<string, WebsitePage>();
526 $('a[href]').each((_, element) => {
527 try {
528 const url = new URL($(element).attr('href') ?? '', origin);
529 if (url.origin !== origin || !['http:', 'https:'].includes(url.protocol) || url.username || url.password) return;
530 const label = $(element).text().trim();
531 const matched = matchers.find((candidate) => candidate.pattern.test(`${url.pathname} ${label}`));
532 if (!matched) return;
533 url.hash = '';
534 const clean = url.toString();
535 if (clean === `${origin}/` || clean === origin || candidates.has(clean)) return;
536 candidates.set(clean, { url: clean, kind: matched.kind, priority: matched.priority });
537 } catch { }
538 });
539 return [...candidates.values()].sort((left, right) => left.priority - right.priority);
540}
541
542type StaticPage = { ok: true; url: string; html: string } | { ok: false; reason: string };
543
544async function fetchPublicHtml(rawUrl: string, timeoutMs: number): Promise<StaticPage> {
545 let current = canonicalWebsiteUrl(rawUrl);
546 if (!current || !isAllowedExternalWebsite(current)) return { ok: false, reason: 'invalid_or_private_target' };
547 try {
548 for (let hop = 0; hop <= 3; hop += 1) {
549 if (!await isSafePublicNetworkUrl(current)) return { ok: false, reason: 'private_or_unresolvable_target' };
550 const response = await fetch(current, {
551 method: 'GET',
552 redirect: 'manual',
553 headers: { Accept: 'text/html,application/xhtml+xml', 'User-Agent': WEBSITE_USER_AGENT },
554 signal: AbortSignal.timeout(timeoutMs),
555 });
556 if (response.status >= 300 && response.status < 400) {
557 const location = response.headers.get('location');
558 if (!location || hop === 3) return { ok: false, reason: 'redirect_hop_limit' };
559 const next = canonicalWebsiteUrl(new URL(location, current).toString());
560 if (!next || !isAllowedExternalWebsite(next)) return { ok: false, reason: 'redirect_to_private_or_unsupported_target' };
561 if (!await isSafePublicNetworkUrl(next)) return { ok: false, reason: 'redirect_to_private_or_unresolvable_target' };
562 current = next;
563 continue;
564 }
565 if (!response.ok) return { ok: false, reason: `http_${response.status}` };
566 const contentType = response.headers.get('content-type') ?? '';
567 if (!/(?:text\/html|application\/xhtml\+xml)/i.test(contentType)) return { ok: false, reason: 'non_html_response' };
568 const contentLength = Number(response.headers.get('content-length') ?? '0');
569 if (Number.isFinite(contentLength) && contentLength > 1_500_000) return { ok: false, reason: 'response_too_large' };
570 const html = (await response.text()).slice(0, 1_500_000);
571 return { ok: true, url: current, html };
572 }
573 return { ok: false, reason: 'redirect_hop_limit' };
574 } catch (error) {
575 const message = error instanceof Error ? error.message : '';
576 return { ok: false, reason: /timeout|abort/i.test(message) ? 'timeout' : 'network_error' };
577 }
578}
579
580function pageIdentity(html: string): { title: string; description: string; text: string } {
581 const $ = load(html);
582 $('script,style,noscript,svg').remove();
583 return {
584 title: $('title').first().text().trim(),
585 description: $('meta[name="description"]').attr('content')?.trim()
586 || $('meta[property="og:description"]').attr('content')?.trim()
587 || '',
588 text: $('body').text().replace(/\s+/g, ' ').trim().slice(0, 100_000),
589 };
590}
591
592async function resolveProductHuntRedirect(rawUrl: string): Promise<string | null> {
593 let current = canonicalWebsiteUrl(rawUrl);
594 if (!current) return null;
595 try {
596 if (!isProductHuntHost(new URL(current).hostname)) return null;
597 for (let hop = 0; hop < 4; hop += 1) {
598 if (!await isSafePublicNetworkUrl(current)) return null;
599 let response = await fetch(current, {
600 method: 'HEAD',
601 redirect: 'manual',
602 headers: { 'User-Agent': PRODUCT_HUNT_USER_AGENT, Accept: 'text/html,*/*;q=0.5' },
603 signal: AbortSignal.timeout(8_000),
604 });
605 if (response.status === 405) {
606 response = await fetch(current, {
607 method: 'GET',
608 redirect: 'manual',
609 headers: { 'User-Agent': PRODUCT_HUNT_USER_AGENT, Accept: 'text/html,*/*;q=0.5' },
610 signal: AbortSignal.timeout(8_000),
611 });
612 }
613 if (response.status < 300 || response.status >= 400) return null;
614 const location = response.headers.get('location');
615 if (!location) return null;
616 const next = canonicalWebsiteUrl(new URL(location, current).toString());
617 if (!next) return null;
618 if (!isProductHuntHost(new URL(next).hostname)) {
619 return isAllowedExternalWebsite(next) && await isSafePublicNetworkUrl(next) ? next : null;
620 }
621 current = next;
622 }
623 } catch {
624 return null;
625 }
626 return null;
627}
628
629async function verifyWebsiteCandidates(
630 candidates: SearchCandidate[],
631 productName: string,
632 tagline: string,
633 description = '',
634): Promise<{ websiteUrl: string; confidence: number; signals: string[] } | null> {
635 const identityNames = buildProductIdentityNames(productName);
636 const purpose = `${tagline} ${description}`.trim().slice(0, 1_200);
637 const ranked = [...candidates]
638 .map((candidate) => ({
639 candidate,
640 score: Math.max(...identityNames.map((identity) => scoreSearchCandidate(candidate, identity, purpose))),
641 }))
642 .filter(({ score }) => score >= 34)
643 .sort((left, right) => right.score - left.score || left.candidate.rank - right.candidate.rank)
644 .slice(0, 5);
645
646 for (const { candidate } of ranked) {
647 let candidateUrl: URL;
648 try {
649 candidateUrl = new URL(candidate.url);
650 } catch {
651 continue;
652 }
653 if (!isProtectedProductHostAllowed(productName, candidateUrl.toString())) continue;
654 const homepage = `${candidateUrl.origin}/`;
655 const verificationUrls = candidateUrl.pathname !== '/' || candidateUrl.search
656 ? [candidateUrl.toString(), homepage]
657 : [homepage];
658 for (const verificationUrl of [...new Set(verificationUrls)]) {
659 const fetched = await fetchPublicHtml(verificationUrl, 6_000);
660 if (!fetched.ok) continue;
661 const identity = pageIdentity(fetched.html);
662 const assessments = identityNames.map((identityName) => ({
663 identityName,
664 assessment: evaluateWebsiteIdentity({
665 candidate: { ...candidate, url: fetched.url },
666 productName: identityName,
667 tagline: purpose,
668 pageTitle: identity.title,
669 pageDescription: identity.description,
670 pageText: identity.text,
671 }),
672 })).filter(({ assessment }) => assessment.accepted)
673 .sort((left, right) => right.assessment.confidence - left.assessment.confidence);
674 if (assessments.length === 0) continue;
675 const { assessment, identityName } = assessments[0];
676 if (new URL(verificationUrl).pathname !== '/'
677 && !hasDirectOwnershipSignal(assessment.signals)
678 && !hasProtectedProductVendorMatch(productName, verificationUrl)) {
679 continue;
680 }
681 const finalUrl = new URL(fetched.url);
682 const signals = [...assessment.signals];
683 if (identityName !== identityNames[0]) signals.push('core_brand_alias_confirmed');
684 if (new URL(verificationUrl).pathname !== '/') signals.push('official_product_page_confirmed');
685 return {
686 websiteUrl: `${finalUrl.origin}/`,
687 confidence: assessment.confidence,
688 signals: [...new Set(signals)],
689 };
690 }
691 }
692 return null;
693}
694
695async function discoverMakerLinkedProductCandidates(
696 makerWebsiteUrls: string[],
697 productName: string,
698 tagline: string,
699): Promise<SearchCandidate[]> {
700 const collected = new Map<string, SearchCandidate>();
701 for (const makerWebsiteUrl of makerWebsiteUrls.filter((url) => isAllowedExternalWebsite(url)).slice(0, 2)) {
702 const fetched = await fetchPublicHtml(makerWebsiteUrl, 6_000);
703 if (!fetched.ok) continue;
704 const makerOrigin = new URL(fetched.url).origin;
705 const $ = load(fetched.html);
706 const links: MakerWebsiteLink[] = [];
707 $('a[href]').each((index, element) => {
708 try {
709 const target = new URL($(element).attr('href') ?? '', fetched.url);
710 if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) return;
711 if (target.origin === makerOrigin) return;
712 const text = $(element).text().replace(/\s+/g, ' ').trim();
713 const context = $(element).closest('p,li,article,section,div').first().text().replace(/\s+/g, ' ').trim();
714 links.push({ url: target.toString(), text, context, rank: index + 1 });
715 } catch { }
716 });
717 for (const candidate of rankMakerWebsiteLinkCandidates(links, fetched.url, productName, tagline)) {
718 let key: string;
719 try { key = new URL(candidate.url).origin; } catch { continue; }
720 if (!collected.has(key)) collected.set(key, candidate);
721 }
722 }
723 return [...collected.values()]
724 .sort((left, right) => scoreSearchCandidate(right, productName, tagline) - scoreSearchCandidate(left, productName, tagline))
725 .slice(0, 5);
726}
727
728class DataForSeoSearchError extends Error {
729 constructor(message: string, readonly retryable: boolean, readonly costUsd: number) {
730 super(message);
731 }
732}
733
734async function callDataForSeoWebsiteSearch(
735 query: string,
736 login: string,
737 password: string,
738): Promise<{ candidates: SearchCandidate[]; costUsd: number }> {
739 const response = await fetch(DATAFORSEO_SEARCH_API, {
740 method: 'POST',
741 headers: {
742 Authorization: `Basic ${Buffer.from(`${login}:${password}`).toString('base64')}`,
743 'Content-Type': 'application/json',
744 Accept: 'application/json',
745 },
746 body: JSON.stringify([{
747 keyword: query,
748 location_code: 2840,
749 language_code: 'en',
750 device: 'desktop',
751 os: 'windows',
752 depth: 10,
753 group_organic_results: true,
754 }]),
755 signal: AbortSignal.timeout(45_000),
756 });
757 if (!response.ok) {
758 throw new DataForSeoSearchError(
759 `DataForSEO website search returned HTTP ${response.status}.`,
760 response.status === 429 || response.status >= 500,
761 0,
762 );
763 }
764 const body = await response.json() as Record<string, any>;
765 const parsed = parseDataForSeoCandidates(body);
766 if (body.status_code && body.status_code !== 20000) {
767 throw new DataForSeoSearchError(
768 `DataForSEO website search failed with provider code ${body.status_code}: ${String(body.status_message ?? 'unknown error')}.`,
769 isRetryableDataForSeoStatus(Number(body.status_code)),
770 parsed.costUsd,
771 );
772 }
773 const task = Array.isArray(body.tasks) ? body.tasks[0] : null;
774 if (task?.status_code && task.status_code !== 20000) {
775 throw new DataForSeoSearchError(
776 `DataForSEO website search task failed with provider code ${task.status_code}: ${String(task.status_message ?? 'unknown error')}.`,
777 isRetryableDataForSeoStatus(Number(task.status_code)),
778 parsed.costUsd,
779 );
780 }
781 return parsed;
782}
783
784async function callSerpApiWebsiteSearch(query: string, apiKey: string): Promise<SearchCandidate[]> {
785 const url = new URL(SERPAPI_SEARCH_API);
786 url.searchParams.set('engine', 'google');
787 url.searchParams.set('q', query);
788 url.searchParams.set('num', '10');
789 url.searchParams.set('gl', 'us');
790 url.searchParams.set('hl', 'en');
791 url.searchParams.set('api_key', apiKey);
792 const response = await fetch(url, { signal: AbortSignal.timeout(30_000), headers: { Accept: 'application/json' } });
793 if (!response.ok) throw new Error(`SerpApi website search returned HTTP ${response.status}.`);
794 const body = await response.json() as Record<string, unknown>;
795 if (body.error) throw new Error('SerpApi website search returned a provider error.');
796 return parseSerpApiCandidates(body);
797}
798
799async function resolveWebsite(input: {
800 productName: string;
801 tagline: string;
802 description: string;
803 productHuntRedirect: string;
804 makerWebsiteUrls: string[];
805 dataForSeoLogin: string;
806 dataForSeoPassword: string;
807 serpApiKey: string;
808}): Promise<WebsiteResolution> {
809 const warnings: string[] = [];
810 let providerCostUsd = 0;
811 let providerAttempts = 0;
812 const unresolved = (): WebsiteResolution => ({
813 websiteUrl: null,
814 source: 'none',
815 status: input.productHuntRedirect ? 'unresolved' : 'not_available',
816 confidence: 0,
817 signals: [],
818 providerCostUsd,
819 providerAttempts,
820 warnings,
821 });
822
823 if (input.productHuntRedirect) {
824 const destination = await resolveProductHuntRedirect(input.productHuntRedirect);
825 if (destination) {
826 const verified = await verifyWebsiteCandidates([{
827 url: destination, title: input.productName, snippet: input.tagline, rank: 1,
828 }], input.productName, input.tagline, input.description);
829 if (verified) {
830 return {
831 ...verified,
832 source: 'product_hunt_redirect',
833 status: 'verified_external',
834 providerCostUsd,
835 providerAttempts,
836 warnings,
837 };
838 }
839 }
840 }
841
842 const makerCandidates = input.makerWebsiteUrls
843 .filter((url) => isAllowedExternalWebsite(url))
844 .slice(0, 2)
845 .map((url, index): SearchCandidate => ({ url, title: '', snippet: '', rank: index + 1 }));
846 if (makerCandidates.length > 0) {
847 const verified = await verifyWebsiteCandidates(makerCandidates, input.productName, input.tagline, input.description);
848 if (verified) {
849 return {
850 ...verified,
851 source: 'maker_website',
852 status: 'verified_external',
853 providerCostUsd,
854 providerAttempts,
855 warnings,
856 };
857 }
858
859 const linkedCandidates = await discoverMakerLinkedProductCandidates(
860 input.makerWebsiteUrls,
861 input.productName,
862 input.tagline,
863 );
864 if (linkedCandidates.length > 0) {
865 const linked = await verifyWebsiteCandidates(linkedCandidates, input.productName, input.tagline, input.description);
866 if (linked) {
867 return {
868 ...linked,
869 signals: [...new Set([...linked.signals, 'product_linked_from_maker_website'])],
870 source: 'maker_website_link',
871 status: 'verified_external',
872 providerCostUsd,
873 providerAttempts,
874 warnings,
875 };
876 }
877 warnings.push('Maker websites exposed product-like links, but none passed product-identity verification.');
878 }
879 }
880
881 const vendorCandidate = await verifyWebsiteCandidates(
882 buildProtectedProductCandidates(input.productName).map((candidate) => ({
883 ...candidate,
884 snippet: input.tagline,
885 })),
886 input.productName,
887 input.tagline,
888 input.description,
889 );
890 if (vendorCandidate) {
891 return {
892 ...vendorCandidate,
893 signals: [...new Set([...vendorCandidate.signals, 'protected_vendor_domain_confirmed'])],
894 source: 'protected_vendor_domain',
895 status: 'verified_external',
896 providerCostUsd,
897 providerAttempts,
898 warnings,
899 };
900 }
901
902 const queries = buildWebsiteSearchQueries(input.productName, input.tagline);
903 if (input.dataForSeoLogin && input.dataForSeoPassword) {
904 let queryIndex = 0;
905 while (queryIndex < queries.length && providerAttempts < 2) {
906 const query = queries[queryIndex];
907 providerAttempts += 1;
908 try {
909 const search = await callDataForSeoWebsiteSearch(query, input.dataForSeoLogin, input.dataForSeoPassword);
910 providerCostUsd += search.costUsd;
911 const verified = await verifyWebsiteCandidates(
912 search.candidates,
913 input.productName,
914 input.tagline,
915 input.description,
916 );
917 if (verified) {
918 return {
919 ...verified,
920 source: 'dataforseo_organic',
921 status: 'verified_search_match',
922 providerCostUsd,
923 providerAttempts,
924 warnings,
925 };
926 }
927 warnings.push(`DataForSEO query ${queryIndex + 1} returned no website candidate that passed product-identity verification.`);
928 queryIndex += 1;
929 } catch (error) {
930 if (error instanceof DataForSeoSearchError) providerCostUsd += error.costUsd;
931 const message = error instanceof Error ? error.message : 'DataForSEO website search failed.';
932 warnings.push(message.slice(0, 200));
933 if (error instanceof DataForSeoSearchError && error.retryable && providerAttempts < 2) {
934 continue;
935 }
936 queryIndex += 1;
937 }
938 }
939 }
940
941 const directDomain = await verifyWebsiteCandidates(
942 buildDomainHypothesisCandidates(input.productName),
943 input.productName,
944 input.tagline,
945 input.description,
946 );
947 if (directDomain) {
948 return {
949 ...directDomain,
950 signals: [...new Set([...directDomain.signals, 'direct_domain_hypothesis_confirmed'])],
951 source: 'direct_domain_hypothesis',
952 status: 'verified_external',
953 providerCostUsd,
954 providerAttempts,
955 warnings,
956 };
957 }
958
959 if (input.serpApiKey) {
960 providerAttempts += 1;
961 try {
962 const query = queries.at(-1) ?? buildWebsiteSearchQueries(input.productName, '')[0];
963 const candidates = await callSerpApiWebsiteSearch(query, input.serpApiKey);
964 const verified = await verifyWebsiteCandidates(candidates, input.productName, input.tagline, input.description);
965 if (verified) {
966 return {
967 ...verified,
968 source: 'serpapi_organic',
969 status: 'verified_search_match',
970 providerCostUsd,
971 providerAttempts,
972 warnings,
973 };
974 }
975 warnings.push('SerpApi returned no website candidate that passed product-identity verification.');
976 } catch (error) {
977 const message = error instanceof Error ? error.message : 'SerpApi website search failed.';
978 warnings.push(message.slice(0, 160));
979 }
980 }
981
982 if (!input.dataForSeoLogin && !input.serpApiKey) {
983 warnings.push('No owner-managed website search provider was configured for unresolved Product Hunt links.');
984 }
985 return unresolved();
986}
987
988function companyDomainFromWebsite(websiteUrl: string): string {
989 try {
990 const hostname = new URL(websiteUrl).hostname.toLowerCase().replace(/^www\./, '');
991 return getDomain(hostname, { allowPrivateDomains: true }) ?? hostname;
992 } catch {
993 return '';
994 }
995}
996
997async function requestEmailListVerifyContacts(
998 apiKey: string,
999 body: { domain: string; firstName?: string; lastName?: string },
1000): Promise<unknown> {
1001 await waitForEmailListVerifySlot();
1002 const response = await fetch(EMAIL_LIST_VERIFY_CONTACT_API, {
1003 method: 'POST',
1004 headers: { 'x-api-key': apiKey, 'Content-Type': 'application/json', Accept: 'application/json' },
1005 body: JSON.stringify(body),
1006 signal: AbortSignal.timeout(30_000),
1007 });
1008 if (!response.ok) {
1009 if (response.status === 402) emailListVerifyCreditsExhausted = true;
1010 throw new Error(`EmailListVerify Contact Finder returned HTTP ${response.status}.`);
1011 }
1012 return response.json();
1013}
1014
1015async function discoverContacts(
1016 websiteUrl: string,
1017 makerNames: string[],
1018 maxEmails: number,
1019 apiKey: string,
1020): Promise<ContactDiscovery> {
1021 const domain = companyDomainFromWebsite(websiteUrl);
1022 if (!domain || maxEmails <= 0) return { contacts: [], attempts: 0, credits: 0, warnings: [] };
1023 if (!apiKey) {
1024 return {
1025 contacts: [], attempts: 0, credits: 0,
1026 warnings: ['Contact discovery was requested but the owner-managed EmailListVerify key is not configured.'],
1027 };
1028 }
1029 if (emailListVerifyCreditsExhausted) {
1030 return {
1031 contacts: [], attempts: 0, credits: 0,
1032 warnings: ['EmailListVerify credits were exhausted earlier in this run; contact discovery was skipped.'],
1033 };
1034 }
1035
1036 const warnings: string[] = [];
1037 let attempts = 0;
1038 let credits = 0;
1039 let contacts: ContactFinderResult[] = [];
1040 const parsedMakers = makerNames.map(splitMakerName).filter(({ firstName, lastName }) => firstName || lastName);
1041 const maker = parsedMakers.find(({ firstName, lastName }) => firstName && lastName) ?? parsedMakers[0];
1042 if (maker) {
1043 attempts += 1;
1044 credits += 5;
1045 try {
1046 const payload = await requestEmailListVerifyContacts(apiKey, { domain, ...maker });
1047 contacts = normalizeContactFinderResults(payload, domain, maxEmails);
1048 if (contacts.length >= maxEmails) return { contacts, attempts, credits, warnings };
1049 } catch (error) {
1050 const message = error instanceof Error ? error.message : 'EmailListVerify named contact discovery failed.';
1051 warnings.push(message.slice(0, 160));
1052 }
1053 }
1054
1055 if (!emailListVerifyCreditsExhausted) {
1056 attempts += 1;
1057 credits += 10;
1058 try {
1059 const payload = await requestEmailListVerifyContacts(apiKey, { domain });
1060 contacts = normalizeContactFinderResults([
1061 ...contacts,
1062 ...(Array.isArray(payload) ? payload : []),
1063 ], domain, maxEmails);
1064 return { contacts, attempts, credits, warnings };
1065 } catch (error) {
1066 const message = error instanceof Error ? error.message : 'EmailListVerify domain contact discovery failed.';
1067 warnings.push(message.slice(0, 160));
1068 }
1069 }
1070 return { contacts, attempts, credits, warnings };
1071}
1072
1073function selectBestEmails(
1074 publicVerifications: EmailVerification[],
1075 contacts: ContactFinderResult[],
1076 maxEmails: number,
1077): EmailVerification[] {
1078 const contactVerifications: EmailVerification[] = contacts.map((contact) => ({
1079 email: contact.email,
1080 provider: 'emaillistverify',
1081 status: contact.result as EmailListVerifyStatus,
1082 checkedAt: new Date().toISOString(),
1083 source: 'emaillistverify_contact_finder',
1084 confidence: contact.confidence,
1085 }));
1086 const statusScore: Record<string, number> = {
1087 ok: 100, ok_for_all: 70, unknown: 55, antispam_system: 50,
1088 not_requested: 48, not_configured: 45, provider_error: 40, error_credit: 38,
1089 smtp_protocol: 30, dead_server: 10, email_disabled: 5, invalid_mx: 0,
1090 invalid_syntax: 0, disposable: 0, spamtrap: 0,
1091 };
1092 const best = new Map<string, EmailVerification>();
1093 for (const verification of [...publicVerifications, ...contactVerifications]) {
1094 const current = best.get(verification.email);
1095 const sourceBonus = verification.source === 'public_website' ? 3 : verification.source === 'maker_website' ? 2 : 0;
1096 const score = (statusScore[verification.status] ?? 20)
1097 + sourceBonus
1098 + (verification.confidence === 'high' ? 2 : verification.confidence === 'medium' ? 1 : 0);
1099 const currentScore = current
1100 ? (statusScore[current.status] ?? 20) + (current.source === 'public_website' ? 3 : current.source === 'maker_website' ? 2 : 0)
1101 : -1;
1102 if (!current || score > currentScore) best.set(verification.email, verification);
1103 }
1104 return [...best.values()]
1105 .filter(({ status }) => isUsableEmailStatus(status))
1106 .sort((left, right) => {
1107 const leftScore = (statusScore[left.status] ?? 20) + (left.source === 'public_website' ? 3 : left.source === 'maker_website' ? 2 : 0);
1108 const rightScore = (statusScore[right.status] ?? 20) + (right.source === 'public_website' ? 3 : right.source === 'maker_website' ? 2 : 0);
1109 return rightScore - leftScore;
1110 })
1111 .slice(0, maxEmails);
1112}
1113
1114async function findEmails(
1115 rawWebsiteUrl: string,
1116 maxWebsitePages: number,
1117 maxEmailsPerProduct: number,
1118): Promise<EnrichmentResult> {
1119 const websiteUrl = canonicalWebsiteUrl(rawWebsiteUrl);
1120 if (!websiteUrl || !isAllowedExternalWebsite(websiteUrl)) {
1121 return {
1122 emails: [], emailSource: 'none', result: 'not_assessed', coverage: 'none',
1123 terminalReason: 'invalid_or_private_target', pagesVisited: 0, pageUrls: [],
1124 warnings: ['Website URL was not a permitted public external HTTP(S) target.'],
1125 };
1126 }
1127
1128 const initialUrl = new URL(websiteUrl);
1129 const maxEmailCandidates = emailCandidateLimit(maxEmailsPerProduct);
1130 const warnings: string[] = [];
1131 const pageUrls: string[] = [];
1132 const addPageUrl = (url: string): void => {
1133 const canonical = canonicalWebsiteUrl(url);
1134 if (canonical && !pageUrls.includes(canonical) && pageUrls.length < maxWebsitePages) pageUrls.push(canonical);
1135 };
1136 let canonicalOrigin = initialUrl.origin;
1137 let baseDomain = initialUrl.hostname.replace(/^www\./i, '');
1138 let emails: string[] = [];
1139 let coverage: Coverage = 'complete';
1140 let pagePlan: WebsitePage[] = [];
1141
1142 const staticHomepage = await fetchPublicHtml(`${initialUrl.origin}/`, 4_000);
1143 if (staticHomepage.ok) {
1144 const finalHomepage = new URL(staticHomepage.url);
1145 canonicalOrigin = finalHomepage.origin;
1146 baseDomain = finalHomepage.hostname.replace(/^www\./i, '');
1147 addPageUrl(staticHomepage.url);
1148 emails = rankEmails(extractStaticEmails(staticHomepage.html), baseDomain, maxEmailCandidates);
1149 pagePlan = buildWebsitePagePlan(`${canonicalOrigin}/`, extractStaticFallbackPages(staticHomepage.html, canonicalOrigin), maxWebsitePages);
1150 for (const target of pagePlan.slice(1)) {
1151 if (emails.length >= maxEmailCandidates) break;
1152 const staticPage = await fetchPublicHtml(target.url, 3_000);
1153 if (!staticPage.ok) {
1154 coverage = 'partial';
1155 warnings.push(`${target.kind} static assessment failed: ${staticPage.reason}.`);
1156 continue;
1157 }
1158 const finalPage = new URL(staticPage.url);
1159 if (finalPage.origin !== canonicalOrigin) {
1160 coverage = 'partial';
1161 warnings.push(`${target.kind} redirected off the verified homepage origin; it was not extracted.`);
1162 continue;
1163 }
1164 addPageUrl(staticPage.url);
1165 emails = rankEmails([...emails, ...extractStaticEmails(staticPage.html)], baseDomain, maxEmailCandidates);
1166 }
1167 } else {
1168 coverage = 'partial';
1169 warnings.push(`Homepage static assessment failed: ${staticHomepage.reason}.`);
1170 }
1171
1172 if (emails.length < maxEmailCandidates) await withBrowserContextSlot(async () => {
1173 const context = await (await ensureBrowser()).newContext({
1174 userAgent: WEBSITE_USER_AGENT,
1175 locale: 'en-US',
1176 viewport: { width: 1280, height: 800 },
1177 extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' },
1178 });
1179 await context.route('**/*', async (route) => {
1180 if (['image', 'media', 'font'].includes(route.request().resourceType())) return route.abort();
1181 const requestUrl = route.request().url();
1182 if (!/^(?:data|blob|about):/i.test(requestUrl) && !await isSafePublicNetworkUrl(requestUrl)) return route.abort();
1183 return route.continue();
1184 });
1185 try {
1186 const page = await context.newPage();
1187 const response = await page.goto(`${canonicalOrigin}/`, { waitUntil: 'domcontentloaded', timeout: 6_000 });
1188 const finalHomepage = new URL(page.url());
1189 if (!response || finalHomepage.origin !== canonicalOrigin) {
1190 coverage = 'partial';
1191 warnings.push('Homepage DOM assessment redirected off origin or did not return a response.');
1192 } else {
1193 addPageUrl(finalHomepage.toString());
1194 await page.locator('body').waitFor({ state: 'attached', timeout: 750 }).catch(() => undefined);
1195 emails = rankEmails([...emails, ...await page.evaluate(browserExtractEmails)], baseDomain, maxEmailCandidates);
1196 const browserFallbackPages = await page.evaluate(browserExtractLinkedFallbackPages, canonicalOrigin);
1197 pagePlan = buildWebsitePagePlan(
1198 `${canonicalOrigin}/`,
1199 [...pagePlan.slice(1), ...browserFallbackPages],
1200 maxWebsitePages,
1201 );
1202 for (const target of pagePlan.slice(1)) {
1203 if (emails.length >= maxEmailCandidates) break;
1204 try {
1205 const fallbackResponse = await page.goto(target.url, { waitUntil: 'domcontentloaded', timeout: 4_500 });
1206 const finalFallback = new URL(page.url());
1207 if (!fallbackResponse || finalFallback.origin !== canonicalOrigin) {
1208 coverage = 'partial';
1209 warnings.push(`${target.kind} DOM assessment redirected off origin.`);
1210 continue;
1211 }
1212 addPageUrl(finalFallback.toString());
1213 await page.locator('body').waitFor({ state: 'attached', timeout: 750 }).catch(() => undefined);
1214 emails = rankEmails([...emails, ...await page.evaluate(browserExtractEmails)], baseDomain, maxEmailCandidates);
1215 } catch (error) {
1216 coverage = 'partial';
1217 const message = error instanceof Error ? error.message : 'navigation error';
1218 warnings.push(`${target.kind} DOM assessment failed: ${message.slice(0, 90)}.`);
1219 }
1220 }
1221 }
1222 } catch (error) {
1223 coverage = 'partial';
1224 const message = error instanceof Error ? error.message : 'browser error';
1225 warnings.push(`Homepage DOM assessment failed: ${message.slice(0, 90)}.`);
1226 } finally {
1227 await context.close();
1228 }
1229 });
1230
1231 const result: EmailResult = emails.length > 0
1232 ? 'emails_found'
1233 : pageUrls.length > 0 && coverage === 'complete'
1234 ? 'no_public_email_found'
1235 : 'not_assessed';
1236 return {
1237 emails,
1238 emailSource: emails.length > 0 ? 'page_scrape' : 'none',
1239 result,
1240 coverage: pageUrls.length === 0 ? 'none' : coverage,
1241 terminalReason: emails.length >= maxEmailCandidates
1242 ? 'max_public_email_candidates_reached'
1243 : result === 'no_public_email_found'
1244 ? 'eligible_pages_exhausted'
1245 : 'eligible_pages_partially_assessed',
1246 pagesVisited: pageUrls.length,
1247 pageUrls,
1248 warnings,
1249 };
1250}
1251
1252async function findMakerHomepageEmails(
1253 makerWebsiteUrls: string[],
1254 productWebsiteEmails: string[],
1255 maxEmailsPerProduct: number,
1256 filledEmailSlots: number,
1257): Promise<{ emails: string[]; pageUrls: string[]; warnings: string[] }> {
1258 const candidates: string[] = [];
1259 const pageUrls: string[] = [];
1260 const warnings: string[] = [];
1261 const seenOrigins = new Set<string>();
1262
1263 for (const rawUrl of makerWebsiteUrls) {
1264 if (selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots).length
1265 >= Math.max(0, maxEmailsPerProduct - filledEmailSlots)) break;
1266 const websiteUrl = canonicalWebsiteUrl(rawUrl);
1267 if (!websiteUrl || !isAllowedExternalWebsite(websiteUrl)) continue;
1268 const origin = new URL(websiteUrl).origin;
1269 if (seenOrigins.has(origin) || seenOrigins.size >= 2) continue;
1270 seenOrigins.add(origin);
1271
1272 const staticPage = await fetchPublicHtml(`${origin}/`, 4_000);
1273 if (staticPage.ok) {
1274 pageUrls.push(staticPage.url);
1275 const emailDomain = new URL(staticPage.url).hostname.replace(/^www\./i, '');
1276 candidates.push(...rankEmails(extractStaticEmails(staticPage.html), emailDomain, maxEmailsPerProduct));
1277 if (selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots).length > 0) continue;
1278 } else {
1279 warnings.push(`Maker homepage static assessment failed: ${staticPage.reason}.`);
1280 }
1281
1282 await withBrowserContextSlot(async () => {
1283 const context = await (await ensureBrowser()).newContext({
1284 userAgent: WEBSITE_USER_AGENT,
1285 locale: 'en-US',
1286 viewport: { width: 1280, height: 800 },
1287 extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' },
1288 });
1289 await context.route('**/*', async (route) => {
1290 if (['image', 'media', 'font'].includes(route.request().resourceType())) return route.abort();
1291 const requestUrl = route.request().url();
1292 if (!/^(?:data|blob|about):/i.test(requestUrl) && !await isSafePublicNetworkUrl(requestUrl)) return route.abort();
1293 return route.continue();
1294 });
1295 try {
1296 const page = await context.newPage();
1297 const response = await page.goto(`${origin}/`, { waitUntil: 'domcontentloaded', timeout: 10_000 });
1298 if (!response || response.status() >= 400 || new URL(page.url()).origin !== origin) {
1299 warnings.push('Maker homepage browser assessment did not reach the approved origin.');
1300 return;
1301 }
1302 if (!pageUrls.includes(page.url())) pageUrls.push(page.url());
1303 const emailDomain = new URL(page.url()).hostname.replace(/^www\./i, '');
1304 candidates.push(...rankEmails(await page.evaluate(browserExtractEmails), emailDomain, maxEmailsPerProduct));
1305 } catch {
1306 warnings.push('Maker homepage browser assessment failed.');
1307 } finally {
1308 await context.close();
1309 }
1310 });
1311 }
1312
1313 return {
1314 emails: selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots),
1315 pageUrls,
1316 warnings,
1317 };
1318}
1319
1320const POST_FIELDS = `
1321 id name tagline description votesCount commentsCount url website slug
1322 featuredAt createdAt dailyRank weeklyRank monthlyRank yearlyRank
1323 reviewsCount reviewsRating
1324 thumbnail { url }
1325 productLinks { type url }
1326 topics(first: 10) { edges { node { name } } }
1327 makers { id name username twitterUsername url websiteUrl }
1328`;
1329
1330const POSTS_QUERY = `query GetPosts($first:Int $after:String $order:PostsOrder $featured:Boolean $postedAfter:DateTime $postedBefore:DateTime $topic:String) {
1331 posts(first:$first after:$after order:$order featured:$featured postedAfter:$postedAfter postedBefore:$postedBefore topic:$topic) {
1332 edges { node { ${POST_FIELDS} } }
1333 pageInfo { endCursor hasNextPage }
1334 }
1335}`;
1336
1337const POSTS_INDEX_QUERY = `query GetPostIndex($first:Int $after:String $order:PostsOrder $featured:Boolean $postedAfter:DateTime $postedBefore:DateTime) {
1338 posts(first:$first after:$after order:$order featured:$featured postedAfter:$postedAfter postedBefore:$postedBefore) {
1339 edges { node { id url slug } }
1340 pageInfo { endCursor hasNextPage }
1341 }
1342}`;
1343
1344const POST_BY_SLUG_QUERY = `query GetPost($slug:String!) { post(slug:$slug) { ${POST_FIELDS} } }`;
1345
1346function toInt(value: unknown): number | null {
1347 if (value === null || value === undefined || value === '') return null;
1348 const parsed = Number.parseInt(String(value), 10);
1349 return Number.isNaN(parsed) ? null : parsed;
1350}
1351
1352function toFloat(value: unknown): number | null {
1353 if (value === null || value === undefined || value === '') return null;
1354 const parsed = Number.parseFloat(String(value));
1355 return Number.isNaN(parsed) ? null : parsed;
1356}
1357
1358function nodeToRecord(node: Record<string, any>, rank: number): Record<string, unknown> {
1359 const makers = (node.makers ?? []).map((maker: Record<string, unknown>) => ({
1360 maker_name: maker.name ?? null,
1361 maker_id: maker.username ?? null,
1362 maker_ph_id: maker.id ?? null,
1363 twitter_url: maker.twitterUsername ? `https://twitter.com/${String(maker.twitterUsername)}` : null,
1364 maker_url: maker.url ?? null,
1365 website_url: maker.websiteUrl ?? null,
1366 }));
1367 const topics = (node.topics?.edges ?? []).map((edge: any) => edge?.node?.name).filter(Boolean);
1368 const productLinks = Array.isArray(node.productLinks) ? node.productLinks : [];
1369 const directWebsite = productLinks.find((link: Record<string, unknown>) => {
1370 const type = String(link.type ?? '').toLowerCase();
1371 return type === 'website' && typeof link.url === 'string' && isAllowedExternalWebsite(link.url);
1372 })?.url as string | undefined;
1373
1374 const websiteUrl = directWebsite ? canonicalWebsiteUrl(directWebsite) : null;
1375 return {
1376 product_name: node.name ? String(node.name).trim() : null,
1377 tagline: node.tagline ?? null,
1378 description: node.description ?? null,
1379 upvote_count: toInt(node.votesCount),
1380 comment_count: toInt(node.commentsCount),
1381 reviews_count: toInt(node.reviewsCount),
1382 reviews_rating: toFloat(node.reviewsRating),
1383 daily_rank: toInt(node.dailyRank) ?? rank,
1384 weekly_rank: toInt(node.weeklyRank),
1385 monthly_rank: toInt(node.monthlyRank),
1386 yearly_rank: toInt(node.yearlyRank),
1387 launch_date: node.featuredAt ? isoDate(new Date(node.featuredAt)) : node.createdAt ? isoDate(new Date(node.createdAt)) : null,
1388 product_hunt_url: node.url ?? (node.slug ? `https://www.producthunt.com/posts/${node.slug}` : null),
1389 product_hunt_website_url: node.website ?? null,
1390 website_url: websiteUrl,
1391 website_resolution_source: websiteUrl ? 'api_website_product_link' satisfies WebsiteResolutionSource : 'none' satisfies WebsiteResolutionSource,
1392 website_resolution_status: websiteUrl
1393 ? 'verified_external' satisfies WebsiteResolutionStatus
1394 : node.website
1395 ? 'unresolved' satisfies WebsiteResolutionStatus
1396 : 'not_available' satisfies WebsiteResolutionStatus,
1397 website_resolution_confidence: websiteUrl ? 100 : 0,
1398 website_resolution_signals: websiteUrl ? ['official_api_product_link'] : [],
1399 website_resolution_provider_cost_usd: 0,
1400 company_domain: websiteUrl ? companyDomainFromWebsite(websiteUrl) : null,
1401 topics,
1402 thumbnail_url: node.thumbnail?.url ?? null,
1403 makers,
1404 featured: Boolean(node.featuredAt),
1405 emails: [] as string[],
1406 verified_emails: [] as string[],
1407 risky_emails: [] as string[],
1408 rejected_emails: [] as string[],
1409 rejected_email_verifications: [] as EmailVerification[],
1410 email_verifications: [] as EmailVerification[],
1411 email_verification_requested: false,
1412 email_source: 'none' as EmailSource,
1413 email_result: 'not_assessed' as EmailResult,
1414 email_coverage: 'none' as Coverage,
1415 email_terminal_reason: 'website_not_available',
1416 email_pages_visited: 0,
1417 email_page_urls: [] as string[],
1418 maker_email_pages_visited: 0,
1419 maker_email_page_urls: [] as string[],
1420 contact_discovery_requested: false,
1421 contact_discovery_attempts: 0,
1422 contact_discovery_credits: 0,
1423 lead_status: 'unresolved_website',
1424 lead_score: 0,
1425 lead_tier: 'unqualified',
1426 lead_signals: [] as string[],
1427 enrichment_warnings: [] as string[],
1428 scraped_at: new Date().toISOString(),
1429 };
1430}
1431
1432const LEAN_FIELDS = new Set([
1433 'product_name', 'tagline', 'website_url', 'company_domain', 'product_hunt_website_url', 'website_resolution_status',
1434 'website_resolution_source', 'website_resolution_confidence', 'upvote_count', 'daily_rank', 'emails', 'verified_emails',
1435 'risky_emails', 'rejected_emails', 'email_verifications', 'rejected_email_verifications', 'email_verification_requested', 'email_source', 'email_result', 'email_coverage',
1436 'email_pages_visited', 'contact_discovery_attempts', 'lead_status', 'lead_score', 'lead_tier', 'lead_signals',
1437 'maker_email_pages_visited',
1438 'topics', 'launch_date', 'product_hunt_url', 'featured', 'scraped_at',
1439]);
1440
1441function applyOutputMode(record: Record<string, unknown>, outputMode: 'full' | 'lean' | 'leads'): Record<string, unknown> | null {
1442 if (outputMode === 'leads' && (!Array.isArray(record.emails) || record.emails.length === 0)) return null;
1443 if (outputMode !== 'lean') return record;
1444 const output: Record<string, unknown> = {};
1445 for (const field of LEAN_FIELDS) output[field] = record[field] ?? null;
1446 return output;
1447}
1448
1449function validateOutputRecord(record: Record<string, unknown>): string | null {
1450 if (typeof record.product_name !== 'string' || !record.product_name.trim()) return 'product_name is missing.';
1451 if (typeof record.product_hunt_url !== 'string' || !record.product_hunt_url.startsWith('https://')) return 'product_hunt_url is missing or invalid.';
1452 if (!Array.isArray(record.emails) || !record.emails.every((email) => typeof email === 'string')) return 'emails must be a string array.';
1453 if (typeof record.scraped_at !== 'string' || Number.isNaN(Date.parse(record.scraped_at))) return 'scraped_at is invalid.';
1454 return null;
1455}
1456
1457async function run(): Promise<void> {
1458 const rawInput = (await Actor.getInput() ?? {}) as Record<string, unknown>;
1459 const mode = (rawInput.mode ?? rawInput.scrapeMode ?? 'leaderboard') as Mode;
1460 const leaderboardPeriod = (rawInput.leaderboardPeriod ?? rawInput.period ?? 'daily') as LeaderboardPeriod;
1461 const startDate = String(rawInput.startDate ?? rawInput.date ?? '');
1462 const endDate = String(rawInput.endDate ?? '');
1463 const lookbackDays = clampInteger(rawInput.lookbackDays, 1, 1, 365);
1464 const outputMode = (rawInput.outputMode ?? 'full') as 'full' | 'lean' | 'leads';
1465 const searchQuery = typeof (rawInput.searchQuery ?? rawInput.query ?? rawInput.q) === 'string'
1466 ? String(rawInput.searchQuery ?? rawInput.query ?? rawInput.q).trim()
1467 : '';
1468 const topic = typeof (rawInput.topic ?? rawInput.topicSlug) === 'string'
1469 ? String(rawInput.topic ?? rawInput.topicSlug).trim()
1470 : '';
1471 const startUrls = Array.isArray(rawInput.startUrls) ? rawInput.startUrls as Array<{ url?: unknown } | string> : [];
1472 const maxResults = clampInteger(rawInput.maxResults ?? rawInput.maxItems ?? rawInput.limit, 100, 1, 20_000);
1473 const maxConcurrency = clampInteger(rawInput.maxConcurrency, 3, 1, 5);
1474 const maxWebsitePages = clampInteger(rawInput.maxWebsitePages, DEFAULT_MAX_WEBSITE_PAGES, 1, MAX_WEBSITE_PAGES);
1475 const maxEmailsPerProduct = clampInteger(rawInput.maxEmailsPerProduct, DEFAULT_MAX_EMAILS_PER_PRODUCT, 1, MAX_EMAILS_PER_PRODUCT);
1476 const includeAllProducts = Boolean(rawInput.includeAllProducts ?? rawInput.allProducts ?? rawInput.includeAll ?? true);
1477 const enrichEmails = Boolean(rawInput.enrichEmails ?? rawInput.findEmails ?? true);
1478 const resolveWebsites = Boolean(rawInput.resolveWebsites ?? true);
1479 const verifyEmails = Boolean(rawInput.verifyEmails ?? rawInput.verifyWithEmailListVerify ?? true);
1480 const findContacts = Boolean(rawInput.findContacts ?? true);
1481 const emailListVerifyApiKey = String(process.env.EMAILLISTVERIFY_API_KEY ?? '').trim();
1482 const phApiToken = String(process.env.PH_API_TOKEN ?? '');
1483 const dataForSeoLogin = String(process.env.DATAFORSEO_LOGIN ?? '').trim();
1484 const dataForSeoPassword = String(process.env.DATAFORSEO_PASSWORD ?? '').trim();
1485 const serpApiKey = String(process.env.SERPAPI_KEY ?? '').trim();
1486 const websiteOverrides = parseWebsiteOverrides(rawInput.websiteUrlOverrides);
1487 const excludedProductHuntUrls = Array.isArray(rawInput.excludeProductHuntUrls)
1488 ? rawInput.excludeProductHuntUrls
1489 .slice(0, 20_000)
1490 .map((value) => productHuntLaunchKey(String(value ?? '')))
1491 .filter((value): value is string => Boolean(value))
1492 : [];
1493 const excludedProductHuntKeys = new Set(excludedProductHuntUrls);
1494 const productHuntResumeCursor = typeof rawInput.productHuntResumeCursor === 'string'
1495 ? rawInput.productHuntResumeCursor.trim().slice(0, 2_000)
1496 : '';
1497 const startedAt = new Date().toISOString();
1498 emailListVerifyCreditsExhausted = false;
1499
1500 const diagnostics = {
1501 itemsPushed: 0,
1502 itemsProcessed: 0,
1503 itemsFiltered: 0,
1504 itemsExcluded: 0,
1505 itemsFailed: 0,
1506 invalidRecords: 0,
1507 apiFailures: 0,
1508 productHuntIndexPages: 0,
1509 productHuntDetailLookups: 0,
1510 productHuntFullPageLookups: 0,
1511 nextProductHuntCursor: null as string | null,
1512 websiteResolved: 0,
1513 websiteUnresolved: 0,
1514 websiteResolutionProviderAttempts: 0,
1515 websiteResolutionProviderCostUsd: 0,
1516 emailsFound: 0,
1517 verifiedEmailsFound: 0,
1518 emailVerificationAttempts: 0,
1519 emailVerificationProviderErrors: 0,
1520 makerEmailPagesVisited: 0,
1521 makerEmailsFound: 0,
1522 contactDiscoveryAttempts: 0,
1523 contactDiscoveryCredits: 0,
1524 contactDiscoveryEmailsFound: 0,
1525 leadTiers: { A: 0, B: 0, C: 0, unqualified: 0 },
1526 resultEvents: 0,
1527 emailEvents: 0,
1528 chargeLimitReached: false,
1529 resultLimitReached: false,
1530 deadlineReached: false,
1531 itemsSkippedAtDeadline: 0,
1532 searchScanCapReached: false,
1533 emailVerificationCreditsExhausted: false,
1534 warnings: [] as string[],
1535 };
1536 let finalised = false;
1537
1538 const paidItemLimit = clampInteger(process.env.ACTOR_MAX_PAID_DATASET_ITEMS, maxResults, 1, maxResults);
1539 const effectiveMaxResults = Math.min(maxResults, paidItemLimit);
1540
1541 const runDeadlineMs = Date.parse(process.env.ACTOR_TIMEOUT_AT ?? '');
1542 const hasRunDeadline = Number.isFinite(runDeadlineMs);
1543 const runBudgetMs = hasRunDeadline ? Math.max(0, runDeadlineMs - Date.now()) : Number.POSITIVE_INFINITY;
1544
1545 const newWorkReserveMs = hasRunDeadline ? Math.min(Math.max(runBudgetMs * 0.15, 45_000), 480_000) : 0;
1546 const stopNewWorkAtMs = hasRunDeadline ? runDeadlineMs - newWorkReserveMs : Number.POSITIVE_INFINITY;
1547 const hardStopAtMs = hasRunDeadline ? runDeadlineMs - 20_000 : Number.POSITIVE_INFINITY;
1548 let hardStopping = false;
1549 let inFlightWrites = 0;
1550 let currentPageStartCursor: string | null = null;
1551 let lastProgressAt = 0;
1552
1553 const addWarning = (message: string): void => {
1554 if (diagnostics.warnings.length < 100 && !diagnostics.warnings.includes(message)) diagnostics.warnings.push(message);
1555 };
1556
1557 const deadlineReached = (): boolean => {
1558 if (Date.now() < stopNewWorkAtMs) return false;
1559 if (!diagnostics.deadlineReached) {
1560 diagnostics.deadlineReached = true;
1561 addWarning('The run deadline approached, so no new products were started. Continue with productHuntResumeCursor from RUN_SUMMARY.nextProductHuntCursor (when present) and excludeProductHuntUrls from this dataset to avoid duplicate charges.');
1562 log.warning('Run deadline approaching; stopped starting new products and preserving persisted results.');
1563 }
1564 return true;
1565 };
1566
1567 const reportProgress = async (force = false): Promise<void> => {
1568 if (!force && Date.now() - lastProgressAt < 15_000) return;
1569 lastProgressAt = Date.now();
1570 await Actor.setStatusMessage(
1571 `Processed ${diagnostics.itemsProcessed}/${effectiveMaxResults} products, saved ${diagnostics.itemsPushed} rows, found ${diagnostics.emailsFound} emails.`
1572 + (emailListVerifyCreditsExhausted ? ' Email verification credits are exhausted; remaining emails are unverified.' : ''),
1573 ).catch(() => undefined);
1574 };
1575
1576 const resolveOutcome = (): RunOutcome => {
1577 diagnostics.emailVerificationCreditsExhausted = emailListVerifyCreditsExhausted;
1578 if (diagnostics.itemsPushed === 0) {
1579 const searchedWithoutMatch = mode === 'search' && (diagnostics.searchScanCapReached || diagnostics.deadlineReached);
1580 return diagnostics.apiFailures > 0 || diagnostics.itemsFailed > 0 || diagnostics.invalidRecords > 0 || (diagnostics.deadlineReached && !searchedWithoutMatch)
1581 ? 'UPSTREAM_FAILED'
1582 : 'VALID_EMPTY';
1583 }
1584 return diagnostics.apiFailures > 0 || diagnostics.itemsFailed > 0 || (resolveWebsites && diagnostics.websiteUnresolved > 0) || diagnostics.chargeLimitReached
1585 || diagnostics.resultLimitReached || diagnostics.deadlineReached || diagnostics.searchScanCapReached || (verifyEmails && emailListVerifyCreditsExhausted)
1586 ? 'PARTIAL'
1587 : 'COMPLETE';
1588 };
1589
1590 const outcomeMessage = (outcome: RunOutcome): string => {
1591 const eventCost = diagnostics.resultEvents * RESULT_PRICE_USD + diagnostics.emailEvents * EMAIL_PRICE_USD;
1592 const deadlineNote = diagnostics.deadlineReached ? ' Stopped before the run deadline; resume details are in RUN_SUMMARY.' : '';
1593 if (diagnostics.itemsPushed === 0 && mode === 'search' && (diagnostics.searchScanCapReached || diagnostics.deadlineReached)) {
1594 return diagnostics.searchScanCapReached
1595 ? `${outcome}: no Product Hunt launch matched "${searchQuery}" in the newest ${SEARCH_PAGES_WITHOUT_MATCH_LIMIT * 20} launches searched.`
1596 : `${outcome}: no Product Hunt launch matched "${searchQuery}" before the run deadline.`;
1597 }
1598 if (diagnostics.itemsPushed === 0 && diagnostics.deadlineReached) {
1599 return `${outcome}: the run deadline was reached before any Product Hunt product could be persisted. Increase the run timeout and re-run.`;
1600 }
1601 return `${outcome}: processed ${diagnostics.itemsProcessed} Product Hunt products, persisted ${diagnostics.itemsPushed} rows, and found ${diagnostics.emailsFound} public email(s). Event charges: $${eventCost.toFixed(3)} plus applicable platform usage.${deadlineNote}`;
1602 };
1603
1604 const finalizeRun = async (outcome: RunOutcome, message: string): Promise<never> => {
1605 if (finalised) throw new Error(message);
1606 finalised = true;
1607 if (!message.startsWith(outcome)) message = `${outcome}: ${message}`;
1608 const finishedAt = new Date().toISOString();
1609 const summary = {
1610 outcome,
1611 message,
1612 startedAt,
1613 finishedAt,
1614 buildNumber: process.env.ACTOR_BUILD_NUMBER ?? null,
1615 runId: process.env.ACTOR_RUN_ID ?? null,
1616 datasetId: process.env.ACTOR_DEFAULT_DATASET_ID ?? null,
1617 keyValueStoreId: process.env.ACTOR_DEFAULT_KEY_VALUE_STORE_ID ?? null,
1618 input: {
1619 mode, leaderboardPeriod, startDate: startDate || null, endDate: endDate || null,
1620 lookbackDays, maxResults, effectiveMaxResults, includeAllProducts, enrichEmails, resolveWebsites, verifyEmails, findContacts,
1621 excludedProductHuntUrls: excludedProductHuntKeys.size,
1622 productHuntResumeCursorUsed: Boolean(productHuntResumeCursor),
1623 emailListVerifyConfigured: Boolean(emailListVerifyApiKey), maxWebsitePages,
1624 websiteSearchConfigured: Boolean((dataForSeoLogin && dataForSeoPassword) || serpApiKey),
1625 maxEmailsPerProduct, outputMode,
1626 },
1627 ...diagnostics,
1628 chargedEventCounts: {
1629 [RESULT_EVENT]: diagnostics.resultEvents,
1630 [EMAIL_EVENT]: diagnostics.emailEvents,
1631 },
1632 };
1633 const output = {
1634 outcome,
1635 message,
1636 itemsPushed: diagnostics.itemsPushed,
1637 itemsFailed: diagnostics.itemsFailed,
1638 invalidRecords: diagnostics.invalidRecords,
1639 deadlineReached: diagnostics.deadlineReached,
1640 nextProductHuntCursor: diagnostics.nextProductHuntCursor,
1641 emailVerificationCreditsExhausted: diagnostics.emailVerificationCreditsExhausted,
1642 chargedEventCounts: summary.chargedEventCounts,
1643 warnings: diagnostics.warnings,
1644 };
1645 await Actor.setValue('OUTPUT', output);
1646 await Actor.setValue('RUN_SUMMARY', summary);
1647 const failed = FAILED_OUTCOMES.has(outcome);
1648 await Actor.setStatusMessage(message, {
1649 isStatusMessageTerminal: true,
1650 level: failed ? 'ERROR' : outcome === 'COMPLETE' ? 'INFO' : 'WARNING',
1651 });
1652 if (failed) {
1653
1654 await Actor.fail(message);
1655 }
1656 await Actor.exit();
1657 throw new Error('Actor terminal call returned unexpectedly.');
1658 };
1659
1660 try {
1661 const modes: Mode[] = ['leaderboard', 'search', 'topic', 'urls'];
1662 const periods: LeaderboardPeriod[] = ['daily', 'weekly', 'monthly', 'yearly'];
1663 const outputModes = ['full', 'lean', 'leads'];
1664 if (!modes.includes(mode)) await finalizeRun('INVALID_INPUT', 'mode must be leaderboard, search, topic, or urls.');
1665 if (!periods.includes(leaderboardPeriod)) await finalizeRun('INVALID_INPUT', 'leaderboardPeriod must be daily, weekly, monthly, or yearly.');
1666 if (!outputModes.includes(outputMode)) await finalizeRun('INVALID_INPUT', 'outputMode must be full, lean, or leads.');
1667 if (!phApiToken) await finalizeRun('CONFIG_ERROR', 'Product Hunt API token is not configured. Set the owner-managed PH_API_TOKEN secret and re-run.');
1668 if (mode === 'search' && !searchQuery) await finalizeRun('INVALID_INPUT', 'searchQuery is required for search mode.');
1669 if (mode === 'topic' && !topic) await finalizeRun('INVALID_INPUT', 'topic is required for topic mode.');
1670 if (mode === 'urls' && startUrls.length === 0) await finalizeRun('INVALID_INPUT', 'startUrls is required for urls mode.');
1671 if (startDate && !validIsoDate(startDate)) await finalizeRun('INVALID_INPUT', 'startDate must be a valid YYYY-MM-DD date.');
1672 if (endDate && !validIsoDate(endDate)) await finalizeRun('INVALID_INPUT', 'endDate must be a valid YYYY-MM-DD date.');
1673 if (startDate && endDate && startDate > endDate) await finalizeRun('INVALID_INPUT', 'endDate must be on or after startDate.');
1674 if (leaderboardPeriod !== 'daily' && endDate) await finalizeRun('INVALID_INPUT', 'endDate is supported only with daily leaderboard ranges.');
1675
1676 const maxPotentialEventCost = effectiveMaxResults * (RESULT_PRICE_USD + (enrichEmails ? maxEmailsPerProduct * EMAIL_PRICE_USD : 0));
1677 const maxPotentialVerificationCredits = enrichEmails && emailListVerifyApiKey
1678 ? effectiveMaxResults * ((verifyEmails ? maxEmailsPerProduct * 3 * 2 : 0) + (findContacts ? 15 : 0))
1679 : 0;
1680 log.info('ProductHunt Scraper starting', {
1681 mode, leaderboardPeriod, startDate: startDate || '(default)', endDate: endDate || null,
1682 lookbackDays, effectiveMaxResults, includeAllProducts, enrichEmails, resolveWebsites, verifyEmails, findContacts,
1683 emailListVerifyConfigured: Boolean(emailListVerifyApiKey), maxWebsitePages, maxEmailsPerProduct,
1684 websiteSearchProviders: {
1685 dataForSeo: Boolean(dataForSeoLogin && dataForSeoPassword),
1686 serpApiFallback: Boolean(serpApiKey),
1687 },
1688 eventCostCeilingUsd: Number(maxPotentialEventCost.toFixed(3)),
1689 excludedProductHuntUrls: excludedProductHuntKeys.size,
1690 emailListVerifyCreditCeiling: maxPotentialVerificationCredits,
1691 websiteSearchQueryCeiling: resolveWebsites
1692 ? effectiveMaxResults * ((dataForSeoLogin && dataForSeoPassword ? 2 : 0) + (serpApiKey ? 1 : 0))
1693 : 0,
1694 platformUsage: 'Additional compute and proxy usage is billed by Apify separately where applicable.',
1695 });
1696 await Actor.setStatusMessage(`Starting: up to ${effectiveMaxResults} products, ${maxWebsitePages} public website pages/product, and ${maxEmailsPerProduct} public emails/product.`);
1697 lastProgressAt = Date.now();
1698
1699 if (hasRunDeadline) {
1700 const hardStopTimer = setInterval(() => {
1701 if (Date.now() < hardStopAtMs || hardStopping || finalised) return;
1702 hardStopping = true;
1703 clearInterval(hardStopTimer);
1704 void (async () => {
1705 deadlineReached();
1706 const waitUntil = Date.now() + 10_000;
1707 while (inFlightWrites > 0 && Date.now() < waitUntil) await new Promise((resolve) => setTimeout(resolve, 200));
1708 if (currentPageStartCursor && !diagnostics.nextProductHuntCursor) diagnostics.nextProductHuntCursor = currentPageStartCursor;
1709 await closeBrowser();
1710 const outcome = resolveOutcome();
1711 await finalizeRun(outcome, outcomeMessage(outcome)).catch(() => undefined);
1712 })();
1713 }, 5_000);
1714 hardStopTimer.unref();
1715 }
1716
1717 const phQuery = async (query: string, variables: Record<string, unknown>, attempt = 0): Promise<any> => {
1718 const response = await fetch(PH_API, {
1719 method: 'POST',
1720 headers: {
1721 'Content-Type': 'application/json',
1722 Accept: 'application/json',
1723 Authorization: `Bearer ${phApiToken}`,
1724 'User-Agent': PRODUCT_HUNT_USER_AGENT,
1725 },
1726 body: JSON.stringify({ query, variables }),
1727 signal: AbortSignal.timeout(30_000),
1728 });
1729 if (response.status === 429 && attempt < 3) {
1730 const waitSeconds = productHuntRateLimitDelaySeconds(
1731 response.headers.get('x-rate-limit-reset'),
1732 response.headers.get('retry-after'),
1733 );
1734 log.warning(`Product Hunt API rate limited; waiting ${waitSeconds}s before retry ${attempt + 1}/3.`);
1735 await new Promise((resolve) => setTimeout(resolve, waitSeconds * 1000));
1736 return phQuery(query, variables, attempt + 1);
1737 }
1738 if (!response.ok) throw new Error(`Product Hunt API returned HTTP ${response.status}.`);
1739 const payload = await response.json() as { data?: unknown; errors?: Array<{ message?: string }> };
1740 if (payload.errors?.length) throw new Error(`Product Hunt GraphQL: ${payload.errors[0]?.message ?? 'unknown error'}`);
1741 return payload.data as any;
1742 };
1743
1744 const seenIds = new Set<string>();
1745 let collectedCandidates = 0;
1746
1747 const pushNode = async (node: Record<string, any>, rank: number): Promise<void> => {
1748 const record = nodeToRecord(node, rank);
1749 record.email_verification_requested = verifyEmails;
1750 const override = lookupWebsiteOverride(node, websiteOverrides);
1751 if (override) {
1752 record.website_url = override;
1753 record.website_resolution_source = 'input_override' satisfies WebsiteResolutionSource;
1754 record.website_resolution_status = 'verified_external' satisfies WebsiteResolutionStatus;
1755 record.website_resolution_confidence = 100;
1756 record.website_resolution_signals = ['approved_input_override'];
1757 }
1758
1759 let websiteUrl = typeof record.website_url === 'string' ? record.website_url : '';
1760 if (!websiteUrl && resolveWebsites) {
1761 const resolution = await resolveWebsite({
1762 productName: String(record.product_name ?? ''),
1763 tagline: String(record.tagline ?? ''),
1764 description: String(record.description ?? ''),
1765 productHuntRedirect: typeof record.product_hunt_website_url === 'string' ? record.product_hunt_website_url : '',
1766 makerWebsiteUrls: (node.makers ?? [])
1767 .map((maker: Record<string, unknown>) => typeof maker.websiteUrl === 'string' ? maker.websiteUrl : '')
1768 .filter(Boolean),
1769 dataForSeoLogin,
1770 dataForSeoPassword,
1771 serpApiKey,
1772 });
1773 diagnostics.websiteResolutionProviderAttempts += resolution.providerAttempts;
1774 diagnostics.websiteResolutionProviderCostUsd += resolution.providerCostUsd;
1775 resolution.warnings.forEach(addWarning);
1776 record.website_resolution_provider_cost_usd = Number(resolution.providerCostUsd.toFixed(6));
1777 if (resolution.websiteUrl) {
1778 websiteUrl = resolution.websiteUrl;
1779 record.website_url = resolution.websiteUrl;
1780 record.website_resolution_source = resolution.source;
1781 record.website_resolution_status = resolution.status;
1782 record.website_resolution_confidence = resolution.confidence;
1783 record.website_resolution_signals = resolution.signals;
1784 }
1785 }
1786
1787 if (websiteUrl) {
1788 diagnostics.websiteResolved += 1;
1789 record.company_domain = companyDomainFromWebsite(websiteUrl);
1790 } else {
1791 diagnostics.websiteUnresolved += 1;
1792 record.website_resolution_status = record.product_hunt_website_url ? 'unresolved' : 'not_available';
1793 }
1794
1795 let publicEmailCount = 0;
1796 let publicVerifications: EmailVerification[] = [];
1797 if (enrichEmails && websiteUrl) {
1798 const enriched = await findEmails(websiteUrl, maxWebsitePages, maxEmailsPerProduct);
1799 record.emails = enriched.emails;
1800 record.email_source = enriched.emailSource;
1801 record.email_result = enriched.result;
1802 record.email_coverage = enriched.coverage;
1803 record.email_terminal_reason = enriched.terminalReason;
1804 record.email_pages_visited = enriched.pagesVisited;
1805 record.email_page_urls = enriched.pageUrls;
1806 record.enrichment_warnings = enriched.warnings;
1807 publicEmailCount = enriched.emails.length;
1808 enriched.warnings.forEach(addWarning);
1809 log.info('Website enrichment complete', {
1810 product: record.product_name,
1811 emailsFound: enriched.emails.length,
1812 source: enriched.emailSource,
1813 pagesVisited: enriched.pagesVisited,
1814 coverage: enriched.coverage,
1815 terminalReason: enriched.terminalReason,
1816 });
1817 }
1818
1819 const selectedEmails = Array.isArray(record.emails) ? record.emails.filter((email): email is string => typeof email === 'string') : [];
1820 if (selectedEmails.length > 0) {
1821 const verification = await verifyEmailsWithEmailListVerify(
1822 selectedEmails,
1823 verifyEmails,
1824 emailListVerifyApiKey,
1825 'public_website',
1826 maxEmailsPerProduct,
1827 );
1828 publicVerifications = verification.verifications;
1829 diagnostics.emailVerificationAttempts += verification.attempts;
1830 diagnostics.emailVerificationProviderErrors += verification.verifications.filter(({ status }) => status === 'provider_error' || status === 'error_credit').length;
1831 verification.warnings.forEach(addWarning);
1832 log.info('Public email verification complete', {
1833 product: record.product_name,
1834 emailsAssessed: selectedEmails.length,
1835 requested: verifyEmails,
1836 attempts: verification.attempts,
1837 statuses: verification.verifications.map(({ status }) => status),
1838 });
1839 }
1840
1841 const makerWebsiteUrls = (node.makers ?? [])
1842 .map((maker: Record<string, unknown>) => typeof maker.websiteUrl === 'string' ? maker.websiteUrl : '')
1843 .filter(Boolean);
1844 const productDeliverableCount = verifyEmails
1845 ? publicVerifications.filter(({ status }) => status === 'ok').length
1846 : publicVerifications.filter(({ status }) => isUsableEmailStatus(status)).length;
1847 if (enrichEmails && productDeliverableCount < maxEmailsPerProduct && makerWebsiteUrls.length > 0) {
1848 const makerEnrichment = await findMakerHomepageEmails(
1849 makerWebsiteUrls,
1850 selectedEmails,
1851 maxEmailsPerProduct,
1852 productDeliverableCount,
1853 );
1854 record.maker_email_pages_visited = makerEnrichment.pageUrls.length;
1855 record.maker_email_page_urls = makerEnrichment.pageUrls;
1856 diagnostics.makerEmailPagesVisited += makerEnrichment.pageUrls.length;
1857 makerEnrichment.warnings.forEach(addWarning);
1858 record.enrichment_warnings = [
1859 ...new Set([
1860 ...(Array.isArray(record.enrichment_warnings) ? record.enrichment_warnings as string[] : []),
1861 ...makerEnrichment.warnings,
1862 ]),
1863 ];
1864 if (makerEnrichment.emails.length > 0) {
1865 const makerVerification = await verifyEmailsWithEmailListVerify(
1866 makerEnrichment.emails,
1867 verifyEmails,
1868 emailListVerifyApiKey,
1869 'maker_website',
1870 );
1871 publicVerifications.push(...makerVerification.verifications);
1872 publicEmailCount += makerEnrichment.emails.length;
1873 diagnostics.makerEmailsFound += makerEnrichment.emails.length;
1874 diagnostics.emailVerificationAttempts += makerVerification.attempts;
1875 diagnostics.emailVerificationProviderErrors += makerVerification.verifications
1876 .filter(({ status }) => status === 'provider_error' || status === 'error_credit').length;
1877 makerVerification.warnings.forEach(addWarning);
1878 log.info('Maker homepage email fallback complete', {
1879 product: record.product_name,
1880 emailsFound: makerEnrichment.emails.length,
1881 pagesVisited: makerEnrichment.pageUrls.length,
1882 statuses: makerVerification.verifications.map(({ status }) => status),
1883 });
1884 }
1885 }
1886
1887 let discoveredContacts: ContactFinderResult[] = [];
1888 const deliverablePublicCount = verifyEmails
1889 ? publicVerifications.filter(({ status }) => status === 'ok').length
1890 : publicVerifications.filter(({ status }) => isUsableEmailStatus(status)).length;
1891 if (enrichEmails && findContacts && websiteUrl && deliverablePublicCount < maxEmailsPerProduct) {
1892 record.contact_discovery_requested = true;
1893 const discovery = await discoverContacts(
1894 websiteUrl,
1895 (node.makers ?? []).map((maker: Record<string, unknown>) => String(maker.name ?? '')).filter(Boolean),
1896 Math.max(1, maxEmailsPerProduct - deliverablePublicCount),
1897 emailListVerifyApiKey,
1898 );
1899 discoveredContacts = discovery.contacts;
1900 record.contact_discovery_attempts = discovery.attempts;
1901 record.contact_discovery_credits = discovery.credits;
1902 diagnostics.contactDiscoveryAttempts += discovery.attempts;
1903 diagnostics.contactDiscoveryCredits += discovery.credits;
1904 diagnostics.contactDiscoveryEmailsFound += discovery.contacts.length;
1905 discovery.warnings.forEach(addWarning);
1906 }
1907
1908 const finalVerifications = selectBestEmails(publicVerifications, discoveredContacts, maxEmailsPerProduct);
1909 const rejectedVerifications = publicVerifications.filter(({ status }) => !isUsableEmailStatus(status));
1910 const finalEmails = finalVerifications.map(({ email }) => email);
1911 const publicSelected = finalVerifications.filter(({ source }) => source === 'public_website').length;
1912 const makerSelected = finalVerifications.filter(({ source }) => source === 'maker_website').length;
1913 const finderSelected = finalVerifications.filter(({ source }) => source === 'emaillistverify_contact_finder').length;
1914 record.emails = finalEmails;
1915 record.verified_emails = finalVerifications.filter(({ status }) => status === 'ok').map(({ email }) => email);
1916 record.risky_emails = finalVerifications.filter(({ status }) => status !== 'ok').map(({ email }) => email);
1917 record.rejected_emails = rejectedVerifications.map(({ email }) => email);
1918 record.email_verifications = finalVerifications;
1919 record.rejected_email_verifications = rejectedVerifications;
1920 const selectedSourceCount = [publicSelected, makerSelected, finderSelected].filter((count) => count > 0).length;
1921 record.email_source = selectedSourceCount > 1
1922 ? 'mixed'
1923 : publicSelected > 0
1924 ? 'page_scrape'
1925 : makerSelected > 0
1926 ? 'maker_website'
1927 : finderSelected > 0
1928 ? 'emaillistverify_contact_finder'
1929 : 'none';
1930 record.email_result = finalEmails.length > 0 ? 'emails_found' : record.email_result;
1931 if (finderSelected > 0) record.email_terminal_reason = 'contact_finder_enriched';
1932 else if (makerSelected > 0) record.email_terminal_reason = 'maker_homepage_enriched';
1933 diagnostics.emailsFound += finalEmails.length;
1934 diagnostics.verifiedEmailsFound += (record.verified_emails as string[]).length;
1935
1936 const lead = buildLeadProfile({
1937 websiteResolved: Boolean(websiteUrl),
1938 websiteConfidence: Number(record.website_resolution_confidence ?? 0),
1939 emailStatuses: finalVerifications.map(({ status }) => status),
1940 publicEmailCount,
1941 upvotes: Number(record.upvote_count ?? 0),
1942 pagesVisited: Number(record.email_pages_visited ?? 0),
1943 });
1944 record.lead_status = lead.leadStatus;
1945 record.lead_score = lead.leadScore;
1946 record.lead_tier = lead.leadTier;
1947 record.lead_signals = lead.signals;
1948 diagnostics.leadTiers[lead.leadTier] += 1;
1949
1950 log.info('Lead enrichment complete', {
1951 product: record.product_name,
1952 websiteResolutionSource: record.website_resolution_source,
1953 websiteResolutionConfidence: record.website_resolution_confidence,
1954 publicEmailsFound: publicEmailCount,
1955 contactFinderEmailsFound: discoveredContacts.length,
1956 finalEmails: finalEmails.length,
1957 verifiedEmails: (record.verified_emails as string[]).length,
1958 leadTier: lead.leadTier,
1959 });
1960
1961 const outputRecord = applyOutputMode(record, outputMode);
1962 if (!outputRecord) {
1963 diagnostics.itemsFiltered += 1;
1964 return;
1965 }
1966 const validationError = validateOutputRecord(outputRecord);
1967 if (validationError) {
1968 diagnostics.invalidRecords += 1;
1969 diagnostics.itemsFailed += 1;
1970 addWarning(`A Product Hunt record was not persisted: ${validationError}`);
1971 return;
1972 }
1973
1974 if (hardStopping) {
1975 diagnostics.itemsSkippedAtDeadline += 1;
1976 return;
1977 }
1978 inFlightWrites += 1;
1979 try {
1980
1981
1982
1983 const primaryResult = await Actor.pushData(outputRecord) as unknown as { chargedCount?: number; eventChargeLimitReached?: boolean };
1984 if (primaryResult.eventChargeLimitReached && (primaryResult.chargedCount ?? 0) === 0) {
1985 diagnostics.chargeLimitReached = true;
1986 addWarning('The event-charge limit was reached before this validated row could be persisted.');
1987 return;
1988 }
1989 diagnostics.itemsPushed += 1;
1990 diagnostics.resultEvents += primaryResult.chargedCount ?? 1;
1991 diagnostics.chargeLimitReached ||= Boolean(primaryResult.eventChargeLimitReached);
1992
1993 const emails = outputRecord.emails as string[];
1994 if (emails.length > 0 && !diagnostics.chargeLimitReached) {
1995 const emailResult = await Actor.charge({ eventName: EMAIL_EVENT, count: emails.length }) as unknown as { chargedCount?: number; eventChargeLimitReached?: boolean };
1996 diagnostics.emailEvents += emailResult.chargedCount ?? emails.length;
1997 diagnostics.chargeLimitReached ||= Boolean(emailResult.eventChargeLimitReached);
1998 if (emailResult.eventChargeLimitReached) addWarning('The event-charge limit was reached after a persisted record; remaining work stopped at the configured cap.');
1999 }
2000 } catch (error) {
2001 diagnostics.itemsFailed += 1;
2002 const message = error instanceof Error ? error.message : 'Unknown dataset-write failure.';
2003 addWarning(`Validated dataset write failed: ${message.slice(0, 120)}`);
2004 log.warning(`Validated dataset write failed for ${String(record.product_name)}: ${message.slice(0, 120)}`);
2005 } finally {
2006 inFlightWrites -= 1;
2007 }
2008 };
2009
2010 const processNodes = async (nodes: Array<Record<string, any>>): Promise<void> => {
2011 const uniqueNodes: Array<Record<string, any>> = [];
2012 for (const node of nodes) {
2013 if (collectedCandidates >= effectiveMaxResults || diagnostics.chargeLimitReached) break;
2014 const id = String(node.id ?? '');
2015 if (!id || seenIds.has(id)) continue;
2016 seenIds.add(id);
2017 const launchKey = productHuntLaunchKey(String(node.url ?? ''));
2018 if (launchKey && excludedProductHuntKeys.has(launchKey)) {
2019 diagnostics.itemsExcluded += 1;
2020 continue;
2021 }
2022 uniqueNodes.push(node);
2023 collectedCandidates += 1;
2024 }
2025 const rankBase = diagnostics.itemsProcessed;
2026 let nextIndex = 0;
2027 const worker = async (): Promise<void> => {
2028 while (nextIndex < uniqueNodes.length && !diagnostics.chargeLimitReached && !hardStopping) {
2029 if (deadlineReached()) break;
2030 const index = nextIndex;
2031 nextIndex += 1;
2032 try {
2033 await pushNode(uniqueNodes[index], rankBase + index + 1);
2034 } catch (error) {
2035 diagnostics.itemsFailed += 1;
2036 const message = error instanceof Error ? error.message : 'Unknown product enrichment failure.';
2037 addWarning(`A Product Hunt product could not be enriched: ${message.slice(0, 120)}`);
2038 log.warning(`Product enrichment failed for ${String(uniqueNodes[index]?.name ?? 'unknown')}: ${message.slice(0, 120)}`);
2039 }
2040 diagnostics.itemsProcessed += 1;
2041 await reportProgress();
2042 }
2043 };
2044 await Promise.all(Array.from({ length: Math.min(maxConcurrency, uniqueNodes.length) }, () => worker()));
2045 };
2046
2047 const streamPosts = async (range: DateRange | null, label: string): Promise<void> => {
2048 let cursor: string | null = productHuntResumeCursor || null;
2049 let pageNumber = 1;
2050 let pagesWithoutSearchMatch = 0;
2051 let useLightweightIndex = shouldUseLightweightProductHuntIndex(
2052 mode,
2053 excludedProductHuntKeys.size,
2054 productHuntResumeCursor,
2055 );
2056 while (collectedCandidates < effectiveMaxResults && !diagnostics.chargeLimitReached) {
2057 if (deadlineReached() || hardStopping) break;
2058 const pageStartCursor = cursor;
2059 currentPageStartCursor = pageStartCursor;
2060 const failuresBeforePage = diagnostics.itemsFailed;
2061 const remaining = effectiveMaxResults - collectedCandidates;
2062 const variables: Record<string, unknown> = {
2063 first: Math.min(20, remaining),
2064 after: cursor,
2065 order: productHuntStreamOrder(Boolean(range)),
2066 };
2067 if (!includeAllProducts) variables.featured = true;
2068 if (range) {
2069 variables.postedAfter = range.postedAfter;
2070 variables.postedBefore = range.postedBefore;
2071 }
2072 if (mode === 'topic') variables.topic = topic;
2073 try {
2074 let data: any = await phQuery(useLightweightIndex ? POSTS_INDEX_QUERY : POSTS_QUERY, variables);
2075 let edges = data?.posts?.edges ?? [];
2076 let pageInfo: { hasNextPage?: boolean; endCursor?: string | null } = data?.posts?.pageInfo ?? {};
2077 if (!edges.length) break;
2078 log.info(`Product Hunt ${label} page ${pageNumber}`, { received: edges.length, processed: diagnostics.itemsProcessed });
2079 let nodes = edges.map((edge: { node?: Record<string, any> }) => edge.node).filter(Boolean);
2080 if (useLightweightIndex) {
2081 diagnostics.productHuntIndexPages += 1;
2082 const hasUnseenLaunch = hasUnexcludedProductHuntLaunch(
2083 nodes.map((node: Record<string, any>) => String(node.url ?? '')),
2084 excludedProductHuntKeys,
2085 );
2086 if (hasUnseenLaunch) {
2087 data = await phQuery(POSTS_QUERY, variables);
2088 diagnostics.productHuntFullPageLookups += 1;
2089 edges = data?.posts?.edges ?? [];
2090 pageInfo = data?.posts?.pageInfo ?? pageInfo;
2091 nodes = edges.map((edge: { node?: Record<string, any> }) => edge.node).filter(Boolean);
2092 useLightweightIndex = false;
2093 } else {
2094 for (const node of nodes) {
2095 const id = String(node.id ?? '');
2096 if (!id || seenIds.has(id)) continue;
2097 const launchKey = productHuntLaunchKey(String(node.url ?? ''));
2098 if (launchKey && excludedProductHuntKeys.has(launchKey)) {
2099 seenIds.add(id);
2100 diagnostics.itemsExcluded += 1;
2101 }
2102 }
2103 nodes = [];
2104 }
2105 }
2106 const selectedNodes = mode === 'search'
2107 ? nodes.filter((node: Record<string, any>) => matchesProductSearch(searchQuery, {
2108 name: node.name,
2109 tagline: node.tagline,
2110 description: node.description,
2111 topics: (node.topics?.edges ?? []).map((edge: any) => edge?.node?.name).filter(Boolean),
2112 }))
2113 : nodes;
2114 if (mode === 'search') {
2115 pagesWithoutSearchMatch = selectedNodes.length > 0 ? 0 : pagesWithoutSearchMatch + 1;
2116 if (pagesWithoutSearchMatch >= SEARCH_PAGES_WITHOUT_MATCH_LIMIT && !diagnostics.searchScanCapReached) {
2117 diagnostics.searchScanCapReached = true;
2118 addWarning(`Search stopped after ${SEARCH_PAGES_WITHOUT_MATCH_LIMIT} consecutive Product Hunt pages without a match. Continue with productHuntResumeCursor from RUN_SUMMARY.nextProductHuntCursor to search older launches.`);
2119 }
2120 }
2121 await processNodes(selectedNodes);
2122 const pageInterrupted = diagnostics.deadlineReached || hardStopping;
2123 diagnostics.nextProductHuntCursor = pageInterrupted
2124 ? pageStartCursor
2125 : pageInfo.hasNextPage && pageInfo.endCursor
2126 ? pageInfo.endCursor
2127 : null;
2128 const checkpointCursor = resumableCheckpointCursor(
2129 pageStartCursor,
2130 Boolean(pageInfo.hasNextPage),
2131 pageInfo.endCursor,
2132 failuresBeforePage,
2133 diagnostics.itemsFailed,
2134 );
2135 if (!useLightweightIndex || pageNumber === 1 || pageNumber % 10 === 0 || !pageInfo.hasNextPage) {
2136 try {
2137 await Actor.setValue('RUN_CHECKPOINT', {
2138 status: 'RUNNING',
2139 updatedAt: new Date().toISOString(),
2140 runId: process.env.ACTOR_RUN_ID ?? null,
2141 buildNumber: process.env.ACTOR_BUILD_NUMBER ?? null,
2142 mode,
2143 label,
2144 range,
2145 productHuntResumeCursor: checkpointCursor,
2146 pageNumber,
2147 itemsProcessed: diagnostics.itemsProcessed,
2148 itemsPushed: diagnostics.itemsPushed,
2149 itemsExcluded: diagnostics.itemsExcluded,
2150 itemsFailed: diagnostics.itemsFailed,
2151 });
2152 } catch (error) {
2153 const message = error instanceof Error ? error.message : 'unknown checkpoint failure';
2154 addWarning(`Run checkpoint could not be persisted: ${message.slice(0, 120)}`);
2155 }
2156 }
2157 if (collectedCandidates >= effectiveMaxResults && pageInfo.hasNextPage) {
2158 diagnostics.resultLimitReached = true;
2159 addWarning('The result limit was reached while Product Hunt still had another page. Continue with excludeProductHuntUrls from this dataset to avoid duplicate enrichment charges.');
2160 }
2161 await reportProgress(true);
2162 if (pageInterrupted || diagnostics.searchScanCapReached) break;
2163 if (!pageInfo.hasNextPage || !pageInfo.endCursor) break;
2164 cursor = pageInfo.endCursor;
2165 pageNumber += 1;
2166 } catch (error) {
2167 diagnostics.apiFailures += 1;
2168 const message = error instanceof Error ? error.message : 'Unknown Product Hunt API error.';
2169 addWarning(`Product Hunt ${label} query failed: ${message}`);
2170 break;
2171 }
2172 }
2173 };
2174
2175 if (mode === 'urls') {
2176 const rawSeeds = startUrls
2177 .map((entry) => typeof entry === 'string' ? entry : typeof entry?.url === 'string' ? entry.url : '')
2178 .filter(Boolean);
2179 const seeds = rawSeeds.map(parseProductHuntSeedUrl);
2180 const validSeeds = seeds.filter((seed) => seed.kind !== 'invalid');
2181 seeds.filter((seed) => seed.kind === 'invalid').forEach((seed) => addWarning(`Ignored start URL: ${seed.reason}`));
2182 if (validSeeds.length === 0) await finalizeRun('INVALID_INPUT', 'No valid Product Hunt post, product launch, or dated leaderboard URLs were supplied.');
2183
2184 for (const seed of validSeeds) {
2185 if (collectedCandidates >= effectiveMaxResults || diagnostics.chargeLimitReached) break;
2186 if (deadlineReached() || hardStopping) break;
2187 if (seed.kind === 'post') {
2188 try {
2189 const data = await phQuery(POST_BY_SLUG_QUERY, { slug: seed.slug });
2190 if (!data?.post) {
2191 addWarning(`No Product Hunt post matched the supplied URL slug: ${seed.slug}.`);
2192 continue;
2193 }
2194 await processNodes([data.post]);
2195 } catch (error) {
2196 diagnostics.apiFailures += 1;
2197 const message = error instanceof Error ? error.message : 'Unknown Product Hunt API error.';
2198 addWarning(`Product Hunt URL lookup failed for ${seed.slug}: ${message}`);
2199 }
2200 } else if (seed.kind === 'leaderboard') {
2201 await streamPosts(dateRangeForPeriod(seed.period, seed.anchorDate), `${seed.period} leaderboard URL`);
2202 }
2203 }
2204 } else {
2205 let range: DateRange | null = null;
2206 if (mode === 'leaderboard') {
2207 if (startDate) {
2208 range = dateRangeForPeriod(leaderboardPeriod, startDate, endDate);
2209 } else if (lookbackDays > 1) {
2210 const end = new Date();
2211 const begin = new Date(end);
2212 begin.setUTCDate(end.getUTCDate() - lookbackDays + 1);
2213 range = { postedAfter: `${isoDate(begin)}T00:00:00+00:00`, postedBefore: `${isoDate(end)}T23:59:59+00:00` };
2214 } else {
2215 range = dateRangeForPeriod(leaderboardPeriod, isoDate(new Date()));
2216 }
2217 }
2218 await streamPosts(range, mode);
2219 }
2220
2221 if (hardStopping) return;
2222 await closeBrowser();
2223 const outcome = resolveOutcome();
2224 await finalizeRun(outcome, outcomeMessage(outcome));
2225 } catch (error) {
2226 if (finalised || hardStopping) return;
2227 await closeBrowser();
2228 const message = error instanceof Error ? error.message : 'Unknown actor failure.';
2229 diagnostics.itemsFailed += 1;
2230 addWarning(`Unhandled actor error: ${message.slice(0, 160)}`);
2231 await finalizeRun('UPSTREAM_FAILED', `The run stopped before completion: ${message.slice(0, 160)}`);
2232 }
2233}
2234
2235await run();
