1import { Actor } from 'apify';
2import { log } from 'crawlee';
3import { load } from 'cheerio';
4import { lookup } from 'node:dns/promises';
5import { isIP } from 'node:net';
6import { chromium, type Browser } from 'playwright';
7import { getDomain } from 'tldts';
8import { clampToFreePlan, freePlanLimitHit, freePlanLimitSummary, freePlanStatus, resolveFreePlanCap } from './free-plan-limit.js';
9import {
10 buildWebsitePagePlan,
11 canonicalWebsiteUrl,
12 clampInteger,
13 decodeCloudflareEmail,
14 DEFAULT_MAX_EMAILS_PER_PRODUCT,
15 DEFAULT_MAX_WEBSITE_PAGES,
16 emailCandidateLimit,
17 extractEmailCandidatesFromText,
18 hasUnexcludedProductHuntLaunch,
19 hasReachedUsableEmailTarget,
20 isRetryableEmailListVerifyHttpStatus,
21 isProductHuntHost,
22 isPublicHttpUrl,
23 MAX_EMAILS_PER_PRODUCT,
24 MAX_WEBSITE_PAGES,
25 normalizeEmailListVerifyStatus,
26 parseProductHuntSeedUrl,
27 productHuntLaunchKey,
28 productHuntRateLimitDelaySeconds,
29 productHuntStreamOrder,
30 shouldUseLightweightProductHuntIndex,
31 rankEmails,
32 resumableCheckpointCursor,
33 type EmailListVerifyStatus,
34 type WebsitePage,
35} from './enrichment-utils.js';
36import {
37 buildLeadProfile,
38 buildDomainHypothesisCandidates,
39 buildProtectedProductCandidates,
40 buildProductIdentityNames,
41 buildWebsiteSearchQueries,
42 evaluateWebsiteIdentity,
43 hasDirectOwnershipSignal,
44 hasProtectedProductVendorMatch,
45 isPrivateNetworkAddress,
46 isProtectedProductHostAllowed,
47 isRetryableDataForSeoStatus,
48 isUsableEmailStatus,
49 matchesProductSearch,
50 normalizeContactFinderResults,
51 parseDataForSeoCandidates,
52 parseSerpApiCandidates,
53 rankMakerWebsiteLinkCandidates,
54 scoreSearchCandidate,
55 selectMakerFallbackEmails,
56 splitMakerName,
57 type ContactFinderResult,
58 type MakerWebsiteLink,
59 type SearchCandidate,
60} from './lead-engine.js';
61
62await Actor.init();
63
64type Mode = 'leaderboard' | 'search' | 'topic' | 'urls';
65type LeaderboardPeriod = 'daily' | 'weekly' | 'monthly' | 'yearly';
66type RunOutcome = 'COMPLETE' | 'PARTIAL' | 'VALID_EMPTY' | 'INVALID_INPUT' | 'UPSTREAM_FAILED' | 'CONFIG_ERROR';
67type EmailSource = 'page_scrape' | 'maker_website' | 'emaillistverify_contact_finder' | 'mixed' | 'none';
68type EmailResult = 'emails_found' | 'no_public_email_found' | 'not_assessed';
69type Coverage = 'complete' | 'partial' | 'none';
70type WebsiteResolutionStatus = 'verified_external' | 'verified_search_match' | 'unresolved' | 'not_available';
71type WebsiteResolutionSource = 'api_website_product_link' | 'input_override' | 'product_hunt_redirect' | 'maker_website' | 'maker_website_link' | 'protected_vendor_domain' | 'dataforseo_organic' | 'direct_domain_hypothesis' | 'serpapi_organic' | 'none';
72type EmailVerification = {
73 email: string;
74 provider: 'emaillistverify';
75 status: EmailListVerifyStatus;
76 checkedAt: string | null;
77 source: 'public_website' | 'maker_website' | 'emaillistverify_contact_finder';
78 confidence: 'high' | 'medium' | 'low' | 'unknown' | null;
79};
80
81type DateRange = { postedAfter: string; postedBefore: string };
82type WebsiteOverride = { keys: string[]; websiteUrl: string };
83type EnrichmentResult = {
84 emails: string[];
85 emailSource: EmailSource;
86 result: EmailResult;
87 coverage: Coverage;
88 terminalReason: string;
89 pagesVisited: number;
90 pageUrls: string[];
91 warnings: string[];
92};
93
94type WebsiteResolution = {
95 websiteUrl: string | null;
96 source: WebsiteResolutionSource;
97 status: WebsiteResolutionStatus;
98 confidence: number;
99 signals: string[];
100 providerCostUsd: number;
101 providerAttempts: number;
102 warnings: string[];
103};
104
105type ContactDiscovery = {
106 contacts: ContactFinderResult[];
107 attempts: number;
108 credits: number;
109 warnings: string[];
110};
111
112const PH_API = 'https://api.producthunt.com/v2/api/graphql';
113const RESULT_EVENT = 'apify-default-dataset-item';
114const EMAIL_EVENT = 'email-found';
115const RESULT_PRICE_USD = 0.002;
116const EMAIL_PRICE_USD = 0.04;
117const EMAIL_LIST_VERIFY_API = 'https://api.emaillistverify.com/api/verifyEmail';
118const EMAIL_LIST_VERIFY_CONTACT_API = 'https://api.emaillistverify.com/api/findContact';
119const DATAFORSEO_SEARCH_API = 'https://api.dataforseo.com/v3/serp/google/organic/live/advanced';
120const SERPAPI_SEARCH_API = 'https://serpapi.com/search.json';
121const EMAIL_LIST_VERIFY_MIN_INTERVAL_MS = 225;
122const SEARCH_PAGES_WITHOUT_MATCH_LIMIT = 50;
123const FAILED_OUTCOMES = new Set<RunOutcome>(['UPSTREAM_FAILED', 'CONFIG_ERROR']);
124
125const SKIP_DOMAINS = new Set([
126 'apps.apple.com', 'play.google.com', 'github.com', 'youtube.com',
127 'twitter.com', 'x.com', 'linkedin.com', 'facebook.com', 'instagram.com',
128 'reddit.com', 'discord.com', 'discord.gg', 'slack.com', 'medium.com',
129 'substack.com', 'producthunt.com', 'amazon.com', 'chrome.google.com',
130 'marketplace.visualstudio.com', 'notion.so', 'figma.com', 'trello.com',
131 'airtable.com',
132]);
133
134const PRODUCT_HUNT_USER_AGENT = 'ProductHunt Lead Generator/1.4 (authorized API client)';
135const WEBSITE_USER_AGENT = 'Mozilla/5.0 (compatible; ProductHuntLeadGenerator/1.4; +https://apify.com/khadinakbar/producthunt-scraper-pro)';
136
137let nextEmailListVerifyRequestAt = 0;
138let emailListVerifyCreditsExhausted = false;
139const publicHostCache = new Map<string, Promise<boolean>>();
140
141
142const MAX_BROWSER_CONTEXTS = 2;
143let activeBrowserContexts = 0;
144const browserContextWaiters: Array<() => void> = [];
145
146async function withBrowserContextSlot<T>(task: () => Promise<T>): Promise<T> {
147 if (activeBrowserContexts >= MAX_BROWSER_CONTEXTS) {
148 await new Promise<void>((resolve) => browserContextWaiters.push(resolve));
149 } else {
150 activeBrowserContexts += 1;
151 }
152 try {
153 return await task();
154 } finally {
155 const next = browserContextWaiters.shift();
156 if (next) next();
157 else activeBrowserContexts -= 1;
158 }
159}
160
161function isoDate(date: Date): string {
162 return date.toISOString().slice(0, 10);
163}
164
165function getWeekBounds(date: Date): { start: string; end: string } {
166 const start = new Date(date);
167
168 const weekday = date.getUTCDay() || 7;
169 start.setUTCDate(date.getUTCDate() - weekday + 1);
170 const end = new Date(start);
171 end.setUTCDate(start.getUTCDate() + 6);
172 return { start: isoDate(start), end: isoDate(end) };
173}
174
175function getMonthBounds(date: Date): { start: string; end: string } {
176 const year = date.getUTCFullYear();
177 const month = date.getUTCMonth();
178 return {
179 start: isoDate(new Date(Date.UTC(year, month, 1))),
180 end: isoDate(new Date(Date.UTC(year, month + 1, 0))),
181 };
182}
183
184function getYearBounds(date: Date): { start: string; end: string } {
185 const year = date.getUTCFullYear();
186 return { start: `${year}-01-01`, end: `${year}-12-31` };
187}
188
189function validIsoDate(value: string): boolean {
190 if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
191 const parsed = new Date(`${value}T12:00:00Z`);
192 return Number.isFinite(parsed.getTime()) && isoDate(parsed) === value;
193}
194
195function dateRangeForPeriod(period: LeaderboardPeriod, anchorDate: string, endDate = ''): DateRange {
196 const base = new Date(`${anchorDate}T12:00:00Z`);
197 if (period === 'daily') {
198 return {
199 postedAfter: `${anchorDate}T00:00:00+00:00`,
200 postedBefore: `${endDate || anchorDate}T23:59:59+00:00`,
201 };
202 }
203 const bounds = period === 'weekly'
204 ? getWeekBounds(base)
205 : period === 'monthly'
206 ? getMonthBounds(base)
207 : getYearBounds(base);
208 return {
209 postedAfter: `${bounds.start}T00:00:00+00:00`,
210 postedBefore: `${bounds.end}T23:59:59+00:00`,
211 };
212}
213
214function normalizeKey(value: unknown): string | null {
215 if (typeof value !== 'string' || !value.trim()) return null;
216 return value.trim().toLowerCase().replace(/[?#].*$/, '').replace(/\/$/, '');
217}
218
219function parseWebsiteOverrides(value: unknown): WebsiteOverride[] {
220 const overrides: WebsiteOverride[] = [];
221 const add = (keys: unknown[], websiteUrl: unknown): void => {
222 if (typeof websiteUrl !== 'string' || !isPublicHttpUrl(websiteUrl)) return;
223 const canonical = canonicalWebsiteUrl(websiteUrl);
224 if (!canonical || isProductHuntHost(new URL(canonical).hostname)) return;
225 const usableKeys = keys.map(normalizeKey).filter((key): key is string => Boolean(key));
226 if (usableKeys.length > 0) overrides.push({ keys: usableKeys, websiteUrl: canonical });
227 };
228
229 if (Array.isArray(value)) {
230 for (const entry of value) {
231 if (!entry || typeof entry !== 'object') continue;
232 const row = entry as Record<string, unknown>;
233 add([row.productHuntUrl, row.productSlug, row.productName, row.productId], row.websiteUrl);
234 }
235 } else if (value && typeof value === 'object') {
236 for (const [key, websiteUrl] of Object.entries(value as Record<string, unknown>)) add([key], websiteUrl);
237 }
238 return overrides;
239}
240
241function lookupWebsiteOverride(node: Record<string, unknown>, overrides: WebsiteOverride[]): string | null {
242 const candidates = [node.url, node.slug, node.id, node.name]
243 .map(normalizeKey)
244 .filter((value): value is string => Boolean(value));
245 return overrides.find((override) => override.keys.some((key) => candidates.includes(key)))?.websiteUrl ?? null;
246}
247
248function rootDomain(hostname: string): string {
249 const labels = hostname.replace(/^www\./i, '').toLowerCase().split('.');
250 return labels.slice(-2).join('.');
251}
252
253function isAllowedExternalWebsite(url: string): boolean {
254 if (!isPublicHttpUrl(url)) return false;
255 try {
256 const host = new URL(url).hostname.toLowerCase();
257 return !isProductHuntHost(host) && !SKIP_DOMAINS.has(host) && !SKIP_DOMAINS.has(rootDomain(host));
258 } catch {
259 return false;
260 }
261}
262
263async function isSafePublicNetworkUrl(rawUrl: string): Promise<boolean> {
264 if (!isPublicHttpUrl(rawUrl)) return false;
265 let hostname: string;
266 try {
267 hostname = new URL(rawUrl).hostname.toLowerCase().replace(/^\[|\]$/g, '');
268 } catch {
269 return false;
270 }
271 if (isIP(hostname)) return !isPrivateNetworkAddress(hostname);
272
273 let assessment = publicHostCache.get(hostname);
274 if (!assessment) {
275 assessment = lookup(hostname, { all: true, verbatim: true })
276 .then((records) => records.length > 0 && records.every(({ address }) => !isPrivateNetworkAddress(address)))
277 .catch(() => false);
278 publicHostCache.set(hostname, assessment);
279 }
280 return assessment;
281}
282
283
284
285function browserExtractEmails(): string[] {
286 const emailRe = /\b[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}\b/g;
287 const obfuscatedRe = /\b[a-zA-Z0-9._%+\-]+\s*(?:\[at\]|\(at\)|\sat\s)\s*[a-zA-Z0-9.\-]+\s*(?:\[dot\]|\(dot\)|\sdot\s)\s*[a-zA-Z]{2,}\b/gi;
288 const collected: string[] = [];
289 const add = (value: unknown): void => {
290 if (typeof value !== 'string') return;
291 let decoded = value;
292 try { decoded = decodeURIComponent(value); } catch { }
293 const email = decoded.trim().toLowerCase().replace(/^mailto:/, '').split('?')[0];
294 if (email.includes('@')) collected.push(email);
295 };
296 const walkJson = (value: unknown, depth = 0): void => {
297 if (depth > 8 || value === null || value === undefined) return;
298 if (Array.isArray(value)) {
299 value.forEach((item) => walkJson(item, depth + 1));
300 return;
301 }
302 if (typeof value !== 'object') return;
303 for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
304 if (key.toLowerCase() === 'email') add(child);
305 walkJson(child, depth + 1);
306 }
307 };
308 const decodeCfEmail = (encoded: string): string | null => {
309 if (!/^[0-9a-f]+$/i.test(encoded) || encoded.length < 4) return null;
310 const key = Number.parseInt(encoded.slice(0, 2), 16);
311 let decoded = '';
312 for (let index = 2; index < encoded.length; index += 2) {
313 decoded += String.fromCharCode(Number.parseInt(encoded.slice(index, index + 2), 16) ^ key);
314 }
315 return decoded;
316 };
317
318 document.querySelectorAll('a[href^="mailto:"]').forEach((element) => add((element as HTMLAnchorElement).href));
319 document.querySelectorAll('[data-email]').forEach((element) => add(element.getAttribute('data-email')));
320 document.querySelectorAll('[data-cfemail]').forEach((element) => {
321 const decoded = decodeCfEmail(element.getAttribute('data-cfemail') ?? '');
322 if (decoded) add(decoded);
323 });
324 document.querySelectorAll('script[type="application/ld+json"]').forEach((element) => {
325 try { walkJson(JSON.parse(element.textContent ?? '')); } catch { }
326 });
327
328 const combined = `${document.body?.innerText ?? ''}\n${document.documentElement.innerHTML}`
329 .replace(/\\(?:x40|u0040)/gi, '@')
330 .replace(/\\(?:x2e|u002e)/gi, '.')
331 .replace(/&#(?:64|x40);|@/gi, '@')
332 .replace(/&#(?:46|x2e);|./gi, '.');
333 (combined.match(emailRe) ?? []).forEach(add);
334 for (const match of combined.match(obfuscatedRe) ?? []) {
335 add(match.replace(/\s*(?:\[at\]|\(at\)|\sat\s)\s*/i, '@').replace(/\s*(?:\[dot\]|\(dot\)|\sdot\s)\s*/ig, '.'));
336 }
337 return collected;
338}
339
340
341
342function browserExtractLinkedFallbackPages(origin: string): WebsitePage[] {
343 const matchers: Array<{ kind: Exclude<WebsitePage['kind'], 'homepage'>; priority: number; pattern: RegExp }> = [
344 { kind: 'contact', priority: 0, pattern: /(contact|contact-us|get-in-touch|reach-us)/i },
345 { kind: 'about', priority: 1, pattern: /(about|about-us|company|our-story)/i },
346 { kind: 'team', priority: 2, pattern: /(team|our-team|meet-the-team|people|founders)/i },
347 { kind: 'privacy', priority: 3, pattern: /(privacy|privacy-policy)/i },
348 { kind: 'terms', priority: 4, pattern: /(terms|terms-of-service|terms-and-conditions|conditions)/i },
349 ];
350 const links = new Map<string, WebsitePage>();
351 document.querySelectorAll('a[href]').forEach((element) => {
352 const anchor = element as HTMLAnchorElement;
353 try {
354 const url = new URL(anchor.href);
355 if (url.origin !== origin || !['http:', 'https:'].includes(url.protocol) || url.username || url.password) return;
356 const label = (anchor.innerText || anchor.textContent || '').trim();
357 const matched = matchers.find((candidate) => candidate.pattern.test(`${url.pathname} ${label}`));
358 if (!matched) return;
359 url.hash = '';
360 const clean = url.toString();
361 if (clean === `${origin}/` || clean === origin || links.has(clean)) return;
362 links.set(clean, { url: clean, kind: matched.kind, priority: matched.priority });
363 } catch { }
364 });
365 return [...links.values()].sort((left, right) => left.priority - right.priority);
366}
367
368async function waitForEmailListVerifySlot(): Promise<void> {
369 const scheduledAt = Math.max(Date.now(), nextEmailListVerifyRequestAt);
370 nextEmailListVerifyRequestAt = scheduledAt + EMAIL_LIST_VERIFY_MIN_INTERVAL_MS;
371 const delayMs = scheduledAt - Date.now();
372 if (delayMs > 0) await new Promise((resolve) => setTimeout(resolve, delayMs));
373}
374
375async function verifyEmailsWithEmailListVerify(
376 emails: string[],
377 requested: boolean,
378 apiKey: string,
379 source: EmailVerification['source'] = 'public_website',
380 maxUsableEmails = emails.length,
381): Promise<{ verifications: EmailVerification[]; attempts: number; warnings: string[] }> {
382 if (emails.length === 0) return { verifications: [], attempts: 0, warnings: [] };
383 if (!requested) {
384 return {
385 verifications: emails.map((email) => ({
386 email, provider: 'emaillistverify', status: 'not_requested', checkedAt: null,
387 source, confidence: null,
388 })),
389 attempts: 0,
390 warnings: [],
391 };
392 }
393 if (!apiKey) {
394 return {
395 verifications: emails.map((email) => ({
396 email, provider: 'emaillistverify', status: 'not_configured', checkedAt: null,
397 source, confidence: null,
398 })),
399 attempts: 0,
400 warnings: ['EmailListVerify was requested but no owner-managed API key was configured; public emails were kept unverified.'],
401 };
402 }
403 if (emailListVerifyCreditsExhausted) {
404 return {
405 verifications: emails.map((email) => ({
406 email, provider: 'emaillistverify', status: 'error_credit', checkedAt: null,
407 source, confidence: null,
408 })),
409 attempts: 0,
410 warnings: ['EmailListVerify credits were exhausted earlier in this run; no additional verification requests were sent.'],
411 };
412 }
413
414 const warnings = new Set<string>();
415 const verifications: EmailVerification[] = [];
416 let attempts = 0;
417 for (const email of emails) {
418 if (hasReachedUsableEmailTarget(verifications.map(({ status }) => status), maxUsableEmails)) break;
419 for (let requestAttempt = 0; requestAttempt < 2; requestAttempt += 1) {
420 await waitForEmailListVerifySlot();
421 attempts += 1;
422 const checkedAt = new Date().toISOString();
423 try {
424 const requestUrl = new URL(EMAIL_LIST_VERIFY_API);
425 requestUrl.searchParams.set('email', email);
426 const response = await fetch(requestUrl, {
427 headers: { 'x-api-key': apiKey, Accept: 'text/plain, application/json;q=0.9' },
428 signal: AbortSignal.timeout(12_000),
429 });
430 if (!response.ok) {
431 if (requestAttempt === 0 && isRetryableEmailListVerifyHttpStatus(response.status)) {
432 await new Promise((resolve) => setTimeout(resolve, 500));
433 continue;
434 }
435 warnings.add(`EmailListVerify returned HTTP ${response.status}; affected public emails were retained with provider_error status.`);
436 if (response.status === 402) emailListVerifyCreditsExhausted = true;
437 verifications.push({
438 email, provider: 'emaillistverify', status: response.status === 402 ? 'error_credit' : 'provider_error', checkedAt,
439 source, confidence: null,
440 });
441 break;
442 }
443 const status = normalizeEmailListVerifyStatus(await response.text());
444 if (status === 'provider_error') warnings.add('EmailListVerify returned an undocumented response; affected public emails were retained with provider_error status.');
445 if (status === 'error_credit') {
446 emailListVerifyCreditsExhausted = true;
447 warnings.add('EmailListVerify reported insufficient verification credits; remaining public emails were retained with error_credit status.');
448 }
449 verifications.push({ email, provider: 'emaillistverify', status, checkedAt, source, confidence: null });
450 break;
451 } catch {
452 if (requestAttempt === 0) {
453 await new Promise((resolve) => setTimeout(resolve, 500));
454 continue;
455 }
456 warnings.add('EmailListVerify request failed twice; affected public emails were retained with provider_error status.');
457 verifications.push({ email, provider: 'emaillistverify', status: 'provider_error', checkedAt, source, confidence: null });
458 }
459 }
460 if (emailListVerifyCreditsExhausted) break;
461 }
462 return { verifications, attempts, warnings: [...warnings] };
463}
464
465let browser: Browser | null = null;
466
467async function ensureBrowser(): Promise<Browser> {
468 if (!browser) {
469 browser = await chromium.launch({
470 headless: true,
471 args: [
472 '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage',
473 '--disable-gpu', '--no-zygote', '--disable-extensions',
474 ],
475 });
476 log.info('Browser launched for verified external website enrichment.');
477 }
478 return browser;
479}
480
481async function closeBrowser(): Promise<void> {
482 if (!browser) return;
483 await browser.close().catch(() => undefined);
484 browser = null;
485}
486
487function extractStaticEmails(html: string): string[] {
488 const $ = load(html);
489 const collected: string[] = [];
490 const add = (value: unknown): void => {
491 if (typeof value !== 'string') return;
492 let decoded = value;
493 try { decoded = decodeURIComponent(value); } catch { }
494 const email = decoded.trim().toLowerCase().replace(/^mailto:/, '').split('?')[0];
495 if (email.includes('@')) collected.push(email);
496 };
497 const walkJson = (value: unknown, depth = 0): void => {
498 if (depth > 8 || value === null || value === undefined) return;
499 if (Array.isArray(value)) return value.forEach((item) => walkJson(item, depth + 1));
500 if (typeof value !== 'object') return;
501 for (const [key, child] of Object.entries(value as Record<string, unknown>)) {
502 if (key.toLowerCase() === 'email') add(child);
503 walkJson(child, depth + 1);
504 }
505 };
506 $('a[href^="mailto:"]').each((_, element) => add($(element).attr('href')));
507 $('[data-email]').each((_, element) => add($(element).attr('data-email')));
508 $('[data-cfemail]').each((_, element) => add(decodeCloudflareEmail($(element).attr('data-cfemail') ?? '')));
509 $('script[type="application/ld+json"]').each((_, element) => {
510 try { walkJson(JSON.parse($(element).text())); } catch { }
511 });
512 const combined = `${$.text()}\n${html}`;
513 extractEmailCandidatesFromText(combined).forEach(add);
514 return collected;
515}
516
517function extractStaticFallbackPages(html: string, origin: string): WebsitePage[] {
518 const $ = load(html);
519 const matchers: Array<{ kind: Exclude<WebsitePage['kind'], 'homepage'>; priority: number; pattern: RegExp }> = [
520 { kind: 'contact', priority: 0, pattern: /(contact|contact-us|get-in-touch|reach-us)/i },
521 { kind: 'about', priority: 1, pattern: /(about|about-us|company|our-story)/i },
522 { kind: 'team', priority: 2, pattern: /(team|our-team|meet-the-team|people|founders)/i },
523 { kind: 'privacy', priority: 3, pattern: /(privacy|privacy-policy)/i },
524 { kind: 'terms', priority: 4, pattern: /(terms|terms-of-service|terms-and-conditions|conditions)/i },
525 ];
526 const candidates = new Map<string, WebsitePage>();
527 $('a[href]').each((_, element) => {
528 try {
529 const url = new URL($(element).attr('href') ?? '', origin);
530 if (url.origin !== origin || !['http:', 'https:'].includes(url.protocol) || url.username || url.password) return;
531 const label = $(element).text().trim();
532 const matched = matchers.find((candidate) => candidate.pattern.test(`${url.pathname} ${label}`));
533 if (!matched) return;
534 url.hash = '';
535 const clean = url.toString();
536 if (clean === `${origin}/` || clean === origin || candidates.has(clean)) return;
537 candidates.set(clean, { url: clean, kind: matched.kind, priority: matched.priority });
538 } catch { }
539 });
540 return [...candidates.values()].sort((left, right) => left.priority - right.priority);
541}
542
543type StaticPage = { ok: true; url: string; html: string } | { ok: false; reason: string };
544
545async function fetchPublicHtml(rawUrl: string, timeoutMs: number): Promise<StaticPage> {
546 let current = canonicalWebsiteUrl(rawUrl);
547 if (!current || !isAllowedExternalWebsite(current)) return { ok: false, reason: 'invalid_or_private_target' };
548 try {
549 for (let hop = 0; hop <= 3; hop += 1) {
550 if (!await isSafePublicNetworkUrl(current)) return { ok: false, reason: 'private_or_unresolvable_target' };
551 const response = await fetch(current, {
552 method: 'GET',
553 redirect: 'manual',
554 headers: { Accept: 'text/html,application/xhtml+xml', 'User-Agent': WEBSITE_USER_AGENT },
555 signal: AbortSignal.timeout(timeoutMs),
556 });
557 if (response.status >= 300 && response.status < 400) {
558 const location = response.headers.get('location');
559 if (!location || hop === 3) return { ok: false, reason: 'redirect_hop_limit' };
560 const next = canonicalWebsiteUrl(new URL(location, current).toString());
561 if (!next || !isAllowedExternalWebsite(next)) return { ok: false, reason: 'redirect_to_private_or_unsupported_target' };
562 if (!await isSafePublicNetworkUrl(next)) return { ok: false, reason: 'redirect_to_private_or_unresolvable_target' };
563 current = next;
564 continue;
565 }
566 if (!response.ok) return { ok: false, reason: `http_${response.status}` };
567 const contentType = response.headers.get('content-type') ?? '';
568 if (!/(?:text\/html|application\/xhtml\+xml)/i.test(contentType)) return { ok: false, reason: 'non_html_response' };
569 const contentLength = Number(response.headers.get('content-length') ?? '0');
570 if (Number.isFinite(contentLength) && contentLength > 1_500_000) return { ok: false, reason: 'response_too_large' };
571 const html = (await response.text()).slice(0, 1_500_000);
572 return { ok: true, url: current, html };
573 }
574 return { ok: false, reason: 'redirect_hop_limit' };
575 } catch (error) {
576 const message = error instanceof Error ? error.message : '';
577 return { ok: false, reason: /timeout|abort/i.test(message) ? 'timeout' : 'network_error' };
578 }
579}
580
581function pageIdentity(html: string): { title: string; description: string; text: string } {
582 const $ = load(html);
583 $('script,style,noscript,svg').remove();
584 return {
585 title: $('title').first().text().trim(),
586 description: $('meta[name="description"]').attr('content')?.trim()
587 || $('meta[property="og:description"]').attr('content')?.trim()
588 || '',
589 text: $('body').text().replace(/\s+/g, ' ').trim().slice(0, 100_000),
590 };
591}
592
593async function resolveProductHuntRedirect(rawUrl: string): Promise<string | null> {
594 let current = canonicalWebsiteUrl(rawUrl);
595 if (!current) return null;
596 try {
597 if (!isProductHuntHost(new URL(current).hostname)) return null;
598 for (let hop = 0; hop < 4; hop += 1) {
599 if (!await isSafePublicNetworkUrl(current)) return null;
600 let response = await fetch(current, {
601 method: 'HEAD',
602 redirect: 'manual',
603 headers: { 'User-Agent': PRODUCT_HUNT_USER_AGENT, Accept: 'text/html,*/*;q=0.5' },
604 signal: AbortSignal.timeout(8_000),
605 });
606 if (response.status === 405) {
607 response = await fetch(current, {
608 method: 'GET',
609 redirect: 'manual',
610 headers: { 'User-Agent': PRODUCT_HUNT_USER_AGENT, Accept: 'text/html,*/*;q=0.5' },
611 signal: AbortSignal.timeout(8_000),
612 });
613 }
614 if (response.status < 300 || response.status >= 400) return null;
615 const location = response.headers.get('location');
616 if (!location) return null;
617 const next = canonicalWebsiteUrl(new URL(location, current).toString());
618 if (!next) return null;
619 if (!isProductHuntHost(new URL(next).hostname)) {
620 return isAllowedExternalWebsite(next) && await isSafePublicNetworkUrl(next) ? next : null;
621 }
622 current = next;
623 }
624 } catch {
625 return null;
626 }
627 return null;
628}
629
630async function verifyWebsiteCandidates(
631 candidates: SearchCandidate[],
632 productName: string,
633 tagline: string,
634 description = '',
635): Promise<{ websiteUrl: string; confidence: number; signals: string[] } | null> {
636 const identityNames = buildProductIdentityNames(productName);
637 const purpose = `${tagline} ${description}`.trim().slice(0, 1_200);
638 const ranked = [...candidates]
639 .map((candidate) => ({
640 candidate,
641 score: Math.max(...identityNames.map((identity) => scoreSearchCandidate(candidate, identity, purpose))),
642 }))
643 .filter(({ score }) => score >= 34)
644 .sort((left, right) => right.score - left.score || left.candidate.rank - right.candidate.rank)
645 .slice(0, 5);
646
647 for (const { candidate } of ranked) {
648 let candidateUrl: URL;
649 try {
650 candidateUrl = new URL(candidate.url);
651 } catch {
652 continue;
653 }
654 if (!isProtectedProductHostAllowed(productName, candidateUrl.toString())) continue;
655 const homepage = `${candidateUrl.origin}/`;
656 const verificationUrls = candidateUrl.pathname !== '/' || candidateUrl.search
657 ? [candidateUrl.toString(), homepage]
658 : [homepage];
659 for (const verificationUrl of [...new Set(verificationUrls)]) {
660 const fetched = await fetchPublicHtml(verificationUrl, 6_000);
661 if (!fetched.ok) continue;
662 const identity = pageIdentity(fetched.html);
663 const assessments = identityNames.map((identityName) => ({
664 identityName,
665 assessment: evaluateWebsiteIdentity({
666 candidate: { ...candidate, url: fetched.url },
667 productName: identityName,
668 tagline: purpose,
669 pageTitle: identity.title,
670 pageDescription: identity.description,
671 pageText: identity.text,
672 }),
673 })).filter(({ assessment }) => assessment.accepted)
674 .sort((left, right) => right.assessment.confidence - left.assessment.confidence);
675 if (assessments.length === 0) continue;
676 const { assessment, identityName } = assessments[0];
677 if (new URL(verificationUrl).pathname !== '/'
678 && !hasDirectOwnershipSignal(assessment.signals)
679 && !hasProtectedProductVendorMatch(productName, verificationUrl)) {
680 continue;
681 }
682 const finalUrl = new URL(fetched.url);
683 const signals = [...assessment.signals];
684 if (identityName !== identityNames[0]) signals.push('core_brand_alias_confirmed');
685 if (new URL(verificationUrl).pathname !== '/') signals.push('official_product_page_confirmed');
686 return {
687 websiteUrl: `${finalUrl.origin}/`,
688 confidence: assessment.confidence,
689 signals: [...new Set(signals)],
690 };
691 }
692 }
693 return null;
694}
695
696async function discoverMakerLinkedProductCandidates(
697 makerWebsiteUrls: string[],
698 productName: string,
699 tagline: string,
700): Promise<SearchCandidate[]> {
701 const collected = new Map<string, SearchCandidate>();
702 for (const makerWebsiteUrl of makerWebsiteUrls.filter((url) => isAllowedExternalWebsite(url)).slice(0, 2)) {
703 const fetched = await fetchPublicHtml(makerWebsiteUrl, 6_000);
704 if (!fetched.ok) continue;
705 const makerOrigin = new URL(fetched.url).origin;
706 const $ = load(fetched.html);
707 const links: MakerWebsiteLink[] = [];
708 $('a[href]').each((index, element) => {
709 try {
710 const target = new URL($(element).attr('href') ?? '', fetched.url);
711 if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) return;
712 if (target.origin === makerOrigin) return;
713 const text = $(element).text().replace(/\s+/g, ' ').trim();
714 const context = $(element).closest('p,li,article,section,div').first().text().replace(/\s+/g, ' ').trim();
715 links.push({ url: target.toString(), text, context, rank: index + 1 });
716 } catch { }
717 });
718 for (const candidate of rankMakerWebsiteLinkCandidates(links, fetched.url, productName, tagline)) {
719 let key: string;
720 try { key = new URL(candidate.url).origin; } catch { continue; }
721 if (!collected.has(key)) collected.set(key, candidate);
722 }
723 }
724 return [...collected.values()]
725 .sort((left, right) => scoreSearchCandidate(right, productName, tagline) - scoreSearchCandidate(left, productName, tagline))
726 .slice(0, 5);
727}
728
729class DataForSeoSearchError extends Error {
730 constructor(message: string, readonly retryable: boolean, readonly costUsd: number) {
731 super(message);
732 }
733}
734
735async function callDataForSeoWebsiteSearch(
736 query: string,
737 login: string,
738 password: string,
739): Promise<{ candidates: SearchCandidate[]; costUsd: number }> {
740 const response = await fetch(DATAFORSEO_SEARCH_API, {
741 method: 'POST',
742 headers: {
743 Authorization: `Basic ${Buffer.from(`${login}:${password}`).toString('base64')}`,
744 'Content-Type': 'application/json',
745 Accept: 'application/json',
746 },
747 body: JSON.stringify([{
748 keyword: query,
749 location_code: 2840,
750 language_code: 'en',
751 device: 'desktop',
752 os: 'windows',
753 depth: 10,
754 group_organic_results: true,
755 }]),
756 signal: AbortSignal.timeout(45_000),
757 });
758 if (!response.ok) {
759 throw new DataForSeoSearchError(
760 `Primary website search returned HTTP ${response.status}.`,
761 response.status === 429 || response.status >= 500,
762 0,
763 );
764 }
765 const body = await response.json() as Record<string, any>;
766 const parsed = parseDataForSeoCandidates(body);
767 if (body.status_code && body.status_code !== 20000) {
768 throw new DataForSeoSearchError(
769 `Primary website search failed with provider code ${body.status_code}: ${String(body.status_message ?? 'unknown error')}.`,
770 isRetryableDataForSeoStatus(Number(body.status_code)),
771 parsed.costUsd,
772 );
773 }
774 const task = Array.isArray(body.tasks) ? body.tasks[0] : null;
775 if (task?.status_code && task.status_code !== 20000) {
776 throw new DataForSeoSearchError(
777 `Primary website search task failed with provider code ${task.status_code}: ${String(task.status_message ?? 'unknown error')}.`,
778 isRetryableDataForSeoStatus(Number(task.status_code)),
779 parsed.costUsd,
780 );
781 }
782 return parsed;
783}
784
785async function callSerpApiWebsiteSearch(query: string, apiKey: string): Promise<SearchCandidate[]> {
786 const url = new URL(SERPAPI_SEARCH_API);
787 url.searchParams.set('engine', 'google');
788 url.searchParams.set('q', query);
789 url.searchParams.set('num', '10');
790 url.searchParams.set('gl', 'us');
791 url.searchParams.set('hl', 'en');
792 url.searchParams.set('api_key', apiKey);
793 const response = await fetch(url, { signal: AbortSignal.timeout(30_000), headers: { Accept: 'application/json' } });
794 if (!response.ok) throw new Error(`Backup website search returned HTTP ${response.status}.`);
795 const body = await response.json() as Record<string, unknown>;
796 if (body.error) throw new Error('Backup website search returned a provider error.');
797 return parseSerpApiCandidates(body);
798}
799
800async function resolveWebsite(input: {
801 productName: string;
802 tagline: string;
803 description: string;
804 productHuntRedirect: string;
805 makerWebsiteUrls: string[];
806 dataForSeoLogin: string;
807 dataForSeoPassword: string;
808 serpApiKey: string;
809}): Promise<WebsiteResolution> {
810 const warnings: string[] = [];
811 let providerCostUsd = 0;
812 let providerAttempts = 0;
813 const unresolved = (): WebsiteResolution => ({
814 websiteUrl: null,
815 source: 'none',
816 status: input.productHuntRedirect ? 'unresolved' : 'not_available',
817 confidence: 0,
818 signals: [],
819 providerCostUsd,
820 providerAttempts,
821 warnings,
822 });
823
824 if (input.productHuntRedirect) {
825 const destination = await resolveProductHuntRedirect(input.productHuntRedirect);
826 if (destination) {
827 const verified = await verifyWebsiteCandidates([{
828 url: destination, title: input.productName, snippet: input.tagline, rank: 1,
829 }], input.productName, input.tagline, input.description);
830 if (verified) {
831 return {
832 ...verified,
833 source: 'product_hunt_redirect',
834 status: 'verified_external',
835 providerCostUsd,
836 providerAttempts,
837 warnings,
838 };
839 }
840 }
841 }
842
843 const makerCandidates = input.makerWebsiteUrls
844 .filter((url) => isAllowedExternalWebsite(url))
845 .slice(0, 2)
846 .map((url, index): SearchCandidate => ({ url, title: '', snippet: '', rank: index + 1 }));
847 if (makerCandidates.length > 0) {
848 const verified = await verifyWebsiteCandidates(makerCandidates, input.productName, input.tagline, input.description);
849 if (verified) {
850 return {
851 ...verified,
852 source: 'maker_website',
853 status: 'verified_external',
854 providerCostUsd,
855 providerAttempts,
856 warnings,
857 };
858 }
859
860 const linkedCandidates = await discoverMakerLinkedProductCandidates(
861 input.makerWebsiteUrls,
862 input.productName,
863 input.tagline,
864 );
865 if (linkedCandidates.length > 0) {
866 const linked = await verifyWebsiteCandidates(linkedCandidates, input.productName, input.tagline, input.description);
867 if (linked) {
868 return {
869 ...linked,
870 signals: [...new Set([...linked.signals, 'product_linked_from_maker_website'])],
871 source: 'maker_website_link',
872 status: 'verified_external',
873 providerCostUsd,
874 providerAttempts,
875 warnings,
876 };
877 }
878 warnings.push('Maker websites exposed product-like links, but none passed product-identity verification.');
879 }
880 }
881
882 const vendorCandidate = await verifyWebsiteCandidates(
883 buildProtectedProductCandidates(input.productName).map((candidate) => ({
884 ...candidate,
885 snippet: input.tagline,
886 })),
887 input.productName,
888 input.tagline,
889 input.description,
890 );
891 if (vendorCandidate) {
892 return {
893 ...vendorCandidate,
894 signals: [...new Set([...vendorCandidate.signals, 'protected_vendor_domain_confirmed'])],
895 source: 'protected_vendor_domain',
896 status: 'verified_external',
897 providerCostUsd,
898 providerAttempts,
899 warnings,
900 };
901 }
902
903 const queries = buildWebsiteSearchQueries(input.productName, input.tagline);
904 if (input.dataForSeoLogin && input.dataForSeoPassword) {
905 let queryIndex = 0;
906 while (queryIndex < queries.length && providerAttempts < 2) {
907 const query = queries[queryIndex];
908 providerAttempts += 1;
909 try {
910 const search = await callDataForSeoWebsiteSearch(query, input.dataForSeoLogin, input.dataForSeoPassword);
911 providerCostUsd += search.costUsd;
912 const verified = await verifyWebsiteCandidates(
913 search.candidates,
914 input.productName,
915 input.tagline,
916 input.description,
917 );
918 if (verified) {
919 return {
920 ...verified,
921 source: 'dataforseo_organic',
922 status: 'verified_search_match',
923 providerCostUsd,
924 providerAttempts,
925 warnings,
926 };
927 }
928 warnings.push(`Primary website search query ${queryIndex + 1} returned no website candidate that passed product-identity verification.`);
929 queryIndex += 1;
930 } catch (error) {
931 if (error instanceof DataForSeoSearchError) providerCostUsd += error.costUsd;
932 const message = error instanceof Error ? error.message : 'Primary website search failed.';
933 warnings.push(message.slice(0, 200));
934 if (error instanceof DataForSeoSearchError && error.retryable && providerAttempts < 2) {
935 continue;
936 }
937 queryIndex += 1;
938 }
939 }
940 }
941
942 const directDomain = await verifyWebsiteCandidates(
943 buildDomainHypothesisCandidates(input.productName),
944 input.productName,
945 input.tagline,
946 input.description,
947 );
948 if (directDomain) {
949 return {
950 ...directDomain,
951 signals: [...new Set([...directDomain.signals, 'direct_domain_hypothesis_confirmed'])],
952 source: 'direct_domain_hypothesis',
953 status: 'verified_external',
954 providerCostUsd,
955 providerAttempts,
956 warnings,
957 };
958 }
959
960 if (input.serpApiKey) {
961 providerAttempts += 1;
962 try {
963 const query = queries.at(-1) ?? buildWebsiteSearchQueries(input.productName, '')[0];
964 const candidates = await callSerpApiWebsiteSearch(query, input.serpApiKey);
965 const verified = await verifyWebsiteCandidates(candidates, input.productName, input.tagline, input.description);
966 if (verified) {
967 return {
968 ...verified,
969 source: 'serpapi_organic',
970 status: 'verified_search_match',
971 providerCostUsd,
972 providerAttempts,
973 warnings,
974 };
975 }
976 warnings.push('Backup website search returned no website candidate that passed product-identity verification.');
977 } catch (error) {
978 const message = error instanceof Error ? error.message : 'Backup website search failed.';
979 warnings.push(message.slice(0, 160));
980 }
981 }
982
983 if (!input.dataForSeoLogin && !input.serpApiKey) {
984 warnings.push('No owner-managed website search provider was configured for unresolved Product Hunt links.');
985 }
986 return unresolved();
987}
988
989function companyDomainFromWebsite(websiteUrl: string): string {
990 try {
991 const hostname = new URL(websiteUrl).hostname.toLowerCase().replace(/^www\./, '');
992 return getDomain(hostname, { allowPrivateDomains: true }) ?? hostname;
993 } catch {
994 return '';
995 }
996}
997
998async function requestEmailListVerifyContacts(
999 apiKey: string,
1000 body: { domain: string; firstName?: string; lastName?: string },
1001): Promise<unknown> {
1002 await waitForEmailListVerifySlot();
1003 const response = await fetch(EMAIL_LIST_VERIFY_CONTACT_API, {
1004 method: 'POST',
1005 headers: { 'x-api-key': apiKey, 'Content-Type': 'application/json', Accept: 'application/json' },
1006 body: JSON.stringify(body),
1007 signal: AbortSignal.timeout(30_000),
1008 });
1009 if (!response.ok) {
1010 if (response.status === 402) emailListVerifyCreditsExhausted = true;
1011 throw new Error(`EmailListVerify Contact Finder returned HTTP ${response.status}.`);
1012 }
1013 return response.json();
1014}
1015
1016async function discoverContacts(
1017 websiteUrl: string,
1018 makerNames: string[],
1019 maxEmails: number,
1020 apiKey: string,
1021): Promise<ContactDiscovery> {
1022 const domain = companyDomainFromWebsite(websiteUrl);
1023 if (!domain || maxEmails <= 0) return { contacts: [], attempts: 0, credits: 0, warnings: [] };
1024 if (!apiKey) {
1025 return {
1026 contacts: [], attempts: 0, credits: 0,
1027 warnings: ['Contact discovery was requested but the owner-managed EmailListVerify key is not configured.'],
1028 };
1029 }
1030 if (emailListVerifyCreditsExhausted) {
1031 return {
1032 contacts: [], attempts: 0, credits: 0,
1033 warnings: ['EmailListVerify credits were exhausted earlier in this run; contact discovery was skipped.'],
1034 };
1035 }
1036
1037 const warnings: string[] = [];
1038 let attempts = 0;
1039 let credits = 0;
1040 let contacts: ContactFinderResult[] = [];
1041 const parsedMakers = makerNames.map(splitMakerName).filter(({ firstName, lastName }) => firstName || lastName);
1042 const maker = parsedMakers.find(({ firstName, lastName }) => firstName && lastName) ?? parsedMakers[0];
1043 if (maker) {
1044 attempts += 1;
1045 credits += 5;
1046 try {
1047 const payload = await requestEmailListVerifyContacts(apiKey, { domain, ...maker });
1048 contacts = normalizeContactFinderResults(payload, domain, maxEmails);
1049 if (contacts.length >= maxEmails) return { contacts, attempts, credits, warnings };
1050 } catch (error) {
1051 const message = error instanceof Error ? error.message : 'EmailListVerify named contact discovery failed.';
1052 warnings.push(message.slice(0, 160));
1053 }
1054 }
1055
1056 if (!emailListVerifyCreditsExhausted) {
1057 attempts += 1;
1058 credits += 10;
1059 try {
1060 const payload = await requestEmailListVerifyContacts(apiKey, { domain });
1061 contacts = normalizeContactFinderResults([
1062 ...contacts,
1063 ...(Array.isArray(payload) ? payload : []),
1064 ], domain, maxEmails);
1065 return { contacts, attempts, credits, warnings };
1066 } catch (error) {
1067 const message = error instanceof Error ? error.message : 'EmailListVerify domain contact discovery failed.';
1068 warnings.push(message.slice(0, 160));
1069 }
1070 }
1071 return { contacts, attempts, credits, warnings };
1072}
1073
1074function selectBestEmails(
1075 publicVerifications: EmailVerification[],
1076 contacts: ContactFinderResult[],
1077 maxEmails: number,
1078): EmailVerification[] {
1079 const contactVerifications: EmailVerification[] = contacts.map((contact) => ({
1080 email: contact.email,
1081 provider: 'emaillistverify',
1082 status: contact.result as EmailListVerifyStatus,
1083 checkedAt: new Date().toISOString(),
1084 source: 'emaillistverify_contact_finder',
1085 confidence: contact.confidence,
1086 }));
1087 const statusScore: Record<string, number> = {
1088 ok: 100, ok_for_all: 70, unknown: 55, antispam_system: 50,
1089 not_requested: 48, not_configured: 45, provider_error: 40, error_credit: 38,
1090 smtp_protocol: 30, dead_server: 10, email_disabled: 5, invalid_mx: 0,
1091 invalid_syntax: 0, disposable: 0, spamtrap: 0,
1092 };
1093 const best = new Map<string, EmailVerification>();
1094 for (const verification of [...publicVerifications, ...contactVerifications]) {
1095 const current = best.get(verification.email);
1096 const sourceBonus = verification.source === 'public_website' ? 3 : verification.source === 'maker_website' ? 2 : 0;
1097 const score = (statusScore[verification.status] ?? 20)
1098 + sourceBonus
1099 + (verification.confidence === 'high' ? 2 : verification.confidence === 'medium' ? 1 : 0);
1100 const currentScore = current
1101 ? (statusScore[current.status] ?? 20) + (current.source === 'public_website' ? 3 : current.source === 'maker_website' ? 2 : 0)
1102 : -1;
1103 if (!current || score > currentScore) best.set(verification.email, verification);
1104 }
1105 return [...best.values()]
1106 .filter(({ status }) => isUsableEmailStatus(status))
1107 .sort((left, right) => {
1108 const leftScore = (statusScore[left.status] ?? 20) + (left.source === 'public_website' ? 3 : left.source === 'maker_website' ? 2 : 0);
1109 const rightScore = (statusScore[right.status] ?? 20) + (right.source === 'public_website' ? 3 : right.source === 'maker_website' ? 2 : 0);
1110 return rightScore - leftScore;
1111 })
1112 .slice(0, maxEmails);
1113}
1114
1115async function findEmails(
1116 rawWebsiteUrl: string,
1117 maxWebsitePages: number,
1118 maxEmailsPerProduct: number,
1119): Promise<EnrichmentResult> {
1120 const websiteUrl = canonicalWebsiteUrl(rawWebsiteUrl);
1121 if (!websiteUrl || !isAllowedExternalWebsite(websiteUrl)) {
1122 return {
1123 emails: [], emailSource: 'none', result: 'not_assessed', coverage: 'none',
1124 terminalReason: 'invalid_or_private_target', pagesVisited: 0, pageUrls: [],
1125 warnings: ['Website URL was not a permitted public external HTTP(S) target.'],
1126 };
1127 }
1128
1129 const initialUrl = new URL(websiteUrl);
1130 const maxEmailCandidates = emailCandidateLimit(maxEmailsPerProduct);
1131 const warnings: string[] = [];
1132 const pageUrls: string[] = [];
1133 const addPageUrl = (url: string): void => {
1134 const canonical = canonicalWebsiteUrl(url);
1135 if (canonical && !pageUrls.includes(canonical) && pageUrls.length < maxWebsitePages) pageUrls.push(canonical);
1136 };
1137 let canonicalOrigin = initialUrl.origin;
1138 let baseDomain = initialUrl.hostname.replace(/^www\./i, '');
1139 let emails: string[] = [];
1140 let coverage: Coverage = 'complete';
1141 let pagePlan: WebsitePage[] = [];
1142
1143 const staticHomepage = await fetchPublicHtml(`${initialUrl.origin}/`, 4_000);
1144 if (staticHomepage.ok) {
1145 const finalHomepage = new URL(staticHomepage.url);
1146 canonicalOrigin = finalHomepage.origin;
1147 baseDomain = finalHomepage.hostname.replace(/^www\./i, '');
1148 addPageUrl(staticHomepage.url);
1149 emails = rankEmails(extractStaticEmails(staticHomepage.html), baseDomain, maxEmailCandidates);
1150 pagePlan = buildWebsitePagePlan(`${canonicalOrigin}/`, extractStaticFallbackPages(staticHomepage.html, canonicalOrigin), maxWebsitePages);
1151 for (const target of pagePlan.slice(1)) {
1152 if (emails.length >= maxEmailCandidates) break;
1153 const staticPage = await fetchPublicHtml(target.url, 3_000);
1154 if (!staticPage.ok) {
1155 coverage = 'partial';
1156 warnings.push(`${target.kind} static assessment failed: ${staticPage.reason}.`);
1157 continue;
1158 }
1159 const finalPage = new URL(staticPage.url);
1160 if (finalPage.origin !== canonicalOrigin) {
1161 coverage = 'partial';
1162 warnings.push(`${target.kind} redirected off the verified homepage origin; it was not extracted.`);
1163 continue;
1164 }
1165 addPageUrl(staticPage.url);
1166 emails = rankEmails([...emails, ...extractStaticEmails(staticPage.html)], baseDomain, maxEmailCandidates);
1167 }
1168 } else {
1169 coverage = 'partial';
1170 warnings.push(`Homepage static assessment failed: ${staticHomepage.reason}.`);
1171 }
1172
1173 if (emails.length < maxEmailCandidates) await withBrowserContextSlot(async () => {
1174 const context = await (await ensureBrowser()).newContext({
1175 userAgent: WEBSITE_USER_AGENT,
1176 locale: 'en-US',
1177 viewport: { width: 1280, height: 800 },
1178 extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' },
1179 });
1180 await context.route('**/*', async (route) => {
1181 if (['image', 'media', 'font'].includes(route.request().resourceType())) return route.abort();
1182 const requestUrl = route.request().url();
1183 if (!/^(?:data|blob|about):/i.test(requestUrl) && !await isSafePublicNetworkUrl(requestUrl)) return route.abort();
1184 return route.continue();
1185 });
1186 try {
1187 const page = await context.newPage();
1188 const response = await page.goto(`${canonicalOrigin}/`, { waitUntil: 'domcontentloaded', timeout: 6_000 });
1189 const finalHomepage = new URL(page.url());
1190 if (!response || finalHomepage.origin !== canonicalOrigin) {
1191 coverage = 'partial';
1192 warnings.push('Homepage DOM assessment redirected off origin or did not return a response.');
1193 } else {
1194 addPageUrl(finalHomepage.toString());
1195 await page.locator('body').waitFor({ state: 'attached', timeout: 750 }).catch(() => undefined);
1196 emails = rankEmails([...emails, ...await page.evaluate(browserExtractEmails)], baseDomain, maxEmailCandidates);
1197 const browserFallbackPages = await page.evaluate(browserExtractLinkedFallbackPages, canonicalOrigin);
1198 pagePlan = buildWebsitePagePlan(
1199 `${canonicalOrigin}/`,
1200 [...pagePlan.slice(1), ...browserFallbackPages],
1201 maxWebsitePages,
1202 );
1203 for (const target of pagePlan.slice(1)) {
1204 if (emails.length >= maxEmailCandidates) break;
1205 try {
1206 const fallbackResponse = await page.goto(target.url, { waitUntil: 'domcontentloaded', timeout: 4_500 });
1207 const finalFallback = new URL(page.url());
1208 if (!fallbackResponse || finalFallback.origin !== canonicalOrigin) {
1209 coverage = 'partial';
1210 warnings.push(`${target.kind} DOM assessment redirected off origin.`);
1211 continue;
1212 }
1213 addPageUrl(finalFallback.toString());
1214 await page.locator('body').waitFor({ state: 'attached', timeout: 750 }).catch(() => undefined);
1215 emails = rankEmails([...emails, ...await page.evaluate(browserExtractEmails)], baseDomain, maxEmailCandidates);
1216 } catch (error) {
1217 coverage = 'partial';
1218 const message = error instanceof Error ? error.message : 'navigation error';
1219 warnings.push(`${target.kind} DOM assessment failed: ${message.slice(0, 90)}.`);
1220 }
1221 }
1222 }
1223 } catch (error) {
1224 coverage = 'partial';
1225 const message = error instanceof Error ? error.message : 'browser error';
1226 warnings.push(`Homepage DOM assessment failed: ${message.slice(0, 90)}.`);
1227 } finally {
1228 await context.close();
1229 }
1230 });
1231
1232 const result: EmailResult = emails.length > 0
1233 ? 'emails_found'
1234 : pageUrls.length > 0 && coverage === 'complete'
1235 ? 'no_public_email_found'
1236 : 'not_assessed';
1237 return {
1238 emails,
1239 emailSource: emails.length > 0 ? 'page_scrape' : 'none',
1240 result,
1241 coverage: pageUrls.length === 0 ? 'none' : coverage,
1242 terminalReason: emails.length >= maxEmailCandidates
1243 ? 'max_public_email_candidates_reached'
1244 : result === 'no_public_email_found'
1245 ? 'eligible_pages_exhausted'
1246 : 'eligible_pages_partially_assessed',
1247 pagesVisited: pageUrls.length,
1248 pageUrls,
1249 warnings,
1250 };
1251}
1252
1253async function findMakerHomepageEmails(
1254 makerWebsiteUrls: string[],
1255 productWebsiteEmails: string[],
1256 maxEmailsPerProduct: number,
1257 filledEmailSlots: number,
1258): Promise<{ emails: string[]; pageUrls: string[]; warnings: string[] }> {
1259 const candidates: string[] = [];
1260 const pageUrls: string[] = [];
1261 const warnings: string[] = [];
1262 const seenOrigins = new Set<string>();
1263
1264 for (const rawUrl of makerWebsiteUrls) {
1265 if (selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots).length
1266 >= Math.max(0, maxEmailsPerProduct - filledEmailSlots)) break;
1267 const websiteUrl = canonicalWebsiteUrl(rawUrl);
1268 if (!websiteUrl || !isAllowedExternalWebsite(websiteUrl)) continue;
1269 const origin = new URL(websiteUrl).origin;
1270 if (seenOrigins.has(origin) || seenOrigins.size >= 2) continue;
1271 seenOrigins.add(origin);
1272
1273 const staticPage = await fetchPublicHtml(`${origin}/`, 4_000);
1274 if (staticPage.ok) {
1275 pageUrls.push(staticPage.url);
1276 const emailDomain = new URL(staticPage.url).hostname.replace(/^www\./i, '');
1277 candidates.push(...rankEmails(extractStaticEmails(staticPage.html), emailDomain, maxEmailsPerProduct));
1278 if (selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots).length > 0) continue;
1279 } else {
1280 warnings.push(`Maker homepage static assessment failed: ${staticPage.reason}.`);
1281 }
1282
1283 await withBrowserContextSlot(async () => {
1284 const context = await (await ensureBrowser()).newContext({
1285 userAgent: WEBSITE_USER_AGENT,
1286 locale: 'en-US',
1287 viewport: { width: 1280, height: 800 },
1288 extraHTTPHeaders: { 'Accept-Language': 'en-US,en;q=0.9' },
1289 });
1290 await context.route('**/*', async (route) => {
1291 if (['image', 'media', 'font'].includes(route.request().resourceType())) return route.abort();
1292 const requestUrl = route.request().url();
1293 if (!/^(?:data|blob|about):/i.test(requestUrl) && !await isSafePublicNetworkUrl(requestUrl)) return route.abort();
1294 return route.continue();
1295 });
1296 try {
1297 const page = await context.newPage();
1298 const response = await page.goto(`${origin}/`, { waitUntil: 'domcontentloaded', timeout: 10_000 });
1299 if (!response || response.status() >= 400 || new URL(page.url()).origin !== origin) {
1300 warnings.push('Maker homepage browser assessment did not reach the approved origin.');
1301 return;
1302 }
1303 if (!pageUrls.includes(page.url())) pageUrls.push(page.url());
1304 const emailDomain = new URL(page.url()).hostname.replace(/^www\./i, '');
1305 candidates.push(...rankEmails(await page.evaluate(browserExtractEmails), emailDomain, maxEmailsPerProduct));
1306 } catch {
1307 warnings.push('Maker homepage browser assessment failed.');
1308 } finally {
1309 await context.close();
1310 }
1311 });
1312 }
1313
1314 return {
1315 emails: selectMakerFallbackEmails(productWebsiteEmails, candidates, maxEmailsPerProduct, filledEmailSlots),
1316 pageUrls,
1317 warnings,
1318 };
1319}
1320
1321const POST_FIELDS = `
1322 id name tagline description votesCount commentsCount url website slug
1323 featuredAt createdAt dailyRank weeklyRank monthlyRank yearlyRank
1324 reviewsCount reviewsRating
1325 thumbnail { url }
1326 productLinks { type url }
1327 topics(first: 10) { edges { node { name } } }
1328 makers { id name username twitterUsername url websiteUrl }
1329`;
1330
1331const POSTS_QUERY = `query GetPosts($first:Int $after:String $order:PostsOrder $featured:Boolean $postedAfter:DateTime $postedBefore:DateTime $topic:String) {
1332 posts(first:$first after:$after order:$order featured:$featured postedAfter:$postedAfter postedBefore:$postedBefore topic:$topic) {
1333 edges { node { ${POST_FIELDS} } }
1334 pageInfo { endCursor hasNextPage }
1335 }
1336}`;
1337
1338const POSTS_INDEX_QUERY = `query GetPostIndex($first:Int $after:String $order:PostsOrder $featured:Boolean $postedAfter:DateTime $postedBefore:DateTime) {
1339 posts(first:$first after:$after order:$order featured:$featured postedAfter:$postedAfter postedBefore:$postedBefore) {
1340 edges { node { id url slug } }
1341 pageInfo { endCursor hasNextPage }
1342 }
1343}`;
1344
1345const POST_BY_SLUG_QUERY = `query GetPost($slug:String!) { post(slug:$slug) { ${POST_FIELDS} } }`;
1346
1347function toInt(value: unknown): number | null {
1348 if (value === null || value === undefined || value === '') return null;
1349 const parsed = Number.parseInt(String(value), 10);
1350 return Number.isNaN(parsed) ? null : parsed;
1351}
1352
1353function toFloat(value: unknown): number | null {
1354 if (value === null || value === undefined || value === '') return null;
1355 const parsed = Number.parseFloat(String(value));
1356 return Number.isNaN(parsed) ? null : parsed;
1357}
1358
1359function nodeToRecord(node: Record<string, any>, rank: number): Record<string, unknown> {
1360 const makers = (node.makers ?? []).map((maker: Record<string, unknown>) => ({
1361 maker_name: maker.name ?? null,
1362 maker_id: maker.username ?? null,
1363 maker_ph_id: maker.id ?? null,
1364 twitter_url: maker.twitterUsername ? `https://twitter.com/${String(maker.twitterUsername)}` : null,
1365 maker_url: maker.url ?? null,
1366 website_url: maker.websiteUrl ?? null,
1367 }));
1368 const topics = (node.topics?.edges ?? []).map((edge: any) => edge?.node?.name).filter(Boolean);
1369 const productLinks = Array.isArray(node.productLinks) ? node.productLinks : [];
1370 const directWebsite = productLinks.find((link: Record<string, unknown>) => {
1371 const type = String(link.type ?? '').toLowerCase();
1372 return type === 'website' && typeof link.url === 'string' && isAllowedExternalWebsite(link.url);
1373 })?.url as string | undefined;
1374
1375 const websiteUrl = directWebsite ? canonicalWebsiteUrl(directWebsite) : null;
1376 return {
1377 product_name: node.name ? String(node.name).trim() : null,
1378 tagline: node.tagline ?? null,
1379 description: node.description ?? null,
1380 upvote_count: toInt(node.votesCount),
1381 comment_count: toInt(node.commentsCount),
1382 reviews_count: toInt(node.reviewsCount),
1383 reviews_rating: toFloat(node.reviewsRating),
1384 daily_rank: toInt(node.dailyRank) ?? rank,
1385 weekly_rank: toInt(node.weeklyRank),
1386 monthly_rank: toInt(node.monthlyRank),
1387 yearly_rank: toInt(node.yearlyRank),
1388 launch_date: node.featuredAt ? isoDate(new Date(node.featuredAt)) : node.createdAt ? isoDate(new Date(node.createdAt)) : null,
1389 product_hunt_url: node.url ?? (node.slug ? `https://www.producthunt.com/posts/${node.slug}` : null),
1390 product_hunt_website_url: node.website ?? null,
1391 website_url: websiteUrl,
1392 website_resolution_source: websiteUrl ? 'api_website_product_link' satisfies WebsiteResolutionSource : 'none' satisfies WebsiteResolutionSource,
1393 website_resolution_status: websiteUrl
1394 ? 'verified_external' satisfies WebsiteResolutionStatus
1395 : node.website
1396 ? 'unresolved' satisfies WebsiteResolutionStatus
1397 : 'not_available' satisfies WebsiteResolutionStatus,
1398 website_resolution_confidence: websiteUrl ? 100 : 0,
1399 website_resolution_signals: websiteUrl ? ['official_api_product_link'] : [],
1400 website_resolution_provider_cost_usd: 0,
1401 company_domain: websiteUrl ? companyDomainFromWebsite(websiteUrl) : null,
1402 topics,
1403 thumbnail_url: node.thumbnail?.url ?? null,
1404 makers,
1405 featured: Boolean(node.featuredAt),
1406 emails: [] as string[],
1407 verified_emails: [] as string[],
1408 risky_emails: [] as string[],
1409 rejected_emails: [] as string[],
1410 rejected_email_verifications: [] as EmailVerification[],
1411 email_verifications: [] as EmailVerification[],
1412 email_verification_requested: false,
1413 email_source: 'none' as EmailSource,
1414 email_result: 'not_assessed' as EmailResult,
1415 email_coverage: 'none' as Coverage,
1416 email_terminal_reason: 'website_not_available',
1417 email_pages_visited: 0,
1418 email_page_urls: [] as string[],
1419 maker_email_pages_visited: 0,
1420 maker_email_page_urls: [] as string[],
1421 contact_discovery_requested: false,
1422 contact_discovery_attempts: 0,
1423 contact_discovery_credits: 0,
1424 lead_status: 'unresolved_website',
1425 lead_score: 0,
1426 lead_tier: 'unqualified',
1427 lead_signals: [] as string[],
1428 enrichment_warnings: [] as string[],
1429 scraped_at: new Date().toISOString(),
1430 };
1431}
1432
1433const LEAN_FIELDS = new Set([
1434 'product_name', 'tagline', 'website_url', 'company_domain', 'product_hunt_website_url', 'website_resolution_status',
1435 'website_resolution_source', 'website_resolution_confidence', 'upvote_count', 'daily_rank', 'emails', 'verified_emails',
1436 'risky_emails', 'rejected_emails', 'email_verifications', 'rejected_email_verifications', 'email_verification_requested', 'email_source', 'email_result', 'email_coverage',
1437 'email_pages_visited', 'contact_discovery_attempts', 'lead_status', 'lead_score', 'lead_tier', 'lead_signals',
1438 'maker_email_pages_visited',
1439 'topics', 'launch_date', 'product_hunt_url', 'featured', 'scraped_at',
1440]);
1441
1442function applyOutputMode(record: Record<string, unknown>, outputMode: 'full' | 'lean' | 'leads'): Record<string, unknown> | null {
1443 if (outputMode === 'leads' && (!Array.isArray(record.emails) || record.emails.length === 0)) return null;
1444 if (outputMode !== 'lean') return record;
1445 const output: Record<string, unknown> = {};
1446 for (const field of LEAN_FIELDS) output[field] = record[field] ?? null;
1447 return output;
1448}
1449
1450function validateOutputRecord(record: Record<string, unknown>): string | null {
1451 if (typeof record.product_name !== 'string' || !record.product_name.trim()) return 'product_name is missing.';
1452 if (typeof record.product_hunt_url !== 'string' || !record.product_hunt_url.startsWith('https://')) return 'product_hunt_url is missing or invalid.';
1453 if (!Array.isArray(record.emails) || !record.emails.every((email) => typeof email === 'string')) return 'emails must be a string array.';
1454 if (typeof record.scraped_at !== 'string' || Number.isNaN(Date.parse(record.scraped_at))) return 'scraped_at is invalid.';
1455 return null;
1456}
1457
1458async function run(): Promise<void> {
1459 const rawInput = (await Actor.getInput() ?? {}) as Record<string, unknown>;
1460 const mode = (rawInput.mode ?? rawInput.scrapeMode ?? 'leaderboard') as Mode;
1461 const leaderboardPeriod = (rawInput.leaderboardPeriod ?? rawInput.period ?? 'daily') as LeaderboardPeriod;
1462 const startDate = String(rawInput.startDate ?? rawInput.date ?? '');
1463 const endDate = String(rawInput.endDate ?? '');
1464 const lookbackDays = clampInteger(rawInput.lookbackDays, 1, 1, 365);
1465 const outputMode = (rawInput.outputMode ?? 'full') as 'full' | 'lean' | 'leads';
1466 const searchQuery = typeof (rawInput.searchQuery ?? rawInput.query ?? rawInput.q) === 'string'
1467 ? String(rawInput.searchQuery ?? rawInput.query ?? rawInput.q).trim()
1468 : '';
1469 const topic = typeof (rawInput.topic ?? rawInput.topicSlug) === 'string'
1470 ? String(rawInput.topic ?? rawInput.topicSlug).trim()
1471 : '';
1472 const startUrls = Array.isArray(rawInput.startUrls) ? rawInput.startUrls as Array<{ url?: unknown } | string> : [];
1473 const maxResults = clampInteger(rawInput.maxResults ?? rawInput.maxItems ?? rawInput.limit, 100, 1, 20_000);
1474 const maxConcurrency = clampInteger(rawInput.maxConcurrency, 3, 1, 5);
1475 const maxWebsitePages = clampInteger(rawInput.maxWebsitePages, DEFAULT_MAX_WEBSITE_PAGES, 1, MAX_WEBSITE_PAGES);
1476 const maxEmailsPerProduct = clampInteger(rawInput.maxEmailsPerProduct, DEFAULT_MAX_EMAILS_PER_PRODUCT, 1, MAX_EMAILS_PER_PRODUCT);
1477 const includeAllProducts = Boolean(rawInput.includeAllProducts ?? rawInput.allProducts ?? rawInput.includeAll ?? true);
1478 const enrichEmails = Boolean(rawInput.enrichEmails ?? rawInput.findEmails ?? true);
1479 const resolveWebsites = Boolean(rawInput.resolveWebsites ?? true);
1480 const verifyEmails = Boolean(rawInput.verifyEmails ?? rawInput.verifyWithEmailListVerify ?? true);
1481 const findContacts = Boolean(rawInput.findContacts ?? true);
1482 const emailListVerifyApiKey = String(process.env.EMAILLISTVERIFY_API_KEY ?? '').trim();
1483 const phApiToken = String(process.env.PH_API_TOKEN ?? '');
1484 const dataForSeoLogin = String(process.env.DATAFORSEO_LOGIN ?? '').trim();
1485 const dataForSeoPassword = String(process.env.DATAFORSEO_PASSWORD ?? '').trim();
1486 const serpApiKey = String(process.env.SERPAPI_KEY ?? '').trim();
1487 const websiteOverrides = parseWebsiteOverrides(rawInput.websiteUrlOverrides);
1488 const excludedProductHuntUrls = Array.isArray(rawInput.excludeProductHuntUrls)
1489 ? rawInput.excludeProductHuntUrls
1490 .slice(0, 20_000)
1491 .map((value) => productHuntLaunchKey(String(value ?? '')))
1492 .filter((value): value is string => Boolean(value))
1493 : [];
1494 const excludedProductHuntKeys = new Set(excludedProductHuntUrls);
1495 const productHuntResumeCursor = typeof rawInput.productHuntResumeCursor === 'string'
1496 ? rawInput.productHuntResumeCursor.trim().slice(0, 2_000)
1497 : '';
1498 const startedAt = new Date().toISOString();
1499 emailListVerifyCreditsExhausted = false;
1500
1501 const diagnostics = {
1502 itemsPushed: 0,
1503 itemsProcessed: 0,
1504 itemsFiltered: 0,
1505 itemsExcluded: 0,
1506 itemsFailed: 0,
1507 invalidRecords: 0,
1508 apiFailures: 0,
1509 productHuntIndexPages: 0,
1510 productHuntDetailLookups: 0,
1511 productHuntFullPageLookups: 0,
1512 nextProductHuntCursor: null as string | null,
1513 websiteResolved: 0,
1514 websiteUnresolved: 0,
1515 websiteResolutionProviderAttempts: 0,
1516 websiteResolutionProviderCostUsd: 0,
1517 emailsFound: 0,
1518 verifiedEmailsFound: 0,
1519 emailVerificationAttempts: 0,
1520 emailVerificationProviderErrors: 0,
1521 makerEmailPagesVisited: 0,
1522 makerEmailsFound: 0,
1523 contactDiscoveryAttempts: 0,
1524 contactDiscoveryCredits: 0,
1525 contactDiscoveryEmailsFound: 0,
1526 leadTiers: { A: 0, B: 0, C: 0, unqualified: 0 },
1527 resultEvents: 0,
1528 emailEvents: 0,
1529 chargeLimitReached: false,
1530 resultLimitReached: false,
1531 deadlineReached: false,
1532 itemsSkippedAtDeadline: 0,
1533 searchScanCapReached: false,
1534 emailVerificationCreditsExhausted: false,
1535 warnings: [] as string[],
1536 };
1537 let finalised = false;
1538
1539 const paidItemLimit = clampInteger(process.env.ACTOR_MAX_PAID_DATASET_ITEMS, maxResults, 1, maxResults);
1540
1541
1542 const freeCap = resolveFreePlanCap();
1543 log.info('Run plan limits', { freePlan: freeCap.isFreePlan, freePlanLimit: freeCap.isFreePlan ? freeCap.limit : null });
1544 const effectiveMaxResults = clampToFreePlan(Math.min(maxResults, paidItemLimit), freeCap);
1545 const freeCapBinding = freeCap.isFreePlan && freeCap.limit < Math.min(maxResults, paidItemLimit);
1546 let candidatesForFreePlan = 0;
1547
1548 const runDeadlineMs = Date.parse(process.env.ACTOR_TIMEOUT_AT ?? '');
1549 const hasRunDeadline = Number.isFinite(runDeadlineMs);
1550 const runBudgetMs = hasRunDeadline ? Math.max(0, runDeadlineMs - Date.now()) : Number.POSITIVE_INFINITY;
1551
1552 const newWorkReserveMs = hasRunDeadline ? Math.min(Math.max(runBudgetMs * 0.15, 45_000), 480_000) : 0;
1553 const stopNewWorkAtMs = hasRunDeadline ? runDeadlineMs - newWorkReserveMs : Number.POSITIVE_INFINITY;
1554 const hardStopAtMs = hasRunDeadline ? runDeadlineMs - 20_000 : Number.POSITIVE_INFINITY;
1555 let hardStopping = false;
1556 let inFlightWrites = 0;
1557 let currentPageStartCursor: string | null = null;
1558 let lastProgressAt = 0;
1559
1560 const addWarning = (message: string): void => {
1561 if (diagnostics.warnings.length < 100 && !diagnostics.warnings.includes(message)) diagnostics.warnings.push(message);
1562 };
1563
1564 const deadlineReached = (): boolean => {
1565 if (Date.now() < stopNewWorkAtMs) return false;
1566 if (!diagnostics.deadlineReached) {
1567 diagnostics.deadlineReached = true;
1568 addWarning('The run deadline approached, so no new products were started. Continue with productHuntResumeCursor from RUN_SUMMARY.nextProductHuntCursor (when present) and excludeProductHuntUrls from this dataset to avoid duplicate charges.');
1569 log.warning('Run deadline approaching; stopped starting new products and preserving persisted results.');
1570 }
1571 return true;
1572 };
1573
1574 const reportProgress = async (force = false): Promise<void> => {
1575 if (!force && Date.now() - lastProgressAt < 15_000) return;
1576 lastProgressAt = Date.now();
1577 await Actor.setStatusMessage(
1578 `Processed ${diagnostics.itemsProcessed}/${effectiveMaxResults} products, saved ${diagnostics.itemsPushed} rows, found ${diagnostics.emailsFound} emails.`
1579 + (emailListVerifyCreditsExhausted ? ' Email verification credits are exhausted; remaining emails are unverified.' : ''),
1580 ).catch(() => undefined);
1581 };
1582
1583 const resolveOutcome = (): RunOutcome => {
1584 diagnostics.emailVerificationCreditsExhausted = emailListVerifyCreditsExhausted;
1585 if (diagnostics.itemsPushed === 0) {
1586 const searchedWithoutMatch = mode === 'search' && (diagnostics.searchScanCapReached || diagnostics.deadlineReached);
1587 return diagnostics.apiFailures > 0 || diagnostics.itemsFailed > 0 || diagnostics.invalidRecords > 0 || (diagnostics.deadlineReached && !searchedWithoutMatch)
1588 ? 'UPSTREAM_FAILED'
1589 : 'VALID_EMPTY';
1590 }
1591 return diagnostics.apiFailures > 0 || diagnostics.itemsFailed > 0 || (resolveWebsites && diagnostics.websiteUnresolved > 0) || diagnostics.chargeLimitReached
1592 || diagnostics.resultLimitReached || diagnostics.deadlineReached || diagnostics.searchScanCapReached || (verifyEmails && emailListVerifyCreditsExhausted)
1593 ? 'PARTIAL'
1594 : 'COMPLETE';
1595 };
1596
1597 const outcomeMessage = (outcome: RunOutcome): string => {
1598 const eventCost = diagnostics.resultEvents * RESULT_PRICE_USD + diagnostics.emailEvents * EMAIL_PRICE_USD;
1599 const deadlineNote = diagnostics.deadlineReached ? ' Stopped before the run deadline; resume details are in RUN_SUMMARY.' : '';
1600 if (diagnostics.itemsPushed === 0 && mode === 'search' && (diagnostics.searchScanCapReached || diagnostics.deadlineReached)) {
1601 return diagnostics.searchScanCapReached
1602 ? `${outcome}: no Product Hunt launch matched "${searchQuery}" in the newest ${SEARCH_PAGES_WITHOUT_MATCH_LIMIT * 20} launches searched.`
1603 : `${outcome}: no Product Hunt launch matched "${searchQuery}" before the run deadline.`;
1604 }
1605 if (diagnostics.itemsPushed === 0 && diagnostics.deadlineReached) {
1606 return `${outcome}: the run deadline was reached before any Product Hunt product could be persisted. Increase the run timeout and re-run.`;
1607 }
1608 return `${outcome}: processed ${diagnostics.itemsProcessed} Product Hunt products, persisted ${diagnostics.itemsPushed} rows, and found ${diagnostics.emailsFound} public email(s). Event charges: $${eventCost.toFixed(3)} plus applicable platform usage.${deadlineNote}`;
1609 };
1610
1611 const finalizeRun = async (outcome: RunOutcome, message: string): Promise<never> => {
1612 if (finalised) throw new Error(message);
1613 finalised = true;
1614 const freePlanBase = freePlanLimitSummary(freeCap, maxResults, diagnostics.itemsPushed);
1615 const freePlanLimit = {
1616 ...freePlanBase,
1617 reached: freePlanBase.reached || (freeCapBinding && diagnostics.itemsPushed > 0 && candidatesForFreePlan >= freeCap.limit),
1618 };
1619 if (freePlanLimit.reached && !FAILED_OUTCOMES.has(outcome)) message = `${outcome}: ${freePlanStatus(diagnostics.itemsPushed, freeCap.limit)}`;
1620 if (!message.startsWith(outcome)) message = `${outcome}: ${message}`;
1621 const finishedAt = new Date().toISOString();
1622 const summary = {
1623 outcome,
1624 message,
1625 startedAt,
1626 finishedAt,
1627 buildNumber: process.env.ACTOR_BUILD_NUMBER ?? null,
1628 runId: process.env.ACTOR_RUN_ID ?? null,
1629 datasetId: process.env.ACTOR_DEFAULT_DATASET_ID ?? null,
1630 keyValueStoreId: process.env.ACTOR_DEFAULT_KEY_VALUE_STORE_ID ?? null,
1631 input: {
1632 mode, leaderboardPeriod, startDate: startDate || null, endDate: endDate || null,
1633 lookbackDays, maxResults, effectiveMaxResults, includeAllProducts, enrichEmails, resolveWebsites, verifyEmails, findContacts,
1634 excludedProductHuntUrls: excludedProductHuntKeys.size,
1635 productHuntResumeCursorUsed: Boolean(productHuntResumeCursor),
1636 emailListVerifyConfigured: Boolean(emailListVerifyApiKey), maxWebsitePages,
1637 websiteSearchConfigured: Boolean((dataForSeoLogin && dataForSeoPassword) || serpApiKey),
1638 maxEmailsPerProduct, outputMode,
1639 },
1640 ...diagnostics,
1641 chargedEventCounts: {
1642 [RESULT_EVENT]: diagnostics.resultEvents,
1643 [EMAIL_EVENT]: diagnostics.emailEvents,
1644 },
1645 freePlanLimit,
1646 };
1647 const output = {
1648 outcome,
1649 message,
1650 itemsPushed: diagnostics.itemsPushed,
1651 itemsFailed: diagnostics.itemsFailed,
1652 invalidRecords: diagnostics.invalidRecords,
1653 deadlineReached: diagnostics.deadlineReached,
1654 nextProductHuntCursor: diagnostics.nextProductHuntCursor,
1655 emailVerificationCreditsExhausted: diagnostics.emailVerificationCreditsExhausted,
1656 chargedEventCounts: summary.chargedEventCounts,
1657 warnings: diagnostics.warnings,
1658 freePlanLimit,
1659 };
1660 await Actor.setValue('OUTPUT', output);
1661 await Actor.setValue('RUN_SUMMARY', summary);
1662 const failed = FAILED_OUTCOMES.has(outcome);
1663 await Actor.setStatusMessage(message, {
1664 isStatusMessageTerminal: true,
1665 level: failed ? 'ERROR' : outcome === 'COMPLETE' ? 'INFO' : 'WARNING',
1666 });
1667 if (failed) {
1668
1669 await Actor.fail(message);
1670 }
1671 await Actor.exit();
1672 throw new Error('Actor terminal call returned unexpectedly.');
1673 };
1674
1675 try {
1676 const modes: Mode[] = ['leaderboard', 'search', 'topic', 'urls'];
1677 const periods: LeaderboardPeriod[] = ['daily', 'weekly', 'monthly', 'yearly'];
1678 const outputModes = ['full', 'lean', 'leads'];
1679 if (!modes.includes(mode)) await finalizeRun('INVALID_INPUT', 'mode must be leaderboard, search, topic, or urls.');
1680 if (!periods.includes(leaderboardPeriod)) await finalizeRun('INVALID_INPUT', 'leaderboardPeriod must be daily, weekly, monthly, or yearly.');
1681 if (!outputModes.includes(outputMode)) await finalizeRun('INVALID_INPUT', 'outputMode must be full, lean, or leads.');
1682 if (!phApiToken) await finalizeRun('CONFIG_ERROR', 'Product Hunt API token is not configured. Set the owner-managed PH_API_TOKEN secret and re-run.');
1683 if (mode === 'search' && !searchQuery) await finalizeRun('INVALID_INPUT', 'searchQuery is required for search mode.');
1684 if (mode === 'topic' && !topic) await finalizeRun('INVALID_INPUT', 'topic is required for topic mode.');
1685 if (mode === 'urls' && startUrls.length === 0) await finalizeRun('INVALID_INPUT', 'startUrls is required for urls mode.');
1686 if (startDate && !validIsoDate(startDate)) await finalizeRun('INVALID_INPUT', 'startDate must be a valid YYYY-MM-DD date.');
1687 if (endDate && !validIsoDate(endDate)) await finalizeRun('INVALID_INPUT', 'endDate must be a valid YYYY-MM-DD date.');
1688 if (startDate && endDate && startDate > endDate) await finalizeRun('INVALID_INPUT', 'endDate must be on or after startDate.');
1689 if (leaderboardPeriod !== 'daily' && endDate) await finalizeRun('INVALID_INPUT', 'endDate is supported only with daily leaderboard ranges.');
1690
1691 const maxPotentialEventCost = effectiveMaxResults * (RESULT_PRICE_USD + (enrichEmails ? maxEmailsPerProduct * EMAIL_PRICE_USD : 0));
1692 const maxPotentialVerificationCredits = enrichEmails && emailListVerifyApiKey
1693 ? effectiveMaxResults * ((verifyEmails ? maxEmailsPerProduct * 3 * 2 : 0) + (findContacts ? 15 : 0))
1694 : 0;
1695 log.info('ProductHunt Scraper starting', {
1696 mode, leaderboardPeriod, startDate: startDate || '(default)', endDate: endDate || null,
1697 lookbackDays, effectiveMaxResults, includeAllProducts, enrichEmails, resolveWebsites, verifyEmails, findContacts,
1698 emailListVerifyConfigured: Boolean(emailListVerifyApiKey), maxWebsitePages, maxEmailsPerProduct,
1699 websiteSearchProviders: {
1700 dataForSeo: Boolean(dataForSeoLogin && dataForSeoPassword),
1701 serpApiFallback: Boolean(serpApiKey),
1702 },
1703 eventCostCeilingUsd: Number(maxPotentialEventCost.toFixed(3)),
1704 excludedProductHuntUrls: excludedProductHuntKeys.size,
1705 emailListVerifyCreditCeiling: maxPotentialVerificationCredits,
1706 websiteSearchQueryCeiling: resolveWebsites
1707 ? effectiveMaxResults * ((dataForSeoLogin && dataForSeoPassword ? 2 : 0) + (serpApiKey ? 1 : 0))
1708 : 0,
1709 platformUsage: 'Additional compute and proxy usage is billed by Apify separately where applicable.',
1710 });
1711 await Actor.setStatusMessage(`Starting: up to ${effectiveMaxResults} products, ${maxWebsitePages} public website pages/product, and ${maxEmailsPerProduct} public emails/product.`);
1712 lastProgressAt = Date.now();
1713
1714 if (hasRunDeadline) {
1715 const hardStopTimer = setInterval(() => {
1716 if (Date.now() < hardStopAtMs || hardStopping || finalised) return;
1717 hardStopping = true;
1718 clearInterval(hardStopTimer);
1719 void (async () => {
1720 deadlineReached();
1721 const waitUntil = Date.now() + 10_000;
1722 while (inFlightWrites > 0 && Date.now() < waitUntil) await new Promise((resolve) => setTimeout(resolve, 200));
1723 if (currentPageStartCursor && !diagnostics.nextProductHuntCursor) diagnostics.nextProductHuntCursor = currentPageStartCursor;
1724 await closeBrowser();
1725 const outcome = resolveOutcome();
1726 await finalizeRun(outcome, outcomeMessage(outcome)).catch(() => undefined);
1727 })();
1728 }, 5_000);
1729 hardStopTimer.unref();
1730 }
1731
1732 const phQuery = async (query: string, variables: Record<string, unknown>, attempt = 0): Promise<any> => {
1733 const response = await fetch(PH_API, {
1734 method: 'POST',
1735 headers: {
1736 'Content-Type': 'application/json',
1737 Accept: 'application/json',
1738 Authorization: `Bearer ${phApiToken}`,
1739 'User-Agent': PRODUCT_HUNT_USER_AGENT,
1740 },
1741 body: JSON.stringify({ query, variables }),
1742 signal: AbortSignal.timeout(30_000),
1743 });
1744 if (response.status === 429 && attempt < 3) {
1745 const waitSeconds = productHuntRateLimitDelaySeconds(
1746 response.headers.get('x-rate-limit-reset'),
1747 response.headers.get('retry-after'),
1748 );
1749 log.warning(`Product Hunt API rate limited; waiting ${waitSeconds}s before retry ${attempt + 1}/3.`);
1750 await new Promise((resolve) => setTimeout(resolve, waitSeconds * 1000));
1751 return phQuery(query, variables, attempt + 1);
1752 }
1753 if (!response.ok) throw new Error(`Product Hunt API returned HTTP ${response.status}.`);
1754 const payload = await response.json() as { data?: unknown; errors?: Array<{ message?: string }> };
1755 if (payload.errors?.length) throw new Error(`Product Hunt GraphQL: ${payload.errors[0]?.message ?? 'unknown error'}`);
1756 return payload.data as any;
1757 };
1758
1759 const seenIds = new Set<string>();
1760 let collectedCandidates = 0;
1761
1762 const pushNode = async (node: Record<string, any>, rank: number): Promise<void> => {
1763 const record = nodeToRecord(node, rank);
1764 record.email_verification_requested = verifyEmails;
1765 const override = lookupWebsiteOverride(node, websiteOverrides);
1766 if (override) {
1767 record.website_url = override;
1768 record.website_resolution_source = 'input_override' satisfies WebsiteResolutionSource;
1769 record.website_resolution_status = 'verified_external' satisfies WebsiteResolutionStatus;
1770 record.website_resolution_confidence = 100;
1771 record.website_resolution_signals = ['approved_input_override'];
1772 }
1773
1774 let websiteUrl = typeof record.website_url === 'string' ? record.website_url : '';
1775 if (!websiteUrl && resolveWebsites) {
1776 const resolution = await resolveWebsite({
1777 productName: String(record.product_name ?? ''),
1778 tagline: String(record.tagline ?? ''),
1779 description: String(record.description ?? ''),
1780 productHuntRedirect: typeof record.product_hunt_website_url === 'string' ? record.product_hunt_website_url : '',
1781 makerWebsiteUrls: (node.makers ?? [])
1782 .map((maker: Record<string, unknown>) => typeof maker.websiteUrl === 'string' ? maker.websiteUrl : '')
1783 .filter(Boolean),
1784 dataForSeoLogin,
1785 dataForSeoPassword,
1786 serpApiKey,
1787 });
1788 diagnostics.websiteResolutionProviderAttempts += resolution.providerAttempts;
1789 diagnostics.websiteResolutionProviderCostUsd += resolution.providerCostUsd;
1790 resolution.warnings.forEach(addWarning);
1791 record.website_resolution_provider_cost_usd = Number(resolution.providerCostUsd.toFixed(6));
1792 if (resolution.websiteUrl) {
1793 websiteUrl = resolution.websiteUrl;
1794 record.website_url = resolution.websiteUrl;
1795 record.website_resolution_source = resolution.source;
1796 record.website_resolution_status = resolution.status;
1797 record.website_resolution_confidence = resolution.confidence;
1798 record.website_resolution_signals = resolution.signals;
1799 }
1800 }
1801
1802 if (websiteUrl) {
1803 diagnostics.websiteResolved += 1;
1804 record.company_domain = companyDomainFromWebsite(websiteUrl);
1805 } else {
1806 diagnostics.websiteUnresolved += 1;
1807 record.website_resolution_status = record.product_hunt_website_url ? 'unresolved' : 'not_available';
1808 }
1809
1810 let publicEmailCount = 0;
1811 let publicVerifications: EmailVerification[] = [];
1812 if (enrichEmails && websiteUrl) {
1813 const enriched = await findEmails(websiteUrl, maxWebsitePages, maxEmailsPerProduct);
1814 record.emails = enriched.emails;
1815 record.email_source = enriched.emailSource;
1816 record.email_result = enriched.result;
1817 record.email_coverage = enriched.coverage;
1818 record.email_terminal_reason = enriched.terminalReason;
1819 record.email_pages_visited = enriched.pagesVisited;
1820 record.email_page_urls = enriched.pageUrls;
1821 record.enrichment_warnings = enriched.warnings;
1822 publicEmailCount = enriched.emails.length;
1823 enriched.warnings.forEach(addWarning);
1824 log.info('Website enrichment complete', {
1825 product: record.product_name,
1826 emailsFound: enriched.emails.length,
1827 source: enriched.emailSource,
1828 pagesVisited: enriched.pagesVisited,
1829 coverage: enriched.coverage,
1830 terminalReason: enriched.terminalReason,
1831 });
1832 }
1833
1834 const selectedEmails = Array.isArray(record.emails) ? record.emails.filter((email): email is string => typeof email === 'string') : [];
1835 if (selectedEmails.length > 0) {
1836 const verification = await verifyEmailsWithEmailListVerify(
1837 selectedEmails,
1838 verifyEmails,
1839 emailListVerifyApiKey,
1840 'public_website',
1841 maxEmailsPerProduct,
1842 );
1843 publicVerifications = verification.verifications;
1844 diagnostics.emailVerificationAttempts += verification.attempts;
1845 diagnostics.emailVerificationProviderErrors += verification.verifications.filter(({ status }) => status === 'provider_error' || status === 'error_credit').length;
1846 verification.warnings.forEach(addWarning);
1847 log.info('Public email verification complete', {
1848 product: record.product_name,
1849 emailsAssessed: selectedEmails.length,
1850 requested: verifyEmails,
1851 attempts: verification.attempts,
1852 statuses: verification.verifications.map(({ status }) => status),
1853 });
1854 }
1855
1856 const makerWebsiteUrls = (node.makers ?? [])
1857 .map((maker: Record<string, unknown>) => typeof maker.websiteUrl === 'string' ? maker.websiteUrl : '')
1858 .filter(Boolean);
1859 const productDeliverableCount = verifyEmails
1860 ? publicVerifications.filter(({ status }) => status === 'ok').length
1861 : publicVerifications.filter(({ status }) => isUsableEmailStatus(status)).length;
1862 if (enrichEmails && productDeliverableCount < maxEmailsPerProduct && makerWebsiteUrls.length > 0) {
1863 const makerEnrichment = await findMakerHomepageEmails(
1864 makerWebsiteUrls,
1865 selectedEmails,
1866 maxEmailsPerProduct,
1867 productDeliverableCount,
1868 );
1869 record.maker_email_pages_visited = makerEnrichment.pageUrls.length;
1870 record.maker_email_page_urls = makerEnrichment.pageUrls;
1871 diagnostics.makerEmailPagesVisited += makerEnrichment.pageUrls.length;
1872 makerEnrichment.warnings.forEach(addWarning);
1873 record.enrichment_warnings = [
1874 ...new Set([
1875 ...(Array.isArray(record.enrichment_warnings) ? record.enrichment_warnings as string[] : []),
1876 ...makerEnrichment.warnings,
1877 ]),
1878 ];
1879 if (makerEnrichment.emails.length > 0) {
1880 const makerVerification = await verifyEmailsWithEmailListVerify(
1881 makerEnrichment.emails,
1882 verifyEmails,
1883 emailListVerifyApiKey,
1884 'maker_website',
1885 );
1886 publicVerifications.push(...makerVerification.verifications);
1887 publicEmailCount += makerEnrichment.emails.length;
1888 diagnostics.makerEmailsFound += makerEnrichment.emails.length;
1889 diagnostics.emailVerificationAttempts += makerVerification.attempts;
1890 diagnostics.emailVerificationProviderErrors += makerVerification.verifications
1891 .filter(({ status }) => status === 'provider_error' || status === 'error_credit').length;
1892 makerVerification.warnings.forEach(addWarning);
1893 log.info('Maker homepage email fallback complete', {
1894 product: record.product_name,
1895 emailsFound: makerEnrichment.emails.length,
1896 pagesVisited: makerEnrichment.pageUrls.length,
1897 statuses: makerVerification.verifications.map(({ status }) => status),
1898 });
1899 }
1900 }
1901
1902 let discoveredContacts: ContactFinderResult[] = [];
1903 const deliverablePublicCount = verifyEmails
1904 ? publicVerifications.filter(({ status }) => status === 'ok').length
1905 : publicVerifications.filter(({ status }) => isUsableEmailStatus(status)).length;
1906 if (enrichEmails && findContacts && websiteUrl && deliverablePublicCount < maxEmailsPerProduct) {
1907 record.contact_discovery_requested = true;
1908 const discovery = await discoverContacts(
1909 websiteUrl,
1910 (node.makers ?? []).map((maker: Record<string, unknown>) => String(maker.name ?? '')).filter(Boolean),
1911 Math.max(1, maxEmailsPerProduct - deliverablePublicCount),
1912 emailListVerifyApiKey,
1913 );
1914 discoveredContacts = discovery.contacts;
1915 record.contact_discovery_attempts = discovery.attempts;
1916 record.contact_discovery_credits = discovery.credits;
1917 diagnostics.contactDiscoveryAttempts += discovery.attempts;
1918 diagnostics.contactDiscoveryCredits += discovery.credits;
1919 diagnostics.contactDiscoveryEmailsFound += discovery.contacts.length;
1920 discovery.warnings.forEach(addWarning);
1921 }
1922
1923 const finalVerifications = selectBestEmails(publicVerifications, discoveredContacts, maxEmailsPerProduct);
1924 const rejectedVerifications = publicVerifications.filter(({ status }) => !isUsableEmailStatus(status));
1925 const finalEmails = finalVerifications.map(({ email }) => email);
1926 const publicSelected = finalVerifications.filter(({ source }) => source === 'public_website').length;
1927 const makerSelected = finalVerifications.filter(({ source }) => source === 'maker_website').length;
1928 const finderSelected = finalVerifications.filter(({ source }) => source === 'emaillistverify_contact_finder').length;
1929 record.emails = finalEmails;
1930 record.verified_emails = finalVerifications.filter(({ status }) => status === 'ok').map(({ email }) => email);
1931 record.risky_emails = finalVerifications.filter(({ status }) => status !== 'ok').map(({ email }) => email);
1932 record.rejected_emails = rejectedVerifications.map(({ email }) => email);
1933 record.email_verifications = finalVerifications;
1934 record.rejected_email_verifications = rejectedVerifications;
1935 const selectedSourceCount = [publicSelected, makerSelected, finderSelected].filter((count) => count > 0).length;
1936 record.email_source = selectedSourceCount > 1
1937 ? 'mixed'
1938 : publicSelected > 0
1939 ? 'page_scrape'
1940 : makerSelected > 0
1941 ? 'maker_website'
1942 : finderSelected > 0
1943 ? 'emaillistverify_contact_finder'
1944 : 'none';
1945 record.email_result = finalEmails.length > 0 ? 'emails_found' : record.email_result;
1946 if (finderSelected > 0) record.email_terminal_reason = 'contact_finder_enriched';
1947 else if (makerSelected > 0) record.email_terminal_reason = 'maker_homepage_enriched';
1948 diagnostics.emailsFound += finalEmails.length;
1949 diagnostics.verifiedEmailsFound += (record.verified_emails as string[]).length;
1950
1951 const lead = buildLeadProfile({
1952 websiteResolved: Boolean(websiteUrl),
1953 websiteConfidence: Number(record.website_resolution_confidence ?? 0),
1954 emailStatuses: finalVerifications.map(({ status }) => status),
1955 publicEmailCount,
1956 upvotes: Number(record.upvote_count ?? 0),
1957 pagesVisited: Number(record.email_pages_visited ?? 0),
1958 });
1959 record.lead_status = lead.leadStatus;
1960 record.lead_score = lead.leadScore;
1961 record.lead_tier = lead.leadTier;
1962 record.lead_signals = lead.signals;
1963 diagnostics.leadTiers[lead.leadTier] += 1;
1964
1965 log.info('Lead enrichment complete', {
1966 product: record.product_name,
1967 websiteResolutionSource: record.website_resolution_source,
1968 websiteResolutionConfidence: record.website_resolution_confidence,
1969 publicEmailsFound: publicEmailCount,
1970 contactFinderEmailsFound: discoveredContacts.length,
1971 finalEmails: finalEmails.length,
1972 verifiedEmails: (record.verified_emails as string[]).length,
1973 leadTier: lead.leadTier,
1974 });
1975
1976 const outputRecord = applyOutputMode(record, outputMode);
1977 if (!outputRecord) {
1978 diagnostics.itemsFiltered += 1;
1979 return;
1980 }
1981 const validationError = validateOutputRecord(outputRecord);
1982 if (validationError) {
1983 diagnostics.invalidRecords += 1;
1984 diagnostics.itemsFailed += 1;
1985 addWarning(`A Product Hunt record was not persisted: ${validationError}`);
1986 return;
1987 }
1988
1989 if (hardStopping) {
1990 diagnostics.itemsSkippedAtDeadline += 1;
1991 return;
1992 }
1993
1994 if (freePlanLimitHit(diagnostics.itemsPushed + inFlightWrites, freeCap)) return;
1995 inFlightWrites += 1;
1996 try {
1997
1998
1999
2000 const primaryResult = await Actor.pushData(outputRecord) as unknown as { chargedCount?: number; eventChargeLimitReached?: boolean };
2001 if (primaryResult.eventChargeLimitReached && (primaryResult.chargedCount ?? 0) === 0) {
2002 diagnostics.chargeLimitReached = true;
2003 addWarning('The event-charge limit was reached before this validated row could be persisted.');
2004 return;
2005 }
2006 diagnostics.itemsPushed += 1;
2007 diagnostics.resultEvents += primaryResult.chargedCount ?? 1;
2008 diagnostics.chargeLimitReached ||= Boolean(primaryResult.eventChargeLimitReached);
2009
2010 const emails = outputRecord.emails as string[];
2011 if (emails.length > 0 && !diagnostics.chargeLimitReached) {
2012 const emailResult = await Actor.charge({ eventName: EMAIL_EVENT, count: emails.length }) as unknown as { chargedCount?: number; eventChargeLimitReached?: boolean };
2013 diagnostics.emailEvents += emailResult.chargedCount ?? emails.length;
2014 diagnostics.chargeLimitReached ||= Boolean(emailResult.eventChargeLimitReached);
2015 if (emailResult.eventChargeLimitReached) addWarning('The event-charge limit was reached after a persisted record; remaining work stopped at the configured cap.');
2016 }
2017 } catch (error) {
2018 diagnostics.itemsFailed += 1;
2019 const message = error instanceof Error ? error.message : 'Unknown dataset-write failure.';
2020 addWarning(`Validated dataset write failed: ${message.slice(0, 120)}`);
2021 log.warning(`Validated dataset write failed for ${String(record.product_name)}: ${message.slice(0, 120)}`);
2022 } finally {
2023 inFlightWrites -= 1;
2024 }
2025 };
2026
2027 const processNodes = async (nodes: Array<Record<string, any>>): Promise<void> => {
2028 const uniqueNodes: Array<Record<string, any>> = [];
2029 for (const node of nodes) {
2030 if (collectedCandidates >= effectiveMaxResults || diagnostics.chargeLimitReached) break;
2031 const id = String(node.id ?? '');
2032 if (!id || seenIds.has(id)) continue;
2033 seenIds.add(id);
2034 const launchKey = productHuntLaunchKey(String(node.url ?? ''));
2035 if (launchKey && excludedProductHuntKeys.has(launchKey)) {
2036 diagnostics.itemsExcluded += 1;
2037 continue;
2038 }
2039 uniqueNodes.push(node);
2040 collectedCandidates += 1;
2041 candidatesForFreePlan = collectedCandidates;
2042 }
2043 const rankBase = diagnostics.itemsProcessed;
2044 let nextIndex = 0;
2045 const worker = async (): Promise<void> => {
2046 while (nextIndex < uniqueNodes.length && !diagnostics.chargeLimitReached && !hardStopping
2047 && !freePlanLimitHit(diagnostics.itemsPushed, freeCap)) {
2048 if (deadlineReached()) break;
2049 const index = nextIndex;
2050 nextIndex += 1;
2051 try {
2052 await pushNode(uniqueNodes[index], rankBase + index + 1);
2053 } catch (error) {
2054 diagnostics.itemsFailed += 1;
2055 const message = error instanceof Error ? error.message : 'Unknown product enrichment failure.';
2056 addWarning(`A Product Hunt product could not be enriched: ${message.slice(0, 120)}`);
2057 log.warning(`Product enrichment failed for ${String(uniqueNodes[index]?.name ?? 'unknown')}: ${message.slice(0, 120)}`);
2058 }
2059 diagnostics.itemsProcessed += 1;
2060 await reportProgress();
2061 }
2062 };
2063 await Promise.all(Array.from({ length: Math.min(maxConcurrency, uniqueNodes.length) }, () => worker()));
2064 };
2065
2066 const streamPosts = async (range: DateRange | null, label: string): Promise<void> => {
2067 let cursor: string | null = productHuntResumeCursor || null;
2068 let pageNumber = 1;
2069 let pagesWithoutSearchMatch = 0;
2070 let useLightweightIndex = shouldUseLightweightProductHuntIndex(
2071 mode,
2072 excludedProductHuntKeys.size,
2073 productHuntResumeCursor,
2074 );
2075 while (collectedCandidates < effectiveMaxResults && !diagnostics.chargeLimitReached) {
2076 if (deadlineReached() || hardStopping) break;
2077 const pageStartCursor = cursor;
2078 currentPageStartCursor = pageStartCursor;
2079 const failuresBeforePage = diagnostics.itemsFailed;
2080 const remaining = effectiveMaxResults - collectedCandidates;
2081 const variables: Record<string, unknown> = {
2082 first: Math.min(20, remaining),
2083 after: cursor,
2084 order: productHuntStreamOrder(Boolean(range)),
2085 };
2086 if (!includeAllProducts) variables.featured = true;
2087 if (range) {
2088 variables.postedAfter = range.postedAfter;
2089 variables.postedBefore = range.postedBefore;
2090 }
2091 if (mode === 'topic') variables.topic = topic;
2092 try {
2093 let data: any = await phQuery(useLightweightIndex ? POSTS_INDEX_QUERY : POSTS_QUERY, variables);
2094 let edges = data?.posts?.edges ?? [];
2095 let pageInfo: { hasNextPage?: boolean; endCursor?: string | null } = data?.posts?.pageInfo ?? {};
2096 if (!edges.length) break;
2097 log.info(`Product Hunt ${label} page ${pageNumber}`, { received: edges.length, processed: diagnostics.itemsProcessed });
2098 let nodes = edges.map((edge: { node?: Record<string, any> }) => edge.node).filter(Boolean);
2099 if (useLightweightIndex) {
2100 diagnostics.productHuntIndexPages += 1;
2101 const hasUnseenLaunch = hasUnexcludedProductHuntLaunch(
2102 nodes.map((node: Record<string, any>) => String(node.url ?? '')),
2103 excludedProductHuntKeys,
2104 );
2105 if (hasUnseenLaunch) {
2106 data = await phQuery(POSTS_QUERY, variables);
2107 diagnostics.productHuntFullPageLookups += 1;
2108 edges = data?.posts?.edges ?? [];
2109 pageInfo = data?.posts?.pageInfo ?? pageInfo;
2110 nodes = edges.map((edge: { node?: Record<string, any> }) => edge.node).filter(Boolean);
2111 useLightweightIndex = false;
2112 } else {
2113 for (const node of nodes) {
2114 const id = String(node.id ?? '');
2115 if (!id || seenIds.has(id)) continue;
2116 const launchKey = productHuntLaunchKey(String(node.url ?? ''));
2117 if (launchKey && excludedProductHuntKeys.has(launchKey)) {
2118 seenIds.add(id);
2119 diagnostics.itemsExcluded += 1;
2120 }
2121 }
2122 nodes = [];
2123 }
2124 }
2125 const selectedNodes = mode === 'search'
2126 ? nodes.filter((node: Record<string, any>) => matchesProductSearch(searchQuery, {
2127 name: node.name,
2128 tagline: node.tagline,
2129 description: node.description,
2130 topics: (node.topics?.edges ?? []).map((edge: any) => edge?.node?.name).filter(Boolean),
2131 }))
2132 : nodes;
2133 if (mode === 'search') {
2134 pagesWithoutSearchMatch = selectedNodes.length > 0 ? 0 : pagesWithoutSearchMatch + 1;
2135 if (pagesWithoutSearchMatch >= SEARCH_PAGES_WITHOUT_MATCH_LIMIT && !diagnostics.searchScanCapReached) {
2136 diagnostics.searchScanCapReached = true;
2137 addWarning(`Search stopped after ${SEARCH_PAGES_WITHOUT_MATCH_LIMIT} consecutive Product Hunt pages without a match. Continue with productHuntResumeCursor from RUN_SUMMARY.nextProductHuntCursor to search older launches.`);
2138 }
2139 }
2140 await processNodes(selectedNodes);
2141 const pageInterrupted = diagnostics.deadlineReached || hardStopping;
2142 diagnostics.nextProductHuntCursor = pageInterrupted
2143 ? pageStartCursor
2144 : pageInfo.hasNextPage && pageInfo.endCursor
2145 ? pageInfo.endCursor
2146 : null;
2147 const checkpointCursor = resumableCheckpointCursor(
2148 pageStartCursor,
2149 Boolean(pageInfo.hasNextPage),
2150 pageInfo.endCursor,
2151 failuresBeforePage,
2152 diagnostics.itemsFailed,
2153 );
2154 if (!useLightweightIndex || pageNumber === 1 || pageNumber % 10 === 0 || !pageInfo.hasNextPage) {
2155 try {
2156 await Actor.setValue('RUN_CHECKPOINT', {
2157 status: 'RUNNING',
2158 updatedAt: new Date().toISOString(),
2159 runId: process.env.ACTOR_RUN_ID ?? null,
2160 buildNumber: process.env.ACTOR_BUILD_NUMBER ?? null,
2161 mode,
2162 label,
2163 range,
2164 productHuntResumeCursor: checkpointCursor,
2165 pageNumber,
2166 itemsProcessed: diagnostics.itemsProcessed,
2167 itemsPushed: diagnostics.itemsPushed,
2168 itemsExcluded: diagnostics.itemsExcluded,
2169 itemsFailed: diagnostics.itemsFailed,
2170 });
2171 } catch (error) {
2172 const message = error instanceof Error ? error.message : 'unknown checkpoint failure';
2173 addWarning(`Run checkpoint could not be persisted: ${message.slice(0, 120)}`);
2174 }
2175 }
2176 if (collectedCandidates >= effectiveMaxResults && pageInfo.hasNextPage) {
2177 diagnostics.resultLimitReached = true;
2178 addWarning('The result limit was reached while Product Hunt still had another page. Continue with excludeProductHuntUrls from this dataset to avoid duplicate enrichment charges.');
2179 }
2180 await reportProgress(true);
2181 if (pageInterrupted || diagnostics.searchScanCapReached) break;
2182 if (!pageInfo.hasNextPage || !pageInfo.endCursor) break;
2183 cursor = pageInfo.endCursor;
2184 pageNumber += 1;
2185 } catch (error) {
2186 diagnostics.apiFailures += 1;
2187 const message = error instanceof Error ? error.message : 'Unknown Product Hunt API error.';
2188 addWarning(`Product Hunt ${label} query failed: ${message}`);
2189 break;
2190 }
2191 }
2192 };
2193
2194 if (mode === 'urls') {
2195 const rawSeeds = startUrls
2196 .map((entry) => typeof entry === 'string' ? entry : typeof entry?.url === 'string' ? entry.url : '')
2197 .filter(Boolean);
2198 const seeds = rawSeeds.map(parseProductHuntSeedUrl);
2199 const validSeeds = seeds.filter((seed) => seed.kind !== 'invalid');
2200 seeds.filter((seed) => seed.kind === 'invalid').forEach((seed) => addWarning(`Ignored start URL: ${seed.reason}`));
2201 if (validSeeds.length === 0) await finalizeRun('INVALID_INPUT', 'No valid Product Hunt post, product launch, or dated leaderboard URLs were supplied.');
2202
2203 for (const seed of validSeeds) {
2204 if (collectedCandidates >= effectiveMaxResults || diagnostics.chargeLimitReached) break;
2205 if (deadlineReached() || hardStopping) break;
2206 if (seed.kind === 'post') {
2207 try {
2208 const data = await phQuery(POST_BY_SLUG_QUERY, { slug: seed.slug });
2209 if (!data?.post) {
2210 addWarning(`No Product Hunt post matched the supplied URL slug: ${seed.slug}.`);
2211 continue;
2212 }
2213 await processNodes([data.post]);
2214 } catch (error) {
2215 diagnostics.apiFailures += 1;
2216 const message = error instanceof Error ? error.message : 'Unknown Product Hunt API error.';
2217 addWarning(`Product Hunt URL lookup failed for ${seed.slug}: ${message}`);
2218 }
2219 } else if (seed.kind === 'leaderboard') {
2220 await streamPosts(dateRangeForPeriod(seed.period, seed.anchorDate), `${seed.period} leaderboard URL`);
2221 }
2222 }
2223 } else {
2224 let range: DateRange | null = null;
2225 if (mode === 'leaderboard') {
2226 if (startDate) {
2227 range = dateRangeForPeriod(leaderboardPeriod, startDate, endDate);
2228 } else if (lookbackDays > 1) {
2229 const end = new Date();
2230 const begin = new Date(end);
2231 begin.setUTCDate(end.getUTCDate() - lookbackDays + 1);
2232 range = { postedAfter: `${isoDate(begin)}T00:00:00+00:00`, postedBefore: `${isoDate(end)}T23:59:59+00:00` };
2233 } else {
2234 range = dateRangeForPeriod(leaderboardPeriod, isoDate(new Date()));
2235 }
2236 }
2237 await streamPosts(range, mode);
2238 }
2239
2240 if (hardStopping) return;
2241 await closeBrowser();
2242 const outcome = resolveOutcome();
2243 await finalizeRun(outcome, outcomeMessage(outcome));
2244 } catch (error) {
2245 if (finalised || hardStopping) return;
2246 await closeBrowser();
2247 const message = error instanceof Error ? error.message : 'Unknown actor failure.';
2248 diagnostics.itemsFailed += 1;
2249 addWarning(`Unhandled actor error: ${message.slice(0, 160)}`);
2250 await finalizeRun('UPSTREAM_FAILED', `The run stopped before completion: ${message.slice(0, 160)}`);
2251 }
2252}
2253
2254await run();
