1import puppeteer from "puppeteer-extra";
2import StealthPlugin from "puppeteer-extra-plugin-stealth";
3import fs from "fs";
4import { execFileSync } from "child_process";
5import { KeyValueStore } from "apify";
6
7puppeteer.use(StealthPlugin());
8
9const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
10
11function isAborted(signal) {
12 return Boolean(signal?.aborted);
13}
14
15const MAX_SESSION_ATTEMPTS = 3;
16const SESSION_RETRY_BASE_MS = 2500;
17const WARMUP_MS = 1000;
18const DIRECT_WORKERS = 1;
19const PROTOCOL_TIMEOUT_MS = 12000;
20const DIRECT_NAV_TIMEOUT_MS = 10000;
21const DIRECT_TICKER_TIMEOUT_MS = 15000;
22const DIRECT_PHASE_BUDGET_MS = 4 * 60 * 1000;
23const RECOGNIZED_CALENDAR_DIRECT_BUDGET_MS = 60 * 1000;
24const QUALIFY_NAV_TIMEOUT_MS = 15000;
25const USEFUL_BODY_TIMEOUT_MS = 1500;
26
27const BLOCKED_WSN_HOST_SUFFIXES = [
28 "doubleclick.net",
29 "googlesyndication.com",
30 "googleadservices.com",
31 "googletagmanager.com",
32 "google-analytics.com",
33 "adthrive.com",
34 "raptive.com",
35 "raptivecdn.com",
36 "criteo.com",
37 "smartadserver.com",
38 "amazon-adsystem.com",
39 "adnxs.com",
40 "liadm.com",
41 "rubiconproject.com",
42 "scorecardresearch.com",
43 "ay.delivery"
44];
45
46function shouldBlockWsnRequest(request) {
47 const type = request.resourceType();
48
49 if (["image", "media", "font", "stylesheet"].includes(type)) {
50 return true;
51 }
52
53 try {
54 const hostname = new URL(request.url()).hostname.toLowerCase();
55 return BLOCKED_WSN_HOST_SUFFIXES.some(
56 suffix => hostname === suffix || hostname.endsWith(`.${suffix}`)
57 );
58 } catch {
59 return false;
60 }
61}
62
63const USER_AGENT =
64 "Mozilla/5.0 (X11; Linux x86_64) " +
65 "AppleWebKit/537.36 (KHTML, like Gecko) " +
66 "Chrome/127.0.0.0 Safari/537.36";
67
68const BASE_URL = "https://wallstreetnumbers.com";
69const CALENDAR_URL =
70 `${BASE_URL}/screeners/sp-500-earnings-calendar`;
71
72const DAYS_TO_SCAN = 14;
73
74const CACHE_ENV = String(process.env.CACHE_ENV || "prod")
75 .trim()
76 .toLowerCase()
77 .replace(/[^a-z0-9_-]/g, "-");
78const CACHE_STORE_NAME = `${CACHE_ENV}-wallstreetnumbers-cache`;
79const CACHE_KEY =
80 "WALLSTREETNUMBERS_EARNINGS_CALENDAR";
81const CACHE_META_KEY =
82 "WALLSTREETNUMBERS_EARNINGS_CALENDAR_META";
83
84const CACHE_MAX_AGE_MS =
85 24 * 60 * 60 * 1000;
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133function formatDate(date) {
134 const y = date.getFullYear();
135 const m =
136 String(date.getMonth() + 1).padStart(2, "0");
137 const d =
138 String(date.getDate()).padStart(2, "0");
139
140 return `${y}-${m}-${d}`;
141}
142
143function addDays(date, days) {
144 const result = new Date(date);
145 result.setDate(result.getDate() + days);
146 return result;
147}
148
149function normalizeTicker(value) {
150 return String(value || "")
151 .trim()
152 .toUpperCase()
153 .replace(/\s+/g, "")
154 .replace(/[^A-Z0-9.\-]/g, "");
155}
156
157function isTicker(value) {
158 const ticker =
159 normalizeTicker(value);
160
161 return (
162 !!ticker &&
163 /^[A-Z][A-Z0-9.\-]{0,11}$/.test(ticker)
164 );
165}
166
167function normalizeDate(value) {
168 if (!value) {
169 return null;
170 }
171
172 const text =
173 String(value)
174 .replace(/\u00a0/g, " ")
175 .replace(/\s+/g, " ")
176 .trim();
177
178 let m =
179 text.match(
180 /\b(20\d{2})[-/](\d{1,2})[-/](\d{1,2})\b/
181 );
182
183 if (m) {
184 return (
185 `${m[1]}-` +
186 `${String(m[2]).padStart(2, "0")}-` +
187 `${String(m[3]).padStart(2, "0")}`
188 );
189 }
190
191 m =
192 text.match(
193 /\b(\d{1,2})\/(\d{1,2})\/(20\d{2})\b/
194 );
195
196 if (m) {
197 return (
198 `${m[3]}-` +
199 `${String(m[1]).padStart(2, "0")}-` +
200 `${String(m[2]).padStart(2, "0")}`
201 );
202 }
203
204 m =
205 text.match(
206 /\b(Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\.?\s+(\d{1,2})(?:,)?\s+(20\d{2})\b/i
207 );
208
209 if (m) {
210 const months = {
211 jan: 1,
212 feb: 2,
213 mar: 3,
214 apr: 4,
215 may: 5,
216 jun: 6,
217 jul: 7,
218 aug: 8,
219 sep: 9,
220 oct: 10,
221 nov: 11,
222 dec: 12
223 };
224
225 const month =
226 months[
227 m[1]
228 .slice(0, 3)
229 .toLowerCase()
230 ];
231
232 if (month) {
233 return (
234 `${m[3]}-` +
235 `${String(month).padStart(2, "0")}-` +
236 `${String(m[2]).padStart(2, "0")}`
237 );
238 }
239 }
240
241
242
243
244
245 m =
246 text.match(
247 /\b(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\.?\s+(\d{1,2})(?:,)?\s+(\d{2})\b/i
248 );
249
250 if (m) {
251 const months = {
252 jan: 1,
253 feb: 2,
254 mar: 3,
255 apr: 4,
256 may: 5,
257 jun: 6,
258 jul: 7,
259 aug: 8,
260 sep: 9,
261 oct: 10,
262 nov: 11,
263 dec: 12
264 };
265
266 const month =
267 months[m[1].toLowerCase()];
268
269 const year =
270 2000 + Number(m[3]);
271
272 if (month) {
273 return (
274 `${year}-` +
275 `${String(month).padStart(2, "0")}-` +
276 `${String(m[2]).padStart(2, "0")}`
277 );
278 }
279 }
280
281 return null;
282}
283
284function csvEscape(value) {
285 const text =
286 String(value ?? "");
287
288 return /[",\n\r]/.test(text)
289 ? `"${text.replace(/"/g, '""')}"`
290 : text;
291}
292
293function mapToCsv(map) {
294 const rows = [
295 "ticker,date,source"
296 ];
297
298 const entries =
299 [...map.entries()]
300 .filter(
301 ([ticker, date]) =>
302 isTicker(ticker) &&
303 normalizeDate(date)
304 )
305 .sort(
306 (a, b) =>
307 a[1].localeCompare(b[1]) ||
308 a[0].localeCompare(b[0])
309 );
310
311 for (const [ticker, date] of entries) {
312 rows.push(
313 `${csvEscape(ticker)},${csvEscape(date)},${csvEscape("WallStreetNumbers")}`
314 );
315 }
316
317 return rows.join("\n") + "\n";
318}
319
320function csvToMap(csv) {
321 const map =
322 new Map();
323
324 const lines =
325 String(csv || "")
326 .split(/\r?\n/)
327 .filter(Boolean);
328
329 for (let i = 1; i < lines.length; i++) {
330 const match =
331 lines[i].match(
332 /^"?([^",]+)"?,\s*"?([^",]+)"?(?:,\s*"?([^",]*)"?)?$/
333 );
334
335 if (!match) {
336 continue;
337 }
338
339 const ticker =
340 normalizeTicker(match[1]);
341
342 const date =
343 normalizeDate(match[2]);
344
345 if (
346 isTicker(ticker) &&
347 date
348 ) {
349 map.set(ticker, date);
350 }
351 }
352
353 return map;
354}
355
356async function loadSnapshot() {
357 try {
358 const store =
359 await KeyValueStore.open(
360 CACHE_STORE_NAME
361 );
362
363 const [
364 csvValue,
365 meta
366 ] =
367 await Promise.all([
368 store.getValue(CACHE_KEY),
369 store.getValue(CACHE_META_KEY)
370 ]);
371
372 const csv =
373 Buffer.isBuffer(csvValue)
374 ? csvValue.toString("utf8")
375 : csvValue;
376
377 return {
378 map:
379 typeof csv === "string"
380 ? csvToMap(csv)
381 : new Map(),
382 meta:
383 meta || null
384 };
385
386 } catch (err) {
387 console.log(
388 "[WSN CACHE] Load failed:",
389 err.message
390 );
391
392 return {
393 map: new Map(),
394 meta: null
395 };
396 }
397}
398
399function cacheIsFreshComplete(
400 snapshot,
401 rangeStart,
402 rangeEnd
403) {
404 const meta =
405 snapshot?.meta;
406
407 if (
408 !meta?.complete ||
409 meta.status !== "complete"
410 ) {
411 return false;
412 }
413
414 if (
415 meta.rangeStart !== rangeStart ||
416 meta.rangeEnd !== rangeEnd
417 ) {
418 return false;
419 }
420
421 const scrapedAt =
422 Date.parse(meta.scrapedAt || "");
423
424 if (!Number.isFinite(scrapedAt)) {
425 return false;
426 }
427
428 const age =
429 Date.now() - scrapedAt;
430
431 return (
432 age >= 0 &&
433 age <= CACHE_MAX_AGE_MS
434 );
435}
436
437async function saveSnapshotUnlocked({
438 freshMap,
439 rangeStart,
440 rangeEnd,
441 replacementDates = [],
442 complete = false,
443 note = ""
444}) {
445
446
447
448 if (
449 !(freshMap instanceof Map) ||
450 freshMap.size === 0
451 ) {
452 console.log(
453 "[WSN CACHE] Zero fresh verified rows; persistent CSV untouched."
454 );
455
456 return false;
457 }
458
459 try {
460 const store =
461 await KeyValueStore.open(
462 CACHE_STORE_NAME
463 );
464
465 const [
466 oldCsvValue,
467 oldMeta
468 ] =
469 await Promise.all([
470 store.getValue(CACHE_KEY),
471 store.getValue(CACHE_META_KEY)
472 ]);
473
474 const oldCsv =
475 Buffer.isBuffer(oldCsvValue)
476 ? oldCsvValue.toString("utf8")
477 : oldCsvValue;
478
479 const merged =
480 typeof oldCsv === "string"
481 ? csvToMap(oldCsv)
482 : new Map();
483
484
485
486
487
488 const replace =
489 new Set(replacementDates);
490
491 if (replace.size > 0) {
492 for (
493 const [ticker, date]
494 of [...merged.entries()]
495 ) {
496 if (replace.has(date)) {
497 merged.delete(ticker);
498 }
499 }
500 }
501
502 for (
503 const [tickerRaw, dateRaw]
504 of freshMap.entries()
505 ) {
506 const ticker =
507 normalizeTicker(tickerRaw);
508
509 const date =
510 normalizeDate(dateRaw);
511
512 if (
513 isTicker(ticker) &&
514 date
515 ) {
516 merged.set(ticker, date);
517 }
518 }
519
520 const now =
521 new Date().toISOString();
522
523 const tickerScrapedAt = {
524 ...(oldMeta?.tickerScrapedAt || {})
525 };
526
527 for (const ticker of freshMap.keys()) {
528 tickerScrapedAt[ticker] = now;
529 }
530
531 await store.setValue(
532 CACHE_KEY,
533 mapToCsv(merged),
534 {
535 contentType:
536 "text/csv; charset=utf-8"
537 }
538 );
539
540 await store.setValue(
541 CACHE_META_KEY,
542 {
543 scrapedAt: now,
544 status:
545 complete
546 ? "complete"
547 : "partial",
548 complete:
549 Boolean(complete),
550 rangeStart,
551 rangeEnd,
552 tickerScrapedAt,
553 rowCount:
554 merged.size,
555 freshRowCount:
556 freshMap.size,
557 note:
558 note ||
559 (
560 complete
561 ? "Complete Wall Street Numbers earnings snapshot."
562 : "Partial Wall Street Numbers earnings snapshot."
563 )
564 }
565 );
566
567 console.log(
568 `[WSN CACHE] Saved/merged ${freshMap.size} fresh row(s); persistent CSV=${merged.size}; status=${complete ? "complete" : "partial"}.`
569 );
570
571 return true;
572
573 } catch (err) {
574 console.log(
575 "[WSN CACHE] Save failed:",
576 err.message
577 );
578
579 return false;
580 }
581}
582
583
584let snapshotSaveChain =
585 Promise.resolve();
586
587async function saveSnapshot(args) {
588 const runSave =
589 () =>
590 saveSnapshotUnlocked(args);
591
592 const result =
593 snapshotSaveChain.then(
594 runSave,
595 runSave
596 );
597
598 snapshotSaveChain =
599 result.catch(() => {});
600
601 return await result;
602}
603
604function looksBlocked(
605 state,
606 response
607) {
608 const status =
609 response?.status?.() ??
610 0;
611
612 const rawText =
613 [
614 state?.title || "",
615 state?.body || "",
616 state?.html || ""
617 ]
618 .join("\n");
619
620 const text =
621 rawText.toLowerCase();
622
623 const reasons = [];
624
625 if (status === 401) reasons.push("HTTP 401");
626 if (status === 403) reasons.push("HTTP 403");
627 if (status === 407) reasons.push("HTTP 407");
628 if (status === 429) reasons.push("HTTP 429");
629 if (status >= 500) reasons.push(`HTTP ${status}`);
630
631 const phrases = [
632 "access denied",
633 "pardon our interruption",
634 "captcha",
635 "verify you are human",
636 "just a moment",
637 "too many requests",
638 "request unsuccessful"
639 ];
640
641 for (const phrase of phrases) {
642 const index =
643 text.indexOf(
644 phrase
645 );
646
647 if (index === -1) {
648 continue;
649 }
650
651 const context =
652 rawText
653 .slice(
654 Math.max(0, index - 180),
655 Math.min(
656 rawText.length,
657 index + phrase.length + 180
658 )
659 )
660 .replace(/\s+/g, " ")
661 .trim();
662
663 reasons.push(
664 `phrase="${phrase}" context="${context}"`
665 );
666 }
667
668 if (reasons.length > 0) {
669 console.log(
670 "[WSN BLOCK DETECTOR] Trigger(s):",
671 reasons
672 );
673
674 console.log(
675 "[WSN BLOCK DETECTOR] Page diagnostics:",
676 {
677 status,
678 url: state?.url || "",
679 title: state?.title || "",
680 bodyLength: state?.body?.length || 0,
681 htmlLength: state?.html?.length || 0
682 }
683 );
684
685 return true;
686 }
687
688 return false;
689}
690
691function isRecoverableNetworkError(error) {
692 const text = String(error?.message || error || "");
693 return (
694 /ERR_TUNNEL_CONNECTION_FAILED/i.test(text) ||
695 /ERR_CONNECTION_CLOSED/i.test(text) ||
696 /ERR_PROXY_CONNECTION_FAILED/i.test(text) ||
697 /ERR_CONNECTION_RESET/i.test(text) ||
698 /ERR_SOCKS_CONNECTION_FAILED/i.test(text) ||
699 /ERR_ADDRESS_UNREACHABLE/i.test(text) ||
700 /ERR_NETWORK_CHANGED/i.test(text) ||
701 /ERR_INTERNET_DISCONNECTED/i.test(text) ||
702 /ERR_TIMED_OUT/i.test(text) ||
703 /Navigation timeout/i.test(text) ||
704 /TimeoutError/i.test(text)
705 );
706}
707
708async function waitForUsefulBody(
709 page,
710 timeoutMs = USEFUL_BODY_TIMEOUT_MS
711) {
712 try {
713 await page.waitForFunction(
714 () =>
715 (
716 document.body?.innerText ||
717 ""
718 )
719 .trim()
720 .length > 150,
721 {
722 timeout: timeoutMs,
723 polling: 500
724 }
725 );
726
727 return true;
728
729 } catch {
730 return false;
731 }
732}
733
734async function getPageState(page) {
735 return await page.evaluate(
736 () => ({
737 url:
738 location.href,
739 title:
740 document.title || "",
741 readyState:
742 document.readyState,
743 body:
744 document.body?.innerText || "",
745 html:
746 document.documentElement?.outerHTML || ""
747 })
748 );
749}
750
751async function createBrowserSession(
752 proxyConfiguration,
753 attempt = 1
754) {
755 const sessionId =
756 `wallstreetnumbers_${Date.now()}_${attempt}_${Math.random()
757 .toString(36)
758 .slice(2, 10)}`;
759
760 console.log(
761 `[WSN] Creating fresh browser/proxy session ${attempt}/${MAX_SESSION_ATTEMPTS}: ${sessionId}`
762 );
763
764 console.log(
765 "Puppeteer executable:",
766 puppeteer.executablePath()
767 );
768
769 console.log(
770 "Chrome exists:",
771 fs.existsSync(
772 puppeteer.executablePath()
773 )
774 );
775
776 try {
777 const result =
778 execFileSync(
779 puppeteer.executablePath(),
780 ["--version"],
781 {
782 encoding: "utf8",
783 stdio: "pipe"
784 }
785 );
786
787 console.log(
788 "Chrome execution test:",
789 result.trim()
790 );
791
792 } catch (err) {
793 console.log(
794 "Chrome execution test FAILED:",
795 err.message
796 );
797 }
798
799 let proxyHostPort =
800 null;
801
802 let proxyUsername =
803 null;
804
805 let proxyPassword =
806 null;
807
808 if (proxyConfiguration) {
809 try {
810 const proxyUrl =
811 await proxyConfiguration.newUrl(
812 sessionId
813 );
814
815 console.log(
816 "[WSN] Proxy session:",
817 sessionId
818 );
819
820 console.log(
821 "[WSN] Apify proxy URL obtained:",
822 proxyUrl
823 ? "YES"
824 : "NO"
825 );
826
827 if (proxyUrl) {
828 const parsed =
829 new URL(proxyUrl);
830
831 proxyHostPort =
832 `${parsed.hostname}:${parsed.port}`;
833
834 proxyUsername =
835 decodeURIComponent(
836 parsed.username || ""
837 );
838
839 proxyPassword =
840 decodeURIComponent(
841 parsed.password || ""
842 );
843
844 console.log(
845 "[WSN] Using Apify Residential proxy:",
846 proxyHostPort
847 );
848
849 console.log(
850 "[WSN] Proxy authentication available:",
851 Boolean(
852 proxyUsername &&
853 proxyPassword
854 )
855 );
856 }
857
858 } catch (err) {
859 console.log(
860 "[WSN] Failed to obtain/parse proxy URL:",
861 err.message
862 );
863 }
864 }
865
866 const args = [
867 "--no-sandbox",
868 "--disable-setuid-sandbox",
869 "--disable-blink-features=AutomationControlled",
870 "--disable-dev-shm-usage",
871 "--no-first-run",
872 "--no-default-browser-check"
873 ];
874
875 if (proxyHostPort) {
876 args.push(
877 `--proxy-server=${proxyHostPort}`
878 );
879 }
880
881 const browser =
882 await puppeteer.launch({
883 headless: true,
884 protocolTimeout: PROTOCOL_TIMEOUT_MS,
885 executablePath:
886 puppeteer.executablePath(),
887 args
888 });
889
890 const page =
891 await browser.newPage();
892
893 await page.setUserAgent(
894 USER_AGENT
895 );
896
897 await page.setViewport({
898 width: 1440,
899 height: 1000,
900 deviceScaleFactor: 1
901 });
902
903 await page.setExtraHTTPHeaders({
904 "Accept-Language":
905 "en-US,en;q=0.9",
906 "Upgrade-Insecure-Requests":
907 "1"
908 });
909
910 await page.evaluateOnNewDocument(
911 () => {
912 Object.defineProperty(
913 navigator,
914 "webdriver",
915 {
916 get: () => false
917 }
918 );
919 }
920 );
921
922 if (
923 proxyUsername &&
924 proxyPassword
925 ) {
926 await page.authenticate({
927 username: proxyUsername,
928 password: proxyPassword
929 });
930
931 console.log(
932 "[WSN] Proxy authentication configured."
933 );
934 }
935
936 await page.setRequestInterception(
937 true
938 );
939
940 page.on(
941 "request",
942 request => {
943 if (shouldBlockWsnRequest(request)) {
944 request.abort().catch(() => {});
945 return;
946 }
947
948 request.continue().catch(() => {});
949 }
950 );
951
952 page.on(
953 "requestfailed",
954 request => {
955 if (shouldBlockWsnRequest(request)) {
956 return;
957 }
958
959 console.log(
960 "[WSN REQUEST FAILED]",
961 request.url(),
962 request.failure()?.errorText || ""
963 );
964 }
965 );
966
967 return {
968 browser,
969 page,
970 sessionId,
971 proxyUsername,
972 proxyPassword
973 };
974}
975
976async function safeGoto(
977 page,
978 url,
979 label,
980 timeout = DIRECT_NAV_TIMEOUT_MS
981) {
982 const safeGotoStartedAt = Date.now();
983
984 console.log(
985 `[WSN:${label}] URL:`,
986 url
987 );
988
989 let response =
990 null;
991
992 let navigationError =
993 null;
994
995 try {
996 response =
997 await page.goto(
998 url,
999 {
1000 waitUntil:
1001 "domcontentloaded",
1002 timeout
1003 }
1004 );
1005
1006 } catch (err) {
1007 navigationError = err;
1008
1009 console.log(
1010 `[WSN:${label}] Navigation failed:`,
1011 err.message
1012 );
1013
1014 if (isRecoverableNetworkError(err)) {
1015
1016
1017
1018
1019
1020
1021
1022
1023 const navigationTimedOut =
1024 /Navigation timeout|TimeoutError/i.test(
1025 String(err?.message || err || "")
1026 );
1027
1028 if (!navigationTimedOut) {
1029 console.log(
1030 `[WSN:${label}] NETWORK_FAILURE_NO_DOM_RECOVERY: ${err.message}`
1031 );
1032
1033 return {
1034 response: null,
1035 state: null,
1036 blocked: false,
1037 networkFailed: true,
1038 navigationRecovered: false,
1039 failureReason: err.message
1040 };
1041 }
1042
1043 console.log(
1044 `[WSN:${label}] Navigation timed out; inspecting live DOM before rejecting the session.`
1045 );
1046
1047 try {
1048
1049
1050
1051
1052 await waitForUsefulBody(
1053 page,
1054 Math.min(
1055 1500,
1056 USEFUL_BODY_TIMEOUT_MS
1057 )
1058 );
1059
1060 const recoveredState =
1061 await getPageState(page);
1062
1063 const recoveredBlocked =
1064 looksBlocked(
1065 recoveredState,
1066 null
1067 );
1068
1069 const recoveredSuspiciousEmpty =
1070 recoveredState.html.length < 1000 ||
1071 recoveredState.body.trim().length < 100;
1072
1073 console.log(
1074 `[WSN:${label}] NAV_TIMEOUT_DOM_DIAG url=${recoveredState.url || ""} title=${JSON.stringify(recoveredState.title || "")} readyState=${recoveredState.readyState || ""} body=${recoveredState.body.length} html=${recoveredState.html.length} blocked=${recoveredBlocked} suspiciousEmpty=${recoveredSuspiciousEmpty}`
1075 );
1076
1077 if (
1078 !recoveredBlocked &&
1079 !recoveredSuspiciousEmpty
1080 ) {
1081 console.log(
1082 `[WSN:${label}] NAV_TIMEOUT_DOM_RECOVERED: useful unblocked DOM loaded despite navigation timeout; continuing with this session.`
1083 );
1084
1085 return {
1086 response: null,
1087 state: recoveredState,
1088 blocked: false,
1089 networkFailed: false,
1090 suspiciousEmpty: false,
1091 notFound: false,
1092 inspectionFailed: false,
1093 navigationRecovered: true,
1094 failureReason: null
1095 };
1096 }
1097
1098 console.log(
1099 `[WSN:${label}] NAV_TIMEOUT_NO_USABLE_DOM: recovered DOM was ${recoveredBlocked ? "blocked" : "too small/empty"}; rejecting this navigation.`
1100 );
1101
1102 return {
1103 response: null,
1104 state: recoveredState,
1105 blocked: recoveredBlocked,
1106 networkFailed: false,
1107 suspiciousEmpty:
1108 recoveredSuspiciousEmpty,
1109 notFound: false,
1110 inspectionFailed: false,
1111 navigationRecovered: false,
1112 failureReason:
1113 recoveredBlocked
1114 ? "Navigation timeout recovered blocked DOM"
1115 : "Navigation timeout recovered no usable DOM"
1116 };
1117
1118 } catch (recoveryErr) {
1119 console.log(
1120 `[WSN:${label}] NAV_TIMEOUT_NO_USABLE_DOM: DOM recovery inspection failed: ${recoveryErr.message}`
1121 );
1122
1123 return {
1124 response: null,
1125 state: null,
1126 blocked: false,
1127 networkFailed: true,
1128 suspiciousEmpty: false,
1129 notFound: false,
1130 inspectionFailed: true,
1131 navigationRecovered: false,
1132 failureReason:
1133 `${err.message}; DOM recovery failed: ${recoveryErr.message}`
1134 };
1135 }
1136 }
1137 }
1138
1139
1140
1141
1142
1143 if ((response?.status?.() ?? 0) === 404) {
1144 console.log(
1145 `[WSN:${label}] HTTP: 404`
1146 );
1147 console.log(
1148 `[WSN:${label}] HTTP 404; skipping expensive page inspection.`
1149 );
1150
1151 return {
1152 response,
1153 state: null,
1154 blocked: false,
1155 networkFailed: false,
1156 suspiciousEmpty: false,
1157 notFound: true,
1158 inspectionFailed: false,
1159 failureReason: "HTTP 404"
1160 };
1161 }
1162
1163 const navigationElapsedMs =
1164 Date.now() - safeGotoStartedAt;
1165
1166 const bodyWaitStartedAt = Date.now();
1167
1168 await waitForUsefulBody(page);
1169
1170 const bodyWaitElapsedMs =
1171 Date.now() - bodyWaitStartedAt;
1172
1173 let state;
1174
1175 const inspectionStartedAt = Date.now();
1176
1177 try {
1178 state =
1179 await getPageState(page);
1180
1181 } catch (err) {
1182 console.log(
1183 `[WSN:${label}] Inspection failed:`,
1184 err.message
1185 );
1186
1187
1188
1189
1190
1191
1192 if (label === "CALENDAR") {
1193 try {
1194 console.log(
1195 "[WSN:CALENDAR] Retrying calendar page once after inspection failure."
1196 );
1197
1198 response =
1199 await page.goto(
1200 url,
1201 {
1202 waitUntil:
1203 "domcontentloaded",
1204 timeout
1205 }
1206 );
1207
1208 await waitForUsefulBody(
1209 page
1210 );
1211
1212 state =
1213 await getPageState(
1214 page
1215 );
1216
1217 navigationError =
1218 null;
1219
1220 } catch (retryErr) {
1221 console.log(
1222 "[WSN:CALENDAR] Calendar inspection retry failed:",
1223 retryErr.message
1224 );
1225
1226 return {
1227 response,
1228 state: null,
1229 blocked: false,
1230 networkFailed: Boolean(navigationError),
1231 suspiciousEmpty: false,
1232 notFound: false,
1233 inspectionFailed: true,
1234 failureReason:
1235 navigationError?.message ||
1236 retryErr.message
1237 };
1238 }
1239 } else {
1240 return {
1241 response,
1242 state: null,
1243 blocked: false,
1244 networkFailed: Boolean(navigationError),
1245 suspiciousEmpty: false,
1246 notFound: false,
1247 inspectionFailed: true,
1248 failureReason: navigationError?.message || err.message
1249 };
1250 }
1251 }
1252
1253 const inspectionElapsedMs =
1254 Date.now() - inspectionStartedAt;
1255
1256 console.log(
1257 `[WSN:${label}] HTTP:`,
1258 response?.status?.() ??
1259 "NO RESPONSE"
1260 );
1261
1262 console.log(
1263 `[WSN:${label}] Body length:`,
1264 state.body.length
1265 );
1266
1267 console.log(
1268 `[WSN:${label}] Timing: navigation=${(navigationElapsedMs / 1000).toFixed(1)}s bodyWait=${(bodyWaitElapsedMs / 1000).toFixed(1)}s inspection=${(inspectionElapsedMs / 1000).toFixed(1)}s total=${((Date.now() - safeGotoStartedAt) / 1000).toFixed(1)}s`
1269 );
1270
1271 const blocked =
1272 looksBlocked(
1273 state,
1274 response
1275 );
1276
1277 const suspiciousEmpty =
1278 state.html.length < 1000 ||
1279 state.body.trim().length < 100;
1280
1281 if (suspiciousEmpty) {
1282 console.log(
1283 `[WSN:${label}] Suspicious empty page; html=${state.html.length}; body=${state.body.length}.`
1284 );
1285 }
1286
1287 return {
1288 response,
1289 state,
1290 blocked,
1291 networkFailed: false,
1292 suspiciousEmpty,
1293 notFound: false,
1294 inspectionFailed: false,
1295 failureReason:
1296 suspiciousEmpty
1297 ? "Suspicious empty page"
1298 : null
1299 };
1300}
1301
1302async function proxyIpTest(page) {
1303 try {
1304 const response =
1305 await page.goto(
1306 "https://api.ipify.org?format=json",
1307 {
1308 waitUntil: "domcontentloaded",
1309 timeout: 30000
1310 }
1311 );
1312
1313 const body =
1314 await page.evaluate(
1315 () => document.body?.innerText || ""
1316 );
1317
1318 console.log(
1319 "[WSN] Proxy public IP test:",
1320 body
1321 );
1322
1323 return (
1324 (response?.status?.() ?? 0) >= 200 &&
1325 (response?.status?.() ?? 0) < 400 &&
1326 body.trim().length > 0
1327 );
1328
1329 } catch (err) {
1330 console.log(
1331 "[WSN] Proxy public IP test FAILED:",
1332 err.message
1333 );
1334
1335 return false;
1336 }
1337}
1338
1339async function qualifySession(page, sessionId) {
1340 console.log(
1341 `[WSN] Qualifying fresh session ${sessionId}.`
1342 );
1343
1344 const home =
1345 await safeGoto(
1346 page,
1347 BASE_URL,
1348 "WARMUP:HOME",
1349 QUALIFY_NAV_TIMEOUT_MS
1350 );
1351
1352 if (
1353 home.networkFailed ||
1354 home.blocked ||
1355 home.suspiciousEmpty ||
1356 !home.state
1357 ) {
1358 return {
1359 usable: false,
1360 reason:
1361 home.failureReason ||
1362 (home.blocked
1363 ? "Homepage blocked"
1364 : "Homepage unusable")
1365 };
1366 }
1367
1368 await sleep(WARMUP_MS);
1369
1370 const calendarProbe =
1371 await safeGoto(
1372 page,
1373 CALENDAR_URL,
1374 "WARMUP:CALENDAR-PROBE",
1375 QUALIFY_NAV_TIMEOUT_MS
1376 );
1377
1378 if (
1379 calendarProbe.networkFailed ||
1380 calendarProbe.blocked ||
1381 calendarProbe.suspiciousEmpty ||
1382 !calendarProbe.state
1383 ) {
1384 return {
1385 usable: false,
1386 reason:
1387 calendarProbe.failureReason ||
1388 (calendarProbe.blocked
1389 ? "Calendar probe blocked"
1390 : "Calendar probe unusable")
1391 };
1392 }
1393
1394 console.log(
1395 `[WSN] Session ${sessionId} PASSED warm-up/calendar probe.`
1396 );
1397
1398 return {
1399 usable: true
1400 };
1401}
1402
1403
1404async function configureDirectWorkerPage(
1405 page,
1406 proxyUsername,
1407 proxyPassword
1408) {
1409 await page.setUserAgent(
1410 USER_AGENT
1411 );
1412
1413 await page.setViewport({
1414 width: 1440,
1415 height: 1000,
1416 deviceScaleFactor: 1
1417 });
1418
1419 await page.setExtraHTTPHeaders({
1420 "Accept-Language":
1421 "en-US,en;q=0.9",
1422 "Upgrade-Insecure-Requests":
1423 "1"
1424 });
1425
1426 await page.evaluateOnNewDocument(
1427 () => {
1428 Object.defineProperty(
1429 navigator,
1430 "webdriver",
1431 {
1432 get: () => false
1433 }
1434 );
1435 }
1436 );
1437
1438 if (
1439 proxyUsername &&
1440 proxyPassword
1441 ) {
1442 await page.authenticate({
1443 username: proxyUsername,
1444 password: proxyPassword
1445 });
1446 }
1447
1448 await page.setRequestInterception(
1449 true
1450 );
1451
1452 page.on(
1453 "request",
1454 request => {
1455 if (shouldBlockWsnRequest(request)) {
1456 request.abort().catch(() => {});
1457 return;
1458 }
1459
1460 request.continue().catch(() => {});
1461 }
1462 );
1463
1464 page.on(
1465 "requestfailed",
1466 request => {
1467 if (shouldBlockWsnRequest(request)) {
1468 return;
1469 }
1470
1471 console.log(
1472 "[WSN REQUEST FAILED]",
1473 request.url(),
1474 request.failure()?.errorText || ""
1475 );
1476 }
1477 );
1478}
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492async function extractDirectDate(
1493 state,
1494 ticker,
1495 rangeStart,
1496 rangeEnd
1497) {
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510 if (!state) {
1511 return null;
1512 }
1513
1514 const wanted = normalizeTicker(ticker);
1515 const escaped = wanted.replace(
1516 /[.*+?^${}()|[\]\\]/g,
1517 "\\$&"
1518 );
1519
1520 const title = String(state.title || "");
1521 const body = String(state.body || "")
1522 .replace(/\u00a0/g, " ")
1523 .replace(/\s+/g, " ")
1524 .trim();
1525 const html = String(state.html || "");
1526
1527 const identityPattern = new RegExp(
1528 `(?:\\(|\\b)${escaped}(?:\\)|\\b)`,
1529 "i"
1530 );
1531
1532
1533
1534
1535
1536
1537 const identity = `${title} ${body.slice(0, 2500)}`;
1538
1539 if (!identityPattern.test(identity)) {
1540 console.log(
1541 `[WSN:${ticker}] Page identity did not verify requested ticker.`
1542 );
1543 return null;
1544 }
1545
1546 function decodeHtmlText(value) {
1547 return String(value || "")
1548 .replace(/<script\b[^>]*>[\s\S]*?<\/script>/gi, " ")
1549 .replace(/<style\b[^>]*>[\s\S]*?<\/style>/gi, " ")
1550 .replace(/<[^>]+>/g, " ")
1551 .replace(/ | /gi, " ")
1552 .replace(/&/gi, "&")
1553 .replace(/"|"/gi, '"')
1554 .replace(/'|'/gi, "'")
1555 .replace(/\s+/g, " ")
1556 .trim();
1557 }
1558
1559
1560
1561
1562
1563 const timeTagPattern = /<time\b[^>]*\bdatetime\s*=\s*["']([^"']+)["'][^>]*>[\s\S]*?<\/time>/gi;
1564 let timeMatch;
1565 let checkedTimes = 0;
1566 let rejectedTimeCandidates = 0;
1567
1568 while (
1569 (timeMatch = timeTagPattern.exec(html)) !== null &&
1570 checkedTimes < 24
1571 ) {
1572 checkedTimes++;
1573
1574 const windowStart = Math.max(0, timeMatch.index - 700);
1575 const windowEnd = Math.min(
1576 html.length,
1577 timeTagPattern.lastIndex + 700
1578 );
1579 const context = decodeHtmlText(
1580 html.slice(windowStart, windowEnd)
1581 );
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591 const exactContextTicker = context.match(
1592 /\bNEXT\s+EARNINGS\s+RELEASE\s+FOR\s+([A-Z][A-Z0-9.\-]{0,11})\b/i
1593 );
1594
1595 if (
1596 !exactContextTicker ||
1597 normalizeTicker(exactContextTicker[1]) !== wanted
1598 ) {
1599 rejectedTimeCandidates++;
1600 continue;
1601 }
1602
1603 const date = normalizeDate(timeMatch[1]);
1604
1605 if (
1606 date &&
1607 date >= rangeStart &&
1608 date <= rangeEnd
1609 ) {
1610 console.log(
1611 `OK [WSN:${ticker}] Direct earnings date from ticker-verified <time datetime>: ${date}`
1612 );
1613 return date;
1614 }
1615 }
1616
1617 if (rejectedTimeCandidates > 0) {
1618 console.log(
1619 `[WSN:${ticker}] Ignored ${rejectedTimeCandidates} unrelated <time datetime> candidate(s).`
1620 );
1621 }
1622
1623 const fullMonth =
1624 "(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:t(?:ember)?)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)";
1625
1626 const exactTickerSentence = new RegExp(
1627 `\\bnext\\s+earnings\\s+release\\s+for\\s+${escaped}\\s+is\\s+on\\s+(${fullMonth}\\.?\\s+\\d{1,2},?\\s+20\\d{2})`,
1628 "i"
1629 );
1630
1631 const eventsField = new RegExp(
1632 `\\bnext\\s+earnings\\b.{0,140}?(${fullMonth}\\.?\\s+\\d{1,2},?\\s+(?:20)?\\d{2})`,
1633 "i"
1634 );
1635
1636 for (const pattern of [
1637 exactTickerSentence,
1638 eventsField
1639 ]) {
1640 const match = body.match(pattern);
1641
1642 if (!match) {
1643 continue;
1644 }
1645
1646 const date = normalizeDate(match[1]);
1647
1648 if (
1649 date &&
1650 date >= rangeStart &&
1651 date <= rangeEnd
1652 ) {
1653 console.log(
1654 `OK [WSN:${ticker}] Direct earnings date from captured text: ${date}`
1655 );
1656 return date;
1657 }
1658 }
1659
1660 return null;
1661}
1662
1663async function tryDirectTicker(
1664 page,
1665 ticker,
1666 rangeStart,
1667 rangeEnd
1668) {
1669 const slug =
1670 encodeURIComponent(
1671 ticker.toLowerCase()
1672 );
1673
1674 const urls = [
1675 `${BASE_URL}/stocks/${slug}`,
1676 `${BASE_URL}/stocks/${slug}/eps`
1677 ];
1678
1679 for (
1680 let i = 0;
1681 i < urls.length;
1682 i++
1683 ) {
1684 const label =
1685 i === 0
1686 ? "STOCK"
1687 : "EPS";
1688
1689 const nav =
1690 await safeGoto(
1691 page,
1692 urls[i],
1693 `${ticker}:DIRECT:${label}`
1694 );
1695
1696 if (
1697 nav.networkFailed ||
1698 nav.suspiciousEmpty
1699 ) {
1700 console.log(
1701 `[WSN:${ticker}] Direct page unhealthy: ${nav.failureReason || "unusable page"}; replacing the page before the next ticker.`
1702 );
1703
1704 return {
1705 blocked: false,
1706 sessionFailure: false,
1707 pageUnhealthy: true,
1708 date: null,
1709 failureReason:
1710 nav.failureReason ||
1711 "Direct page unusable"
1712 };
1713 }
1714
1715 if (nav.blocked) {
1716 console.log(
1717 `BLOCKED [WSN:${ticker}] Direct page appears blocked.`
1718 );
1719
1720 return {
1721 blocked: true,
1722 sessionFailure: true,
1723 date: null,
1724 failureReason: "Direct page blocked"
1725 };
1726 }
1727
1728
1729
1730
1731
1732 if (nav.notFound && i === 0) {
1733 console.log(
1734 `[WSN:${ticker}] Main stock route returned HTTP 404; skipping /eps and leaving ticker unresolved.`
1735 );
1736
1737 return {
1738 blocked: false,
1739 sessionFailure: false,
1740 date: null,
1741 failureReason: "Main stock route HTTP 404"
1742 };
1743 }
1744
1745 if (nav.notFound) {
1746 continue;
1747 }
1748
1749
1750
1751
1752
1753
1754
1755 if (nav.inspectionFailed) {
1756 console.log(
1757 `[WSN:${ticker}] Direct page inspection failed; leaving ticker unresolved without /eps fallback.`
1758 );
1759
1760 return {
1761 blocked: false,
1762 sessionFailure: false,
1763 pageUnhealthy: true,
1764 date: null,
1765 failureReason:
1766 nav.failureReason ||
1767 "Direct page inspection failed"
1768 };
1769 }
1770
1771 if (!nav.state) {
1772 continue;
1773 }
1774
1775 const date =
1776 await extractDirectDate(
1777 nav.state,
1778 ticker,
1779 rangeStart,
1780 rangeEnd
1781 );
1782
1783 if (date) {
1784 return {
1785 blocked: false,
1786 date
1787 };
1788 }
1789
1790 if (i === 0) {
1791 console.log(
1792 `[WSN:${ticker}] Main stock page had no verified in-range date; trying /eps.`
1793 );
1794 }
1795 }
1796
1797 console.log(
1798 `[WSN:${ticker}] No verified direct earnings date inside ${rangeStart} -> ${rangeEnd}.`
1799 );
1800
1801 return {
1802 blocked: false,
1803 sessionFailure: false,
1804 date: null
1805 };
1806}
1807
1808
1809async function runDirectWorkers({
1810 browser,
1811 firstPage,
1812 proxyUsername,
1813 proxyPassword,
1814 tickers,
1815 freshRequested,
1816 rangeStart,
1817 rangeEnd,
1818 signal,
1819 diag = null,
1820 directPhaseBudgetMs = DIRECT_PHASE_BUDGET_MS
1821}) {
1822 const workerCount =
1823 Math.min(
1824 DIRECT_WORKERS,
1825 tickers.length
1826 );
1827
1828 const pages =
1829 [firstPage];
1830
1831 for (
1832 let i = 1;
1833 i < workerCount;
1834 i++
1835 ) {
1836 const page =
1837 await browser.newPage();
1838
1839 await configureDirectWorkerPage(
1840 page,
1841 proxyUsername,
1842 proxyPassword
1843 );
1844
1845 pages.push(page);
1846 }
1847
1848 let nextIndex =
1849 0;
1850
1851 let sessionFailed =
1852 false;
1853
1854 let failureReason =
1855 null;
1856
1857 const directPhaseStartedAt =
1858 Date.now();
1859
1860 const directPhaseDeadline =
1861 directPhaseStartedAt +
1862 directPhaseBudgetMs;
1863
1864 let directBudgetExhausted =
1865 false;
1866
1867 console.log(
1868 `[WSN DIRECT BUDGET] Shared direct-fallback budget=${(directPhaseBudgetMs / 1000).toFixed(0)}s for ${tickers.length} unresolved ticker(s); per-ticker ceiling=${(DIRECT_TICKER_TIMEOUT_MS / 1000).toFixed(0)}s; workers=${workerCount}.`
1869 );
1870
1871 const markBudgetExhausted =
1872 reason => {
1873 if (directBudgetExhausted) {
1874 return;
1875 }
1876
1877 directBudgetExhausted =
1878 true;
1879
1880 if (diag) {
1881 diag.directBudgetExhausted =
1882 true;
1883 }
1884
1885 console.log(
1886 `[WSN DIRECT BUDGET] Exhausted after ${((Date.now() - directPhaseStartedAt) / 1000).toFixed(1)}s; ${reason}. Remaining unresolved tickers will be left for persistent-cache fallback.`
1887 );
1888 };
1889
1890 async function worker(
1891 initialPage,
1892 workerNumber
1893 ) {
1894 let page = initialPage;
1895
1896 while (
1897 !sessionFailed &&
1898 !directBudgetExhausted
1899 ) {
1900 if (isAborted(signal)) {
1901 return;
1902 }
1903
1904 const remainingPhaseMs =
1905 directPhaseDeadline -
1906 Date.now();
1907
1908 if (remainingPhaseMs <= 0) {
1909 markBudgetExhausted(
1910 "shared phase deadline reached before starting another ticker"
1911 );
1912 return;
1913 }
1914
1915 const index =
1916 nextIndex++;
1917
1918 if (
1919 index >= tickers.length
1920 ) {
1921 return;
1922 }
1923
1924 const ticker =
1925 tickers[index];
1926
1927 if (
1928 freshRequested.has(ticker)
1929 ) {
1930 continue;
1931 }
1932
1933 const directStartedAt =
1934 Date.now();
1935
1936 if (diag) {
1937 diag.directAttempted++;
1938 }
1939
1940 const remainingBeforeTickerMs =
1941 Math.max(
1942 1,
1943 directPhaseDeadline -
1944 Date.now()
1945 );
1946
1947 const tickerCeilingMs =
1948 Math.min(
1949 DIRECT_TICKER_TIMEOUT_MS,
1950 remainingBeforeTickerMs
1951 );
1952
1953 const phaseLimitedTicker =
1954 tickerCeilingMs <
1955 DIRECT_TICKER_TIMEOUT_MS;
1956
1957 console.log(
1958 `[WSN DIRECT WORKER ${workerNumber}/${workerCount}] ${ticker} phaseRemaining=${(remainingBeforeTickerMs / 1000).toFixed(1)}s tickerCeiling=${(tickerCeilingMs / 1000).toFixed(1)}s`
1959 );
1960
1961 let direct;
1962 let directTimedOut = false;
1963
1964 try {
1965 direct =
1966 await Promise.race([
1967 tryDirectTicker(
1968 page,
1969 ticker,
1970 rangeStart,
1971 rangeEnd
1972 ),
1973 new Promise(resolve => {
1974 setTimeout(
1975 () => resolve({
1976 blocked: false,
1977 sessionFailure: false,
1978 date: null,
1979 timedOut: true,
1980 phaseBudgetTimeout:
1981 phaseLimitedTicker,
1982 failureReason:
1983 phaseLimitedTicker
1984 ? `Shared direct phase budget reached while processing ${ticker}`
1985 : `Per-ticker direct ceiling ${DIRECT_TICKER_TIMEOUT_MS}ms reached`
1986 }),
1987 tickerCeilingMs
1988 );
1989 })
1990 ]);
1991
1992 directTimedOut =
1993 Boolean(direct?.timedOut);
1994
1995 if (directTimedOut) {
1996 if (diag) {
1997 diag.directTickerTimeouts++;
1998 }
1999
2000 if (direct?.phaseBudgetTimeout) {
2001 console.log(
2002 `[WSN DIRECT BUDGET:${ticker}] Shared phase budget reached during this ticker after ${((Date.now() - directStartedAt) / 1000).toFixed(1)}s.`
2003 );
2004 } else {
2005 console.log(
2006 `[WSN DIRECT TIMEOUT:${ticker}] Hard per-ticker ceiling reached after ${(DIRECT_TICKER_TIMEOUT_MS / 1000).toFixed(0)}s; abandoning this page and continuing with a fresh worker page.`
2007 );
2008 }
2009 }
2010
2011 } catch (err) {
2012 console.log(
2013 `[WSN DIRECT WORKER ${workerNumber}] ${ticker} failed:`,
2014 err.message
2015 );
2016
2017 direct = {
2018 blocked: false,
2019 sessionFailure: false,
2020 date: null,
2021 failureReason: err.message
2022 };
2023 }
2024
2025 if (diag) {
2026 diag.directCompleted++;
2027
2028 if (direct.date) {
2029 diag.directMatched++;
2030 } else {
2031 diag.directNoMatch++;
2032 }
2033
2034 if (
2035 direct.blocked ||
2036 direct.sessionFailure
2037 ) {
2038 diag.directSessionFailures++;
2039 }
2040 }
2041
2042 console.log(
2043 `[WSN DIRECT DIAG:${ticker}] outcome=${direct.date ? "MATCH" : (direct.blocked ? "BLOCKED" : (direct.sessionFailure ? "SESSION_FAILURE" : (direct?.phaseBudgetTimeout ? "PHASE_BUDGET_TIMEOUT" : (directTimedOut ? "TICKER_TIMEOUT" : "NO_MATCH"))))}${direct.date ? ` date=${direct.date}` : ""} elapsed=${((Date.now() - directStartedAt) / 1000).toFixed(1)}s`
2044 );
2045
2046 if (direct.date) {
2047 freshRequested.set(
2048 ticker,
2049 direct.date
2050 );
2051
2052 await saveSnapshot({
2053 freshMap:
2054 new Map([
2055 [
2056 ticker,
2057 direct.date
2058 ]
2059 ]),
2060 rangeStart,
2061 rangeEnd,
2062 replacementDates: [],
2063 complete: false,
2064 note:
2065 `Fresh Wall Street Numbers direct result for ${ticker}.`
2066 });
2067 }
2068
2069
2070
2071
2072
2073
2074 if (
2075 direct?.phaseBudgetTimeout ||
2076 Date.now() >= directPhaseDeadline
2077 ) {
2078 try {
2079 await page.close();
2080 } catch {}
2081
2082 markBudgetExhausted(
2083 direct?.phaseBudgetTimeout
2084 ? `deadline reached while processing ${ticker}`
2085 : `deadline reached immediately after processing ${ticker}`
2086 );
2087
2088 return;
2089 }
2090
2091
2092
2093
2094
2095 if (
2096 (directTimedOut || direct?.pageUnhealthy) &&
2097 !isAborted(signal)
2098 ) {
2099 if (diag) {
2100 diag.directPageResets++;
2101 }
2102
2103 try {
2104 await page.close();
2105 } catch {}
2106
2107 try {
2108 page =
2109 await browser.newPage();
2110
2111 await configureDirectWorkerPage(
2112 page,
2113 proxyUsername,
2114 proxyPassword
2115 );
2116
2117 console.log(
2118 `[WSN DIRECT PAGE RESET:${ticker}] Fresh worker page created after ${directTimedOut ? "ticker timeout" : "unhealthy page"}; continuing remaining tickers.`
2119 );
2120
2121 } catch (err) {
2122 sessionFailed = true;
2123 failureReason =
2124 `Failed to replace timed-out worker page: ${err.message}`;
2125
2126 console.log(
2127 `[WSN DIRECT TIMEOUT:${ticker}] ${failureReason}`
2128 );
2129
2130 return;
2131 }
2132 }
2133
2134 if (
2135 diag &&
2136 diag.directAttempted % 25 === 0
2137 ) {
2138 console.log(
2139 `[WSN DIRECT PROGRESS] attempted=${diag.directAttempted}/${tickers.length} completed=${diag.directCompleted} matched=${diag.directMatched} noMatch=${diag.directNoMatch} tickerTimeouts=${diag.directTickerTimeouts} pageResets=${diag.directPageResets} budgetExhausted=${diag.directBudgetExhausted} phaseElapsed=${((Date.now() - directPhaseStartedAt) / 1000).toFixed(1)}s`
2140 );
2141 }
2142
2143 if (
2144 direct.blocked ||
2145 direct.sessionFailure
2146 ) {
2147 sessionFailed =
2148 true;
2149
2150 failureReason =
2151 direct.failureReason ||
2152 "Direct worker session failure";
2153
2154 return;
2155 }
2156
2157 if (!isAborted(signal)) {
2158 const remainingBeforeSleepMs =
2159 directPhaseDeadline -
2160 Date.now();
2161
2162 if (remainingBeforeSleepMs <= 0) {
2163 markBudgetExhausted(
2164 "deadline reached before inter-ticker delay"
2165 );
2166 return;
2167 }
2168
2169 await sleep(
2170 Math.min(
2171 350,
2172 remainingBeforeSleepMs
2173 )
2174 );
2175 }
2176 }
2177 }
2178
2179 try {
2180 await Promise.all(
2181 pages.map(
2182 (page, index) =>
2183 worker(
2184 page,
2185 index + 1
2186 )
2187 )
2188 );
2189 } finally {
2190 if (diag) {
2191 diag.elapsedDirectMs =
2192 Date.now() -
2193 directPhaseStartedAt;
2194
2195 diag.directBudgetExhausted =
2196 diag.directBudgetExhausted ||
2197 directBudgetExhausted;
2198 }
2199
2200 for (
2201 let i = 1;
2202 i < pages.length;
2203 i++
2204 ) {
2205 try {
2206 await pages[i].close();
2207 } catch {}
2208 }
2209 }
2210
2211 console.log(
2212 `[WSN DIRECT SUMMARY] requestedForDirect=${tickers.length} attempted=${diag?.directAttempted ?? "n/a"} completed=${diag?.directCompleted ?? "n/a"} matched=${diag?.directMatched ?? "n/a"} noMatch=${diag?.directNoMatch ?? "n/a"} tickerTimeouts=${diag?.directTickerTimeouts ?? "n/a"} pageResets=${diag?.directPageResets ?? "n/a"} sessionFailures=${diag?.directSessionFailures ?? "n/a"} budgetExhausted=${directBudgetExhausted} elapsedDirect=${((Date.now() - directPhaseStartedAt) / 1000).toFixed(1)}s`
2213 );
2214
2215 return {
2216 sessionFailed,
2217 failureReason,
2218 budgetExhausted:
2219 directBudgetExhausted,
2220 elapsedDirectMs:
2221 Date.now() -
2222 directPhaseStartedAt
2223 };
2224}
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237async function parseCalendarPage(
2238 page,
2239 rangeStart,
2240 rangeEnd
2241) {
2242 const raw =
2243 await page.evaluate(
2244 () => {
2245 const clean =
2246 value =>
2247 String(value || "")
2248 .replace(/\u00a0/g, " ")
2249 .replace(/\s+/g, " ")
2250 .trim();
2251
2252 const rows =
2253 [];
2254
2255 const rowElements =
2256 Array.from(
2257 document.querySelectorAll(
2258 "tr,[role=row]"
2259 )
2260 );
2261
2262 for (const row of rowElements) {
2263 const cells =
2264 Array.from(
2265 row.querySelectorAll(
2266 ":scope > th,:scope > td,[role=columnheader],[role=cell]"
2267 )
2268 )
2269 .map(
2270 cell =>
2271 clean(cell.innerText)
2272 )
2273 .filter(Boolean);
2274
2275 if (!cells.length) {
2276 continue;
2277 }
2278
2279 rows.push({
2280 cells,
2281 text:
2282 clean(row.innerText)
2283 });
2284
2285 if (rows.length >= 3000) {
2286 break;
2287 }
2288 }
2289
2290 const body =
2291 clean(
2292 document.body?.innerText || ""
2293 );
2294
2295 return {
2296 title:
2297 document.title || "",
2298 rows,
2299 bodySample:
2300 body.slice(0, 150000)
2301 };
2302 }
2303 );
2304
2305 const map =
2306 new Map();
2307
2308 let recognized =
2309 false;
2310
2311 let tickerIndex =
2312 -1;
2313
2314 let dateIndex =
2315 -1;
2316
2317 for (const row of raw.rows) {
2318 const normalized =
2319 row.cells.map(
2320 value =>
2321 value.toLowerCase()
2322 );
2323
2324 const ti =
2325 normalized.findIndex(
2326 value =>
2327 value === "ticker"
2328 );
2329
2330 const di =
2331 normalized.findIndex(
2332 value =>
2333 value.includes(
2334 "next earnings date"
2335 )
2336 );
2337
2338 if (
2339 ti >= 0 &&
2340 di >= 0
2341 ) {
2342 tickerIndex = ti;
2343 dateIndex = di;
2344 recognized = true;
2345 break;
2346 }
2347 }
2348
2349 if (
2350 tickerIndex >= 0 &&
2351 dateIndex >= 0
2352 ) {
2353 for (const row of raw.rows) {
2354 if (
2355 row.cells.length <=
2356 Math.max(
2357 tickerIndex,
2358 dateIndex
2359 )
2360 ) {
2361 continue;
2362 }
2363
2364 const ticker =
2365 normalizeTicker(
2366 row.cells[tickerIndex]
2367 );
2368
2369 const date =
2370 normalizeDate(
2371 row.cells[dateIndex]
2372 );
2373
2374 if (
2375 isTicker(ticker) &&
2376 date &&
2377 date >= rangeStart &&
2378 date <= rangeEnd
2379 ) {
2380 map.set(ticker, date);
2381 }
2382 }
2383 }
2384
2385
2386
2387
2388
2389
2390
2391 if (map.size === 0) {
2392 for (const row of raw.rows) {
2393 if (
2394 row.cells.length < 3
2395 ) {
2396 continue;
2397 }
2398
2399 const ticker =
2400 normalizeTicker(
2401 row.cells[0]
2402 );
2403
2404 if (!isTicker(ticker)) {
2405 continue;
2406 }
2407
2408 let date =
2409 null;
2410
2411
2412
2413
2414
2415 for (
2416 let i = 1;
2417 i < Math.min(
2418 row.cells.length,
2419 8
2420 );
2421 i++
2422 ) {
2423 const candidate =
2424 normalizeDate(
2425 row.cells[i]
2426 );
2427
2428 if (
2429 candidate &&
2430 candidate >= rangeStart &&
2431 candidate <= rangeEnd
2432 ) {
2433 date = candidate;
2434 break;
2435 }
2436 }
2437
2438 if (date) {
2439 map.set(ticker, date);
2440 }
2441 }
2442 }
2443
2444 if (
2445 /next earnings date/i.test(
2446 raw.bodySample
2447 ) &&
2448 /ticker/i.test(
2449 raw.bodySample
2450 )
2451 ) {
2452 recognized = true;
2453 }
2454
2455 return {
2456 map,
2457 recognized,
2458 rawRowCount:
2459 raw.rows.length,
2460 title:
2461 raw.title
2462 };
2463}
2464
2465async function scanCalendarOnce({
2466 page,
2467 rangeStart,
2468 rangeEnd,
2469 signal
2470}) {
2471 console.log(
2472 "\n[WSN] PHASE 2 - ONE Wall Street Numbers calendar scan for this run."
2473 );
2474
2475 const freshMap =
2476 new Map();
2477
2478 if (isAborted(signal)) {
2479 return {
2480 freshMap,
2481 complete: false,
2482 hardBlocked: false,
2483 sessionFailure: false,
2484 recognized: false
2485 };
2486 }
2487
2488 const nav =
2489 await safeGoto(
2490 page,
2491 CALENDAR_URL,
2492 "CALENDAR",
2493 QUALIFY_NAV_TIMEOUT_MS
2494 );
2495
2496 if (
2497 nav.networkFailed ||
2498 nav.suspiciousEmpty
2499 ) {
2500 console.log(
2501 `[WSN CALENDAR] Session/navigation failure: ${nav.failureReason || "unusable calendar page"}`
2502 );
2503
2504 return {
2505 freshMap,
2506 complete: false,
2507 hardBlocked: false,
2508 sessionFailure: true,
2509 recognized: false
2510 };
2511 }
2512
2513 if (nav.blocked) {
2514 console.log(
2515 "BLOCKED [WSN CALENDAR] Calendar blocked; no second calendar scan will be attempted."
2516 );
2517
2518 return {
2519 freshMap,
2520 complete: false,
2521 hardBlocked: true,
2522 sessionFailure: true,
2523 recognized: false
2524 };
2525 }
2526
2527 if (!nav.state) {
2528 return {
2529 freshMap,
2530 complete: false,
2531 hardBlocked: false,
2532 sessionFailure: true,
2533 recognized: false
2534 };
2535 }
2536
2537
2538
2539
2540 if (!isAborted(signal)) {
2541 await sleep(1800);
2542 }
2543
2544 if (isAborted(signal)) {
2545 return {
2546 freshMap,
2547 complete: false,
2548 hardBlocked: false,
2549 sessionFailure: false,
2550 recognized: false
2551 };
2552 }
2553
2554 const parsed =
2555 await parseCalendarPage(
2556 page,
2557 rangeStart,
2558 rangeEnd
2559 );
2560
2561 console.log(
2562 `[WSN CALENDAR] recognized=${parsed.recognized}; rawRows=${parsed.rawRowCount}; inRangeRows=${parsed.map.size}`
2563 );
2564
2565 for (
2566 const [ticker, date]
2567 of parsed.map.entries()
2568 ) {
2569 freshMap.set(ticker, date);
2570 }
2571
2572
2573
2574
2575
2576
2577
2578
2579 const complete =
2580 false;
2581
2582 if (freshMap.size > 0) {
2583 await saveSnapshot({
2584 freshMap,
2585 rangeStart,
2586 rangeEnd,
2587 replacementDates: [],
2588 complete,
2589 note:
2590 "Successful Wall Street Numbers S&P 500 earnings-calendar scan. Stored as partial because the public calendar is index-scoped."
2591 });
2592 } else {
2593 console.log(
2594 "[WSN CALENDAR] Zero verified rows; persistent CSV untouched."
2595 );
2596 }
2597
2598 return {
2599 freshMap,
2600 complete,
2601 hardBlocked: false,
2602 sessionFailure: false,
2603 recognized:
2604 parsed.recognized
2605 };
2606}
2607
2608export async function run({
2609 tickers = [],
2610 proxyConfiguration,
2611 signal,
2612 includeIncidental = false
2613} = {}) {
2614 console.log(
2615 "Launching FINAL Wall Street Numbers hybrid earnings scraper..."
2616 );
2617
2618 const wsnRunStartedAt = Date.now();
2619 const wsnDiag = {
2620 sessionsAttempted: 0,
2621 sessionsQualified: 0,
2622 sessionQualificationFailures: 0,
2623 directAttempted: 0,
2624 directCompleted: 0,
2625 directMatched: 0,
2626 directNoMatch: 0,
2627 directSessionFailures: 0,
2628 directTickerTimeouts: 0,
2629 directPageResets: 0,
2630 directBudgetExhausted: false,
2631 directBudgetCappedAfterCalendar: false,
2632 directBudgetLimitMs: DIRECT_PHASE_BUDGET_MS,
2633 elapsedDirectMs: 0,
2634 calendarAttempted: false,
2635 calendarRows: 0,
2636 calendarRecognized: false,
2637 calendarBlocked: false,
2638 calendarSessionFailure: false,
2639 aborted: false,
2640 runStartedAt: wsnRunStartedAt
2641 };
2642
2643 const requestedTickers =
2644 [
2645 ...new Set(
2646 (tickers || [])
2647 .map(normalizeTicker)
2648 .filter(isTicker)
2649 )
2650 ];
2651
2652 console.log(
2653 "Requested tickers:",
2654 requestedTickers
2655 );
2656
2657 if (!requestedTickers.length) {
2658 console.log(
2659 "No ticker supplied."
2660 );
2661
2662 return [];
2663 }
2664
2665 const startDate =
2666 new Date();
2667
2668 startDate.setHours(
2669 0,
2670 0,
2671 0,
2672 0
2673 );
2674
2675 const endDate =
2676 addDays(
2677 startDate,
2678 DAYS_TO_SCAN - 1
2679 );
2680
2681 const rangeStart =
2682 formatDate(startDate);
2683
2684 const rangeEnd =
2685 formatDate(endDate);
2686
2687 console.log(
2688 "================================================"
2689 );
2690
2691 console.log(
2692 "WALL STREET NUMBERS HYBRID EARNINGS SCRAPER"
2693 );
2694
2695 console.log(
2696 "Range:",
2697 rangeStart,
2698 "->",
2699 rangeEnd
2700 );
2701
2702 console.log(
2703 "Phase 0: persistent WallStreetNumbers CSV"
2704 );
2705
2706 console.log(
2707 `Phase 1: direct /stocks/TICKER then /stocks/TICKER/eps (${DIRECT_WORKERS} sequential worker)`
2708 );
2709
2710 console.log(
2711 `Phase 2: direct fallback for requested tickers still unresolved (${DIRECT_WORKERS} sequential worker; ${(DIRECT_TICKER_TIMEOUT_MS / 1000).toFixed(0)}s hard ceiling each)`
2712 );
2713
2714 console.log(
2715 "Phase 3: reload same persistent CSV"
2716 );
2717
2718 console.log(
2719 "Persistence: direct/calendar successes merge; zero-row failures never erase good CSV."
2720 );
2721
2722 console.log(
2723 "Calendar policy: calendar runs first, at most ONCE per Actor run; direct lookup only handles requested tickers it did not resolve."
2724 );
2725
2726 console.log(
2727 "Calendar limitation: public WSN calendar is S&P 500 scoped, so calendar cache is intentionally PARTIAL."
2728 );
2729
2730 console.log(
2731 "================================================"
2732 );
2733
2734
2735
2736
2737 const initialSnapshot =
2738 await loadSnapshot();
2739
2740 if (
2741 cacheIsFreshComplete(
2742 initialSnapshot,
2743 rangeStart,
2744 rangeEnd
2745 )
2746 ) {
2747 const cachedResults =
2748 requestedTickers
2749 .filter(
2750 ticker => {
2751 const date =
2752 initialSnapshot
2753 .map
2754 .get(ticker);
2755
2756 return (
2757 date &&
2758 date >= rangeStart &&
2759 date <= rangeEnd
2760 );
2761 }
2762 )
2763 .map(
2764 ticker => ({
2765 ticker,
2766 date:
2767 initialSnapshot
2768 .map
2769 .get(ticker),
2770 source:
2771 "WallStreetNumbers"
2772 })
2773 );
2774
2775 if (
2776 cachedResults.length ===
2777 requestedTickers.length
2778 ) {
2779 console.log(
2780 "[WSN CACHE] Fresh complete exact-range cache resolves all requested tickers."
2781 );
2782
2783 if (includeIncidental) {
2784 const allCachedResults =
2785 [...initialSnapshot.map.entries()]
2786 .filter(
2787 ([ticker, date]) =>
2788 isTicker(ticker) &&
2789 date >= rangeStart &&
2790 date <= rangeEnd
2791 )
2792 .map(
2793 ([ticker, date]) => ({
2794 ticker,
2795 date,
2796 source: "WallStreetNumbers"
2797 })
2798 );
2799
2800 console.log(
2801 `[WSN INCIDENTAL] Returning ${allCachedResults.length} in-window rows from fresh complete cache ` +
2802 `(${cachedResults.length} requested result(s)).`
2803 );
2804
2805 return allCachedResults;
2806 }
2807
2808 return cachedResults;
2809 }
2810 }
2811
2812 if (
2813 initialSnapshot.map.size > 0
2814 ) {
2815 console.log(
2816 `[WSN CACHE] Existing persistent CSV has ${initialSnapshot.map.size} row(s); retaining as fallback while live refresh proceeds.`
2817 );
2818 }
2819
2820 const freshRequested =
2821 new Map();
2822
2823 let calendarScanned =
2824 false;
2825
2826 let acceptedSession =
2827 false;
2828
2829 let activeBrowser =
2830 null;
2831
2832 const abortHandler = () => {
2833 console.log(
2834 "[WSN] Production abort requested; closing active browser."
2835 );
2836
2837 if (activeBrowser) {
2838 activeBrowser.close().catch(() => {});
2839 }
2840 };
2841
2842 signal?.addEventListener(
2843 "abort",
2844 abortHandler,
2845 { once: true }
2846 );
2847
2848 for (
2849 let attempt = 1;
2850 attempt <= MAX_SESSION_ATTEMPTS;
2851 attempt++
2852 ) {
2853 if (isAborted(signal)) {
2854 console.log(
2855 "[WSN] Production deadline reached; no further session attempts will start."
2856 );
2857 break;
2858 }
2859
2860 let browser =
2861 null;
2862
2863 try {
2864 console.log(
2865 "================================================"
2866 );
2867
2868 console.log(
2869 `[WSN] SESSION ATTEMPT ${attempt}/${MAX_SESSION_ATTEMPTS}`
2870 );
2871
2872 console.log(
2873 "================================================"
2874 );
2875
2876 wsnDiag.sessionsAttempted++;
2877
2878 const session =
2879 await createBrowserSession(
2880 proxyConfiguration,
2881 attempt
2882 );
2883
2884 browser =
2885 session.browser;
2886
2887 activeBrowser =
2888 browser;
2889
2890 if (isAborted(signal)) {
2891 break;
2892 }
2893
2894 const page =
2895 session.page;
2896
2897 const proxyOk =
2898 await proxyIpTest(page);
2899
2900 if (!proxyOk) {
2901 console.log(
2902 `[WSN] Session ${session.sessionId} rejected: proxy/IP test failed.`
2903 );
2904
2905 continue;
2906 }
2907
2908 const qualification =
2909 await qualifySession(
2910 page,
2911 session.sessionId
2912 );
2913
2914 if (qualification?.usable) {
2915 wsnDiag.sessionsQualified++;
2916 } else {
2917 wsnDiag.sessionQualificationFailures++;
2918 }
2919
2920 if (!qualification.usable) {
2921 console.log(
2922 `[WSN] Session ${session.sessionId} rejected before live work: ${qualification.reason}`
2923 );
2924
2925 continue;
2926 }
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937 calendarScanned = true;
2938 wsnDiag.calendarAttempted = true;
2939
2940 let calendarPage = null;
2941 let calendarResult = {
2942 freshMap: new Map(),
2943 complete: false,
2944 hardBlocked: false,
2945 sessionFailure: true,
2946 recognized: false
2947 };
2948
2949 try {
2950 calendarPage =
2951 await browser.newPage();
2952
2953 await configureDirectWorkerPage(
2954 calendarPage,
2955 session.proxyUsername,
2956 session.proxyPassword
2957 );
2958
2959 console.log(
2960 `[WSN CALENDAR FIRST] Fresh calendar page created for session ${session.sessionId}.`
2961 );
2962
2963 calendarResult =
2964 await scanCalendarOnce({
2965 page: calendarPage,
2966 rangeStart,
2967 rangeEnd,
2968 signal
2969 });
2970
2971 } catch (err) {
2972 console.log(
2973 `[WSN CALENDAR FIRST] Calendar setup/scan failed: ${err.message}`
2974 );
2975
2976 calendarResult = {
2977 freshMap: new Map(),
2978 complete: false,
2979 hardBlocked: false,
2980 sessionFailure: true,
2981 recognized: false
2982 };
2983
2984 } finally {
2985 if (calendarPage) {
2986 try {
2987 await calendarPage.close();
2988 } catch {}
2989 }
2990 }
2991
2992 wsnDiag.calendarRows =
2993 calendarResult.freshMap?.size || 0;
2994
2995 wsnDiag.calendarRecognized =
2996 Boolean(calendarResult.recognized);
2997
2998 wsnDiag.calendarBlocked =
2999 Boolean(calendarResult.hardBlocked);
3000
3001 wsnDiag.calendarSessionFailure =
3002 Boolean(calendarResult.sessionFailure);
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012 let requestedCalendarMatches = 0;
3013
3014 for (
3015 const ticker
3016 of requestedTickers
3017 ) {
3018 const calendarDate =
3019 calendarResult.freshMap?.get(ticker);
3020
3021 if (
3022 calendarDate &&
3023 calendarDate >= rangeStart &&
3024 calendarDate <= rangeEnd
3025 ) {
3026 freshRequested.set(
3027 ticker,
3028 calendarDate
3029 );
3030
3031 requestedCalendarMatches++;
3032 }
3033 }
3034
3035 let unresolvedForLiveRefresh =
3036 requestedTickers.filter(
3037 ticker =>
3038 !freshRequested.has(ticker)
3039 );
3040
3041 console.log(
3042 `[WSN CALENDAR FIRST DIAG] calendarRows=${wsnDiag.calendarRows} requestedMatches=${requestedCalendarMatches} unresolvedForDirect=${unresolvedForLiveRefresh.length}/${requestedTickers.length} recognized=${wsnDiag.calendarRecognized} blocked=${wsnDiag.calendarBlocked} sessionFailure=${wsnDiag.calendarSessionFailure}`
3043 );
3044
3045 if (unresolvedForLiveRefresh.length) {
3046 console.log(
3047 "[WSN] Calendar-first unresolved; direct fallback needed:",
3048 unresolvedForLiveRefresh
3049 );
3050 } else {
3051 console.log(
3052 "[WSN] Calendar-first scan resolved all requested tickers; direct fallback not needed."
3053 );
3054 }
3055
3056
3057
3058
3059
3060
3061
3062
3063 if (
3064 unresolvedForLiveRefresh.length &&
3065 !isAborted(signal)
3066 ) {
3067 const directPhaseBudgetMs =
3068 wsnDiag.calendarRecognized
3069 ? RECOGNIZED_CALENDAR_DIRECT_BUDGET_MS
3070 : DIRECT_PHASE_BUDGET_MS;
3071
3072 wsnDiag.directBudgetCappedAfterCalendar =
3073 wsnDiag.calendarRecognized;
3074 wsnDiag.directBudgetLimitMs =
3075 directPhaseBudgetMs;
3076
3077 if (wsnDiag.calendarRecognized) {
3078 console.log(
3079 `[WSN DIRECT CAP] Calendar structure was recognized; limiting unresolved direct fallback to ${(directPhaseBudgetMs / 1000).toFixed(0)}s instead of ${(DIRECT_PHASE_BUDGET_MS / 1000).toFixed(0)}s.`
3080 );
3081 }
3082
3083 const directBatch =
3084 await runDirectWorkers({
3085 browser,
3086 firstPage: page,
3087 proxyUsername:
3088 session.proxyUsername,
3089 proxyPassword:
3090 session.proxyPassword,
3091 tickers:
3092 unresolvedForLiveRefresh,
3093 freshRequested,
3094 rangeStart,
3095 rangeEnd,
3096 signal,
3097 diag: wsnDiag,
3098 directPhaseBudgetMs
3099 });
3100
3101 if (
3102 directBatch.sessionFailed
3103 ) {
3104 console.log(
3105 `[WSN] Direct fallback session ended early (${directBatch.failureReason || "unusable page"}). Calendar results and any completed direct results are preserved.`
3106 );
3107 }
3108 }
3109
3110 unresolvedForLiveRefresh =
3111 requestedTickers.filter(
3112 ticker =>
3113 !freshRequested.has(ticker)
3114 );
3115
3116 console.log(
3117 `[WSN LIVE WORK DIAG] resolved=${freshRequested.size}/${requestedTickers.length} unresolved=${unresolvedForLiveRefresh.length} directAttempted=${wsnDiag.directAttempted} directCompleted=${wsnDiag.directCompleted} directMatched=${wsnDiag.directMatched} directNoMatch=${wsnDiag.directNoMatch} directTickerTimeouts=${wsnDiag.directTickerTimeouts} directPageResets=${wsnDiag.directPageResets} directBudgetExhausted=${wsnDiag.directBudgetExhausted} elapsedDirect=${(wsnDiag.elapsedDirectMs / 1000).toFixed(1)}s aborted=${isAborted(signal)}`
3118 );
3119
3120 if (unresolvedForLiveRefresh.length) {
3121 console.log(
3122 "[WSN] Live work finished with genuinely unresolved requested tickers:",
3123 unresolvedForLiveRefresh
3124 );
3125 } else {
3126 console.log(
3127 "[WSN] Live work resolved all requested tickers."
3128 );
3129 }
3130
3131 acceptedSession =
3132 true;
3133
3134 break;
3135
3136 } catch (err) {
3137 console.log(
3138 `[WSN] Session attempt ${attempt} failed:`,
3139 err.message
3140 );
3141
3142 } finally {
3143
3144 wsnDiag.aborted = wsnDiag.aborted || isAborted(signal);
3145 const wsnElapsedSeconds = ((Date.now() - wsnRunStartedAt) / 1000).toFixed(1);
3146 let wsnDiagnosis = "COMPLETED";
3147 if (wsnDiag.aborted) wsnDiagnosis = "ABORTED_BY_PRODUCTION_TIMEOUT";
3148 else if (
3149 wsnDiag.directBudgetExhausted &&
3150 wsnDiag.directBudgetCappedAfterCalendar
3151 ) wsnDiagnosis = "DIRECT_CAPPED_AFTER_RECOGNIZED_CALENDAR";
3152 else if (wsnDiag.directBudgetExhausted) wsnDiagnosis = "DIRECT_PHASE_BUDGET_EXHAUSTED";
3153 else if (wsnDiag.calendarBlocked) wsnDiagnosis = "CALENDAR_BLOCKED";
3154 else if (wsnDiag.directSessionFailures > 0 || wsnDiag.calendarSessionFailure) wsnDiagnosis = "RESULTS_WITH_SESSION_FAILURES";
3155 else if (wsnDiag.directMatched + wsnDiag.calendarRows > 0) wsnDiagnosis = "RESULTS_FOUND";
3156 else if (
3157 wsnDiag.sessionsAttempted > 0 &&
3158 wsnDiag.sessionsQualified === 0
3159 ) wsnDiagnosis = "NO_QUALIFIED_SESSION";
3160 else wsnDiagnosis = "NO_IN_WINDOW_RESULTS";
3161
3162 console.log(
3163 `[WSN SUMMARY] requested=${requestedTickers.length} sessionsAttempted=${wsnDiag.sessionsAttempted} sessionsQualified=${wsnDiag.sessionsQualified} qualificationFailures=${wsnDiag.sessionQualificationFailures} directAttempted=${wsnDiag.directAttempted} directCompleted=${wsnDiag.directCompleted} directMatched=${wsnDiag.directMatched} directNoMatch=${wsnDiag.directNoMatch} directTickerTimeouts=${wsnDiag.directTickerTimeouts} directPageResets=${wsnDiag.directPageResets} directSessionFailures=${wsnDiag.directSessionFailures} directBudgetExhausted=${wsnDiag.directBudgetExhausted} directBudgetCappedAfterCalendar=${wsnDiag.directBudgetCappedAfterCalendar} directBudgetLimit=${(wsnDiag.directBudgetLimitMs / 1000).toFixed(0)}s elapsedDirect=${(wsnDiag.elapsedDirectMs / 1000).toFixed(1)}s calendarAttempted=${wsnDiag.calendarAttempted} calendarRows=${wsnDiag.calendarRows} calendarRecognized=${wsnDiag.calendarRecognized} calendarBlocked=${wsnDiag.calendarBlocked} calendarSessionFailure=${wsnDiag.calendarSessionFailure} aborted=${wsnDiag.aborted} elapsed=${wsnElapsedSeconds}s diagnosis=${wsnDiagnosis}`
3164 );
3165 if (browser) {
3166 try {
3167 await browser.close();
3168 } catch {}
3169 }
3170
3171 if (activeBrowser === browser) {
3172 activeBrowser = null;
3173 }
3174 }
3175
3176 if (
3177 attempt < MAX_SESSION_ATTEMPTS &&
3178 !calendarScanned
3179 ) {
3180 const delay =
3181 SESSION_RETRY_BASE_MS +
3182 attempt * 1000;
3183
3184 console.log(
3185 `[WSN] Waiting ${delay} ms before fresh session...`
3186 );
3187
3188 if (!isAborted(signal)) {
3189 await sleep(delay);
3190 }
3191 }
3192
3193 if (
3194 calendarScanned ||
3195 isAborted(signal)
3196 ) {
3197 break;
3198 }
3199 }
3200
3201 if (!acceptedSession) {
3202 console.log(
3203 "[WSN] No usable fresh session completed live work; preserved persistent CSV remains fallback."
3204 );
3205 }
3206
3207
3208
3209
3210
3211
3212
3213 const finalSnapshot =
3214 await loadSnapshot();
3215
3216 const results =
3217 [];
3218
3219 for (const ticker of requestedTickers) {
3220 if (
3221 freshRequested.has(ticker)
3222 ) {
3223 results.push({
3224 ticker,
3225 date:
3226 freshRequested.get(ticker),
3227 source:
3228 "WallStreetNumbers"
3229 });
3230
3231 continue;
3232 }
3233
3234 const cachedDate =
3235 finalSnapshot
3236 .map
3237 .get(ticker);
3238
3239 if (
3240 cachedDate &&
3241 cachedDate >= rangeStart &&
3242 cachedDate <= rangeEnd
3243 ) {
3244 console.log(
3245 `[WSN CACHE] Using persisted result for ${ticker}: ${cachedDate}`
3246 );
3247
3248 results.push({
3249 ticker,
3250 date:
3251 cachedDate,
3252 source:
3253 "WallStreetNumbers"
3254 });
3255
3256 } else {
3257 console.log(
3258 `WARN [WSN:${ticker}] No earnings date after direct lookup${calendarScanned ? " + one calendar scan" : ""} + persistent CSV lookup.`
3259 );
3260 }
3261 }
3262
3263 console.log(
3264 "Wall Street Numbers results:",
3265 JSON.stringify(results)
3266 );
3267
3268 signal?.removeEventListener(
3269 "abort",
3270 abortHandler
3271 );
3272
3273 if (includeIncidental) {
3274 const allResults =
3275 [...finalSnapshot.map.entries()]
3276 .filter(
3277 ([ticker, date]) =>
3278 isTicker(ticker) &&
3279 date >= rangeStart &&
3280 date <= rangeEnd
3281 )
3282 .map(
3283 ([ticker, date]) => ({
3284 ticker,
3285 date,
3286 source: "WallStreetNumbers"
3287 })
3288 );
3289
3290 const seen =
3291 new Set(
3292 allResults.map(
3293 row => row.ticker
3294 )
3295 );
3296
3297 for (const row of results) {
3298 if (
3299 !seen.has(row.ticker) &&
3300 row.date >= rangeStart &&
3301 row.date <= rangeEnd
3302 ) {
3303 allResults.push(row);
3304 seen.add(row.ticker);
3305 }
3306 }
3307
3308 allResults.sort(
3309 (a, b) =>
3310 a.date.localeCompare(b.date) ||
3311 a.ticker.localeCompare(b.ticker)
3312 );
3313
3314 console.log(
3315 `[WSN INCIDENTAL] Returning ${allResults.length} total in-window rows ` +
3316 `(${results.length} requested result(s); incidental S&P 500 calendar rows included).`
3317 );
3318
3319 return allResults;
3320 }
3321
3322 return results;
3323}
