1import puppeteer from "puppeteer";
2 import fs from "fs";
3 import { execFileSync } from "child_process";
4 import { KeyValueStore } from "apify";
5
6
7
8
9
10
11
12
13
14 const sleep = ms =>
15 new Promise(resolve => setTimeout(resolve, ms));
16
17 function isAborted(signal) {
18 return Boolean(signal?.aborted);
19 }
20
21 function throwIfAborted(signal) {
22 if (isAborted(signal)) {
23 const error =
24 new Error(
25 "Investing.com scraper aborted by production deadline"
26 );
27
28 error.name = "AbortError";
29 throw error;
30 }
31 }
32
33 const USER_AGENT =
34 "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " +
35 "(KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
36
37 const CALENDAR_URL =
38 "https://www.investing.com/earnings-calendar";
39
40 const CACHE_ENV = String(
41 process.env.CACHE_ENV || "prod0927"
42 )
43 .trim()
44 .toLowerCase()
45 .replace(/[^a-z0-9_-]/g, "-");
46
47 const INV_CACHE_STORE_NAME =
48 `${CACHE_ENV}-investing-week2-cache`;
49
50
51
52
53
54 const PROTOCOL_TIMEOUT_MS = 17000;
55 const WARM_NAV_TIMEOUT_MS = 3000;
56 const CALENDAR_NAV_TIMEOUT_MS = 13000;
57 const CALENDAR_PATH_WAIT_MS = 6000;
58 const CALENDAR_RENDER_SETTLE_MS = 1000;
59
60
61 const CALENDAR_ACCESS_SETTLE_MS = 1800;
62 const CALENDAR_ACCESS_REPROBE_MS = 600;
63 const CALENDAR_EMPTY_SHELL_RELOAD_TIMEOUT_MS = 3500;
64 const INV_ACCESS_TOTAL_BUDGET_MS = 38000;
65
66
67 const INV_ACCESS_FAILURE_CAP_MS = 12000;
68 const NEXT_WEEK_SETTLE_MS = 2500;
69
70
71
72
73const NEXT_WEEK_SAME_IDENTITY_RETRY_WAIT_MS = 3000;
74
75
76
77
78
79
80
81
82
83
84
85
86
87 const NEXT_WEEK_CDP_EXTRA_WAIT_MS = 6000;
88
89
90
91
92
93 const TICKER_SEARCH_NAV_TIMEOUT_MS = 10000;
94 const TICKER_EQUITY_NAV_TIMEOUT_MS = 10000;
95 const TICKER_PAGE_SETTLE_MS = 1500;
96 const TICKER_FALLBACK_TOTAL_BUDGET_MS = 60000;
97
98
99
100
101
102 const INV_TRACE_STARTED_AT = Date.now();
103
104 function invTrace(message) {
105 console.log(
106 `[INV TRACE +${Date.now() - INV_TRACE_STARTED_AT}ms] ${message}`
107 );
108 }
109
110
111
112
113
114
115
116
117
118
119
120
121
122 const INV_CACHE_KEY =
123 "INV_EARNINGS_CALENDAR_WEEK2";
124
125 const INV_CACHE_META_KEY =
126 "INV_EARNINGS_CALENDAR_WEEK2_META";
127
128 const INV_CACHE_MAX_AGE_MS =
129 24 * 60 * 60 * 1000;
130
131 const INV_POSITIVE_TICKER_CACHE_KEY =
132 "INV_EARNINGS_WEEK2_POSITIVE_TICKER_V1";
133
134 const INV_POSITIVE_TICKER_CACHE_MAX_AGE_MS =
135 48 * 60 * 60 * 1000;
136
137 const INV_NEGATIVE_CACHE_KEY =
138 "INV_EARNINGS_WEEK2_CONFIRMED_NO_DATE_V1";
139
140 const INV_NEGATIVE_CACHE_MAX_AGE_MS =
141 48 * 60 * 60 * 1000;
142
143 function csvEscape(value) {
144 const text =
145 String(value ?? "");
146
147 if (
148 text.includes(",") ||
149 text.includes('"') ||
150 text.includes("\n") ||
151 text.includes("\r")
152 ) {
153 return (
154 '"' +
155 text.replace(
156 /"/g,
157 '""'
158 ) +
159 '"'
160 );
161 }
162
163 return text;
164
165 }
166
167 function calendarMapToCsv(
168 calendarMap
169 ) {
170 const rows =
171 [
172 ...calendarMap.entries()
173 ]
174 .sort(
175 (a, b) =>
176 a[1].localeCompare(
177 b[1]
178 ) ||
179 a[0].localeCompare(
180 b[0]
181 )
182 );
183
184 const lines = [
185 "ticker,date,source"
186 ];
187
188 for (
189 const [
190 ticker,
191 date
192 ]
193 of rows
194 ) {
195 lines.push(
196 `${csvEscape(ticker)},${csvEscape(date)},${csvEscape("Investing.com")}`
197 );
198 }
199
200 return (
201 lines.join("\n") +
202 "\n"
203 );
204
205 }
206
207 function csvToCalendarMap(
208 csv
209 ) {
210 const map =
211 new Map();
212
213 if (
214 !csv ||
215 typeof csv !== "string"
216 ) {
217 return map;
218 }
219
220 const lines =
221 csv
222 .split(/\r?\n/)
223 .map(
224 line =>
225 line.trim()
226 )
227 .filter(Boolean);
228
229 if (!lines.length) {
230 return map;
231 }
232
233 for (
234 let i = 1;
235 i < lines.length;
236 i++
237 ) {
238
239
240
241
242 const comma =
243 lines[i].indexOf(",");
244
245 if (comma < 0) {
246 continue;
247 }
248
249 const ticker =
250 normalizeTicker(
251 lines[i]
252 .slice(
253 0,
254 comma
255 )
256 .replace(
257 /^"|"$/g,
258 ""
259 )
260 .replace(
261 /""/g,
262 '"'
263 )
264 );
265
266 const remainder =
267 lines[i].slice(
268 comma + 1
269 );
270
271 const secondComma =
272 remainder.indexOf(",");
273
274 const dateField =
275 secondComma >= 0
276 ? remainder.slice(
277 0,
278 secondComma
279 )
280 : remainder;
281
282 const date =
283 normalizeDate(
284 dateField
285 .replace(
286 /^"|"$/g,
287 ""
288 )
289 .replace(
290 /""/g,
291 '"'
292 )
293 );
294
295 if (
296 ticker &&
297 date
298 ) {
299 map.set(
300 ticker,
301 date
302 );
303 }
304 }
305
306 return map;
307
308 }
309
310 async function loadInvestingPositiveTickerCache(todayIso) {
311 try {
312 const store = await KeyValueStore.open(INV_CACHE_STORE_NAME);
313 const raw = await store.getValue(INV_POSITIVE_TICKER_CACHE_KEY);
314 const cache = raw && typeof raw === "object" ? { ...raw } : {};
315 const now = Date.now();
316 const map = new Map();
317 let changed = false;
318
319 for (const [rawTicker, entry] of Object.entries(cache)) {
320 const ticker = normalizeTicker(rawTicker);
321 const date = normalizeDate(entry?.date);
322 const savedAt = Date.parse(entry?.savedAt || "");
323 const ageMs = now - savedAt;
324
325 if (
326 !ticker ||
327 !date ||
328 date < todayIso ||
329 !Number.isFinite(savedAt) ||
330 ageMs < 0 ||
331 ageMs >= INV_POSITIVE_TICKER_CACHE_MAX_AGE_MS
332 ) {
333 delete cache[rawTicker];
334 changed = true;
335 continue;
336 }
337
338 map.set(ticker, date);
339 }
340
341 if (changed) {
342 await store.setValue(INV_POSITIVE_TICKER_CACHE_KEY, cache);
343 }
344
345 console.log(
346 `[INV POS CACHE] Loaded ${map.size} fresh successful direct ticker result(s) (48h TTL).`
347 );
348
349 return map;
350 } catch (error) {
351 console.log(
352 "[INV POS CACHE] Cache read failed:",
353 error.message
354 );
355 return new Map();
356 }
357 }
358
359 async function saveInvestingPositiveTicker(ticker, date) {
360 const normalizedTicker = normalizeTicker(ticker);
361 const normalizedDate = normalizeDate(date);
362
363 if (!normalizedTicker || !normalizedDate) {
364 return false;
365 }
366
367 try {
368 const store = await KeyValueStore.open(INV_CACHE_STORE_NAME);
369 const raw = await store.getValue(INV_POSITIVE_TICKER_CACHE_KEY);
370 const cache = raw && typeof raw === "object" ? { ...raw } : {};
371
372 cache[normalizedTicker] = {
373 date: normalizedDate,
374 savedAt: new Date().toISOString()
375 };
376
377 await store.setValue(INV_POSITIVE_TICKER_CACHE_KEY, cache);
378
379 console.log(
380 `[INV POS CACHE:${normalizedTicker}] Successful direct result ${normalizedDate} saved; TTL reset to 48h.`
381 );
382
383 return true;
384 } catch (error) {
385 console.log(
386 `[INV POS CACHE:${normalizedTicker}] Save failed: ${error.message}`
387 );
388 return false;
389 }
390 }
391
392 async function loadInvestingNegativeCache() {
393 try {
394 const store =
395 await KeyValueStore.open(
396 INV_CACHE_STORE_NAME
397 );
398
399 const raw =
400 await store.getValue(
401 INV_NEGATIVE_CACHE_KEY
402 );
403
404 const cache =
405 raw && typeof raw === "object"
406 ? { ...raw }
407 : {};
408
409 const now = Date.now();
410 let changed = false;
411
412 for (const [ticker, entry] of Object.entries(cache)) {
413 const confirmedAt =
414 Date.parse(
415 entry?.confirmedAt || ""
416 );
417
418 if (
419 entry?.status !== "NO_DATE" ||
420 !Number.isFinite(confirmedAt) ||
421 now - confirmedAt < 0 ||
422 now - confirmedAt >=
423 INV_NEGATIVE_CACHE_MAX_AGE_MS
424 ) {
425 delete cache[ticker];
426 changed = true;
427 }
428 }
429
430 if (changed) {
431 await store.setValue(
432 INV_NEGATIVE_CACHE_KEY,
433 cache
434 );
435 }
436
437 return cache;
438
439 } catch (error) {
440 console.log(
441 "[INV NEG CACHE] Cache read failed:",
442 error.message
443 );
444
445 return {};
446 }
447 }
448
449 function isFreshInvestingNegative(
450 negativeCache,
451 ticker
452 ) {
453 const normalizedTicker =
454 normalizeTicker(ticker);
455
456 const entry =
457 negativeCache?.[
458 normalizedTicker
459 ];
460
461 const confirmedAt =
462 Date.parse(
463 entry?.confirmedAt || ""
464 );
465
466 const ageMs =
467 Date.now() -
468 confirmedAt;
469
470 return (
471 entry?.status === "NO_DATE" &&
472 Number.isFinite(confirmedAt) &&
473 ageMs >= 0 &&
474 ageMs <
475 INV_NEGATIVE_CACHE_MAX_AGE_MS
476 );
477 }
478
479 async function saveInvestingConfirmedNegative(
480 ticker,
481 reason =
482 "VERIFIED_EQUITY_PAGE_NO_FUTURE_EARNINGS_DATE"
483 ) {
484 const normalizedTicker =
485 normalizeTicker(ticker);
486
487 if (!normalizedTicker) {
488 return false;
489 }
490
491 try {
492 const store =
493 await KeyValueStore.open(
494 INV_CACHE_STORE_NAME
495 );
496
497 const raw =
498 await store.getValue(
499 INV_NEGATIVE_CACHE_KEY
500 );
501
502 const cache =
503 raw && typeof raw === "object"
504 ? { ...raw }
505 : {};
506
507 cache[normalizedTicker] = {
508 status:
509 "NO_DATE",
510 reason,
511 confirmedAt:
512 new Date().toISOString()
513 };
514
515 await store.setValue(
516 INV_NEGATIVE_CACHE_KEY,
517 cache
518 );
519
520 console.log(
521 `[INV NEG CACHE:${normalizedTicker}] Confirmed NO_DATE saved; TTL reset to 48h.`
522 );
523
524 return true;
525
526 } catch (error) {
527 console.log(
528 `[INV NEG CACHE:${normalizedTicker}] Save failed: ${error.message}`
529 );
530
531 return false;
532 }
533 }
534
535 async function clearInvestingNegative(
536 ticker
537 ) {
538 const normalizedTicker =
539 normalizeTicker(ticker);
540
541 if (!normalizedTicker) {
542 return false;
543 }
544
545 try {
546 const store =
547 await KeyValueStore.open(
548 INV_CACHE_STORE_NAME
549 );
550
551 const raw =
552 await store.getValue(
553 INV_NEGATIVE_CACHE_KEY
554 );
555
556 if (
557 !raw ||
558 typeof raw !== "object" ||
559 !raw[normalizedTicker]
560 ) {
561 return false;
562 }
563
564 const cache = { ...raw };
565 delete cache[normalizedTicker];
566
567 await store.setValue(
568 INV_NEGATIVE_CACHE_KEY,
569 cache
570 );
571
572 console.log(
573 `[INV NEG CACHE:${normalizedTicker}] Cleared because a positive earnings date was found.`
574 );
575
576 return true;
577
578 } catch (error) {
579 console.log(
580 `[INV NEG CACHE:${normalizedTicker}] Clear failed: ${error.message}`
581 );
582
583 return false;
584 }
585 }
586
587 async function loadInvestingCache() {
588 try {
589 const store =
590 await KeyValueStore.open(
591 INV_CACHE_STORE_NAME
592 );
593
594 const meta =
595 await store.getValue(
596 INV_CACHE_META_KEY
597 );
598
599 if (
600 !meta ||
601 !meta.scrapedAt
602 ) {
603 console.log(
604 "[INV CACHE] No previous snapshot metadata found."
605 );
606
607 return null;
608 }
609
610 const scrapedAtMs =
611 Date.parse(
612 meta.scrapedAt
613 );
614
615 if (
616 !Number.isFinite(
617 scrapedAtMs
618 )
619 ) {
620 console.log(
621 "[INV CACHE] Snapshot metadata has an invalid scrapedAt timestamp."
622 );
623
624 return null;
625 }
626
627
628
629
630
631 if (
632 meta.complete === false ||
633 meta.status === "partial"
634 ) {
635 console.log(
636 "[INV CACHE] Existing snapshot is PARTIAL. Refresh required; saved rows remain available for merge."
637 );
638
639 return null;
640 }
641
642 const ageMs =
643 Date.now() -
644 scrapedAtMs;
645
646 const ageHours =
647 ageMs /
648 (
649 60 *
650 60 *
651 1000
652 );
653
654 console.log(
655 `[INV CACHE] Existing snapshot age: ${ageHours.toFixed(2)} hours.`
656 );
657
658 if (
659 ageMs < 0 ||
660 ageMs >=
661 INV_CACHE_MAX_AGE_MS
662 ) {
663 console.log(
664 "[INV CACHE] Snapshot is 24 hours old or older. Refresh required."
665 );
666
667 return null;
668 }
669
670 const csvBuffer =
671 await store.getValue(
672 INV_CACHE_KEY
673 );
674
675 const csv =
676 Buffer.isBuffer(
677 csvBuffer
678 )
679 ? csvBuffer.toString(
680 "utf8"
681 )
682 : typeof csvBuffer ===
683 "string"
684 ? csvBuffer
685 : null;
686
687 if (!csv) {
688 console.log(
689 "[INV CACHE] Metadata exists but CSV is missing. Refresh required."
690 );
691
692 return null;
693 }
694
695 const calendarMap =
696 csvToCalendarMap(
697 csv
698 );
699
700 if (
701 calendarMap.size === 0
702 ) {
703 console.log(
704 "[INV CACHE] CSV contained zero valid ticker/date pairs. Refresh required."
705 );
706
707 return null;
708 }
709
710 console.log(
711 `[INV CACHE] Using fresh persistent CSV snapshot with ${calendarMap.size} ticker/date pairs.`
712 );
713
714 return {
715 calendarMap,
716 meta
717 };
718
719 } catch (
720 error
721 ) {
722 console.log(
723 "[INV CACHE] Cache read failed:",
724 error.message
725 );
726
727 return null;
728 }
729
730 }
731
732 async function saveInvestingCache({
733 calendarMap,
734 rangeStart,
735 rangeEnd,
736 complete = true,
737 freshCoverageStart = null,
738 freshCoverageEnd = null,
739 freshRowCount = null
740 }) {
741 if (
742 !calendarMap ||
743 calendarMap.size === 0
744 ) {
745 console.log(
746 "[INV CACHE] Refusing to overwrite CSV with an empty calendar."
747 );
748
749 return false;
750 }
751
752 try {
753 const store =
754 await KeyValueStore.open(
755 INV_CACHE_STORE_NAME
756 );
757
758 const csv =
759 calendarMapToCsv(
760 calendarMap
761 );
762
763 const scrapedAt =
764 new Date()
765 .toISOString();
766
767 await store.setValue(
768 INV_CACHE_KEY,
769 csv,
770 {
771 contentType:
772 "text/csv; charset=utf-8"
773 }
774 );
775
776 await store.setValue(
777 INV_CACHE_META_KEY,
778 {
779 scrapedAt,
780 rangeStart,
781 rangeEnd,
782 rowCount:
783 calendarMap.size,
784 source:
785 "Investing.com",
786 csvKey:
787 INV_CACHE_KEY,
788 refreshAfterHours:
789 24,
790 complete:
791 Boolean(complete),
792 status:
793 complete
794 ? "complete"
795 : "partial",
796 freshCoverageStart,
797 freshCoverageEnd,
798 freshRowCount:
799 freshRowCount ??
800 calendarMap.size
801 }
802 );
803
804 console.log(
805 `[INV CACHE] Saved ${calendarMap.size} ticker/date pairs to ${INV_CACHE_KEY}.csv (${complete ? "COMPLETE" : "PARTIAL"} snapshot)`
806 );
807
808 console.log(
809 `[INV CACHE] Snapshot timestamp: ${scrapedAt}`
810 );
811
812 return true;
813
814 } catch (
815 error
816 ) {
817 console.log(
818 "[INV CACHE] Cache write failed:",
819 error.message
820 );
821
822 return false;
823 }
824
825 }
826
827 async function loadExistingInvestingSnapshotForMerge() {
828 try {
829 const store =
830 await KeyValueStore.open(
831 INV_CACHE_STORE_NAME
832 );
833
834 const csvBuffer =
835 await store.getValue(
836 INV_CACHE_KEY
837 );
838
839 const csv =
840 Buffer.isBuffer(
841 csvBuffer
842 )
843 ? csvBuffer.toString(
844 "utf8"
845 )
846 : typeof csvBuffer ===
847 "string"
848 ? csvBuffer
849 : null;
850
851 if (!csv) {
852 console.log(
853 "[INV CACHE] No existing CSV available for merge."
854 );
855
856 return {
857 calendarMap:
858 new Map(),
859 meta:
860 null
861 };
862 }
863
864 const calendarMap =
865 csvToCalendarMap(
866 csv
867 );
868
869 const meta =
870 await store.getValue(
871 INV_CACHE_META_KEY
872 );
873
874 console.log(
875 `[INV CACHE] Loaded ${calendarMap.size} existing ticker/date pairs for partial-refresh merge.`
876 );
877
878 return {
879 calendarMap,
880 meta:
881 meta || null
882 };
883
884 } catch (error) {
885 console.log(
886 "[INV CACHE] Existing CSV merge read failed:",
887 error.message
888 );
889
890 return {
891 calendarMap:
892 new Map(),
893 meta:
894 null
895 };
896 }
897 }
898
899 function mergeFreshInvestingSnapshot({
900 existingMap,
901 freshMap,
902 freshCoverageStart,
903 freshCoverageEnd,
904 rangeStart,
905 rangeEnd,
906 requestedTickers = []
907 }) {
908 const merged =
909 new Map();
910
911
912
913
914
915 for (
916 const [
917 ticker,
918 date
919 ]
920 of existingMap || []
921 ) {
922 if (
923 date >= rangeStart
924 ) {
925 merged.set(
926 ticker,
927 date
928 );
929 }
930 }
931
932
933
934
935
936 if (
937 freshCoverageStart &&
938 freshCoverageEnd
939 ) {
940 for (
941 const [
942 ticker,
943 date
944 ]
945 of [...merged.entries()]
946 ) {
947 if (
948 date >= freshCoverageStart &&
949 date <= freshCoverageEnd
950 ) {
951 merged.delete(
952 ticker
953 );
954 }
955 }
956 }
957
958 for (
959 const [
960 ticker,
961 date
962 ]
963 of freshMap || []
964 ) {
965 if (
966 date >= rangeStart
967 ) {
968 merged.set(
969 ticker,
970 date
971 );
972 }
973 }
974
975 return merged;
976 }
977
978 function buildRequestedResults(
979 requestedTickers,
980 calendarMap,
981 daysToScan,
982 sourceLabel = "Investing.com",
983 rangeStart = null,
984 rangeEnd = null,
985 diagnosticRangeStart = rangeStart,
986 diagnosticRangeEnd = rangeEnd
987 ) {
988 const results = [];
989
990 for (
991 const ticker
992 of requestedTickers
993 ) {
994 const date =
995 calendarMap.get(
996 ticker
997 );
998
999 const isCurrentOrFuture =
1000 Boolean(
1001 date &&
1002 (
1003 !rangeStart ||
1004 date >= rangeStart
1005 )
1006 );
1007
1008 if (isCurrentOrFuture) {
1009 console.log(
1010 `✅ [INV:${ticker}] Earnings date found: ${date}`
1011 );
1012
1013 results.push({
1014 ticker,
1015 date,
1016 source:
1017 sourceLabel
1018 });
1019 } else {
1020 const hasSnapshotEntry = calendarMap.has(ticker);
1021 const rawSnapshotDate = hasSnapshotEntry
1022 ? calendarMap.get(ticker)
1023 : undefined;
1024 const normalizedSnapshotDate = normalizeDate(rawSnapshotDate);
1025
1026 if (!hasSnapshotEntry) {
1027 console.log(
1028 `⚠️ [INV:${ticker}] Not present in Investing Week-2 calendar snapshot. ` +
1029 `lookup=MISS range=${diagnosticRangeStart || "NONE"}..${diagnosticRangeEnd || "NONE"}`
1030 );
1031 } else if (!normalizedSnapshotDate) {
1032 console.log(
1033 `⚠️ [INV:${ticker}] Snapshot entry has an invalid/unparseable date. ` +
1034 `rawDate=${JSON.stringify(rawSnapshotDate)} rawType=${typeof rawSnapshotDate} ` +
1035 `normalizedDate=NULL range=${diagnosticRangeStart || "NONE"}..${diagnosticRangeEnd || "NONE"}`
1036 );
1037 } else {
1038 console.log(
1039 `⚠️ [INV:${ticker}] Snapshot date rejected by current/future filter. ` +
1040 `rawDate=${JSON.stringify(rawSnapshotDate)} normalizedDate=${normalizedSnapshotDate} ` +
1041 `range=${diagnosticRangeStart || "NONE"}..${diagnosticRangeEnd || "NONE"}`
1042 );
1043 }
1044 }
1045 }
1046
1047 return results;
1048
1049 }
1050
1051 function formatDate(date) {
1052 const y = date.getUTCFullYear();
1053 const m = String(date.getUTCMonth() + 1).padStart(2, "0");
1054 const d = String(date.getUTCDate()).padStart(2, "0");
1055
1056 return `${y}-${m}-${d}`;
1057
1058 }
1059
1060 function addDays(date, days) {
1061 const result = new Date(date.getTime());
1062
1063 result.setUTCDate(
1064 result.getUTCDate() + days
1065 );
1066
1067 return result;
1068
1069 }
1070
1071 function normalizeTicker(value) {
1072 if (!value) return "";
1073
1074 return String(value)
1075 .trim()
1076 .toUpperCase()
1077 .replace(/\s+/g, "")
1078 .replace(/[^A-Z0-9.\-^=]/g, "");
1079
1080 }
1081
1082 function normalizeDate(value) {
1083 if (!value) return null;
1084
1085 const text = String(value).trim();
1086
1087 let match =
1088 text.match(
1089 /\b(20\d{2})[-/](\d{1,2})[-/](\d{1,2})\b/
1090 );
1091
1092 if (match) {
1093 return (
1094 `${match[1]}-` +
1095 `${String(match[2]).padStart(2, "0")}-` +
1096 `${String(match[3]).padStart(2, "0")}`
1097 );
1098 }
1099
1100 match =
1101 text.match(
1102 /\b(\d{1,2})\/(\d{1,2})\/(20\d{2})\b/
1103 );
1104
1105 if (match) {
1106 return (
1107 `${match[3]}-` +
1108 `${String(match[1]).padStart(2, "0")}-` +
1109 `${String(match[2]).padStart(2, "0")}`
1110 );
1111 }
1112
1113 match =
1114 text.match(
1115 /\b(\d{1,2})-(\d{1,2})-(20\d{2})\b/
1116 );
1117
1118 if (match) {
1119 return (
1120 `${match[3]}-` +
1121 `${String(match[1]).padStart(2, "0")}-` +
1122 `${String(match[2]).padStart(2, "0")}`
1123 );
1124 }
1125
1126 return null;
1127
1128 }
1129
1130 function normalizeInvestingDate(
1131 value
1132 ) {
1133 if (!value) {
1134 return null;
1135 }
1136
1137 const direct =
1138 normalizeDate(
1139 value
1140 );
1141
1142 if (direct) {
1143 return direct;
1144 }
1145
1146 const text =
1147 String(value)
1148 .replace(
1149 /\bSept\b/i,
1150 "Sep"
1151 )
1152 .trim();
1153
1154
1155
1156
1157
1158 const monthNames = {
1159 jan: 0,
1160 feb: 1,
1161 mar: 2,
1162 apr: 3,
1163 may: 4,
1164 jun: 5,
1165 jul: 6,
1166 aug: 7,
1167 sep: 8,
1168 oct: 9,
1169 nov: 10,
1170 dec: 11
1171 };
1172
1173 let match =
1174 text.match(
1175 /\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)(?:day)?[,]?\s+([A-Za-z]+)\.?\s+(\d{1,2})(?:[,]?\s+(20\d{2}))?\b/i
1176 ) ||
1177 text.match(
1178 /\b([A-Za-z]+)\.?\s+(\d{1,2})(?:[,]?\s+(20\d{2}))?\b/i
1179 );
1180
1181 if (match) {
1182 const monthKey =
1183 match[1]
1184 .slice(0, 3)
1185 .toLowerCase();
1186
1187 const month =
1188 monthNames[
1189 monthKey
1190 ];
1191
1192 if (
1193 month !== undefined
1194 ) {
1195 const now =
1196 new Date();
1197
1198 let year =
1199 match[3]
1200 ? Number(
1201 match[3]
1202 )
1203 : now.getUTCFullYear();
1204
1205
1206
1207
1208 if (!match[3]) {
1209 const nowMonth =
1210 now.getUTCMonth();
1211
1212 if (
1213 nowMonth === 11 &&
1214 month === 0
1215 ) {
1216 year++;
1217 } else if (
1218 nowMonth === 0 &&
1219 month === 11
1220 ) {
1221 year--;
1222 }
1223 }
1224
1225 return (
1226 `${year}-` +
1227 `${String(month + 1).padStart(2, "0")}-` +
1228 `${String(Number(match[2])).padStart(2, "0")}`
1229 );
1230 }
1231 }
1232
1233 match =
1234 text.match(
1235 /\b(\d{1,2})\s+([A-Za-z]+)\.?(?:\s+(20\d{2}))?\b/i
1236 );
1237
1238 if (match) {
1239 const monthKey =
1240 match[2]
1241 .slice(0, 3)
1242 .toLowerCase();
1243
1244 const month =
1245 monthNames[
1246 monthKey
1247 ];
1248
1249 if (
1250 month !== undefined
1251 ) {
1252 const now =
1253 new Date();
1254
1255 let year =
1256 match[3]
1257 ? Number(
1258 match[3]
1259 )
1260 : now.getUTCFullYear();
1261
1262 return (
1263 `${year}-` +
1264 `${String(month + 1).padStart(2, "0")}-` +
1265 `${String(Number(match[1])).padStart(2, "0")}`
1266 );
1267 }
1268 }
1269
1270 return null;
1271 }
1272
1273
1274 function looksLikeTicker(value) {
1275 if (!value) return false;
1276
1277 const ticker =
1278 normalizeTicker(value);
1279
1280 if (!ticker) return false;
1281
1282 if (
1283 ticker.length < 1 ||
1284 ticker.length > 12
1285 ) {
1286 return false;
1287 }
1288
1289 const rejected =
1290 new Set([
1291 "SYMBOL",
1292 "TICKER",
1293 "COMPANY",
1294 "NAME",
1295 "DATE",
1296 "TIME",
1297 "EPS",
1298 "EPSFORECAST",
1299 "REVENUE",
1300 "REVENUEFORECAST",
1301 "COUNTRY",
1302 "PERIOD",
1303 "ACTUAL",
1304 "FORECAST",
1305 "SURPRISE",
1306 "CALENDAR",
1307 "EARNINGS",
1308 "REPORTS",
1309 "BEFORE",
1310 "AFTER"
1311 ]);
1312
1313 if (rejected.has(ticker)) {
1314 return false;
1315 }
1316
1317 return /^[A-Z0-9][A-Z0-9.\-^=]*$/.test(
1318 ticker
1319 );
1320
1321 }
1322
1323 function isBlocked(text) {
1324 if (!text) return false;
1325
1326 const value =
1327 text.toLowerCase();
1328
1329 return (
1330 value.includes("captcha") ||
1331 value.includes("cloudflare") ||
1332 value.includes("verify you are human") ||
1333 value.includes("access denied") ||
1334 value.includes("just a moment") ||
1335 value.includes("unusual traffic") ||
1336 value.includes("pardon our interruption")
1337 );
1338
1339 }
1340
1341 function diagnostics(page, label) {
1342 return page.evaluate(label => {
1343 return {
1344 label,
1345 url: location.href,
1346 title: document.title,
1347 htmlLength:
1348 document.documentElement
1349 ?.outerHTML?.length || 0,
1350 bodyLength:
1351 document.body
1352 ?.innerText?.length || 0,
1353 bodyText:
1354 (
1355 document.body
1356 ?.innerText || ""
1357 ).slice(0, 1000)
1358 };
1359 }, label);
1360 }
1361
1362 async function launchBrowser(
1363 proxyConfiguration,
1364 sessionId,
1365 sharedApiState = null
1366 ) {
1367 console.log(
1368 "Puppeteer executable:",
1369 puppeteer.executablePath()
1370 );
1371
1372 try {
1373 console.log(
1374 "Chrome exists:",
1375 fs.existsSync(
1376 puppeteer.executablePath()
1377 )
1378 );
1379 } catch {}
1380
1381 try {
1382 const version =
1383 execFileSync(
1384 puppeteer.executablePath(),
1385 ["--version"],
1386 {
1387 encoding: "utf8",
1388 timeout: 10000
1389 }
1390 ).trim();
1391
1392 console.log(
1393 "Chrome execution test:",
1394 version
1395 );
1396 } catch (error) {
1397 console.log(
1398 "Chrome execution test failed:",
1399 error.message
1400 );
1401 }
1402
1403 let proxyUrl = null;
1404 let proxyHostPort = null;
1405 let proxyUsername = null;
1406 let proxyPassword = null;
1407
1408 if (proxyConfiguration) {
1409 proxyUrl =
1410 await proxyConfiguration.newUrl(
1411 sessionId
1412 );
1413
1414 console.log(
1415 "Investing proxy session:",
1416 sessionId
1417 );
1418
1419 console.log(
1420 "Apify proxy URL obtained:",
1421 proxyUrl ? "YES" : "NO"
1422 );
1423
1424 if (proxyUrl) {
1425 const parsedProxy =
1426 new URL(proxyUrl);
1427
1428 proxyHostPort =
1429 `${parsedProxy.hostname}:${parsedProxy.port}`;
1430
1431 proxyUsername =
1432 decodeURIComponent(
1433 parsedProxy.username || ""
1434 );
1435
1436 proxyPassword =
1437 decodeURIComponent(
1438 parsedProxy.password || ""
1439 );
1440
1441
1442
1443
1444 console.log(
1445 `[INV ACCESS IDENTITY] session=${sessionId} ` +
1446 `sessionInProxyAuth=${proxyUsername.includes(sessionId) ? "YES" : "NO"} ` +
1447 `gateway=${proxyHostPort}`
1448 );
1449
1450 console.log(
1451 "Using Apify Residential proxy:",
1452 proxyHostPort
1453 );
1454
1455 console.log(
1456 "Proxy authentication available:",
1457 Boolean(
1458 proxyUsername &&
1459 proxyPassword
1460 )
1461 );
1462 }
1463 }
1464
1465 const launchOptions = {
1466 headless: true,
1467
1468 protocolTimeout: PROTOCOL_TIMEOUT_MS,
1469
1470 args: [
1471 "--no-sandbox",
1472 "--disable-setuid-sandbox",
1473 "--disable-blink-features=AutomationControlled",
1474 "--disable-dev-shm-usage",
1475 "--disable-gpu",
1476 "--no-first-run",
1477 "--no-default-browser-check",
1478 "--disable-background-networking",
1479 "--disable-background-timer-throttling",
1480 "--disable-renderer-backgrounding",
1481 "--disable-features=Translate,BackForwardCache"
1482 ],
1483
1484 executablePath:
1485 puppeteer.executablePath()
1486 };
1487
1488 if (proxyHostPort) {
1489 launchOptions.args.push(
1490 `--proxy-server=${proxyHostPort}`
1491 );
1492 }
1493
1494 const browser =
1495 await puppeteer.launch(
1496 launchOptions
1497 );
1498
1499 const page =
1500 await browser.newPage();
1501
1502 await page.setUserAgent(
1503 USER_AGENT
1504 );
1505
1506 console.log(
1507 "Investing.com browser user agent set to:",
1508 USER_AGENT
1509 );
1510
1511 if (
1512 proxyUsername &&
1513 proxyPassword
1514 ) {
1515 await page.authenticate({
1516 username: proxyUsername,
1517 password: proxyPassword
1518 });
1519
1520 console.log(
1521 "Proxy authentication configured."
1522 );
1523 }
1524
1525 page.__investingProxyAuth =
1526 proxyUsername && proxyPassword
1527 ? {
1528 username: proxyUsername,
1529 password: proxyPassword
1530 }
1531 : null;
1532
1533 await page.setViewport({
1534 width: 1440,
1535 height: 1000,
1536 deviceScaleFactor: 1
1537 });
1538
1539 await page.setExtraHTTPHeaders({
1540 "Accept-Language":
1541 "en-US,en;q=0.9"
1542 });
1543
1544
1545
1546
1547
1548
1549
1550 await page.evaluateOnNewDocument(() => {
1551 try {
1552 Object.defineProperty(navigator, "webdriver", {
1553 get: () => false,
1554 configurable: true
1555 });
1556 } catch {}
1557 });
1558
1559
1560
1561
1562
1563
1564
1565
1566 await page.setRequestInterception(
1567 true
1568 );
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581 page.__investingBearerToken =
1582 sharedApiState?.authorization ||
1583 null;
1584
1585 if (page.__investingBearerToken) {
1586 console.log(
1587 "[INV API] Seeded new browser attempt with Bearer authorization captured by an earlier attempt."
1588 );
1589 }
1590
1591 page.on(
1592 "request",
1593 request => {
1594 try {
1595 const url =
1596 request.url().toLowerCase();
1597
1598 if (
1599 url.includes("endpoints.investing.com")
1600 ) {
1601 const observedHeaders =
1602 request.headers();
1603
1604 const observedAuthorization =
1605 observedHeaders.authorization ||
1606 observedHeaders.Authorization ||
1607 "";
1608
1609 let observedPath = "UNKNOWN";
1610 let observedRange = "";
1611
1612 try {
1613 const observedUrl =
1614 new URL(request.url());
1615
1616 observedPath =
1617 observedUrl.pathname;
1618
1619 const observedStart =
1620 observedUrl.searchParams.get("start_date");
1621
1622 const observedEnd =
1623 observedUrl.searchParams.get("end_date");
1624
1625 if (observedStart || observedEnd) {
1626 observedRange =
1627 ` range=${observedStart || "NONE"}..${observedEnd || "NONE"}`;
1628 }
1629 } catch {}
1630
1631 invTrace(
1632 `endpoint_request method=${request.method()} path=${observedPath}${observedRange} ` +
1633 `authorization=${/^Bearer\s+\S+/i.test(observedAuthorization) ? "YES" : "NO"}`
1634 );
1635
1636
1637
1638
1639
1640 if (
1641 url.includes("/instruments") &&
1642 url.includes("instrument_ids=") &&
1643 !url.includes("/earnings") &&
1644 !url.includes("key-metrics")
1645 ) {
1646 page.__investingInstrumentTemplate =
1647 request.url();
1648 }
1649
1650 const headers =
1651 request.headers();
1652
1653 const authorization =
1654 headers.authorization ||
1655 headers.Authorization ||
1656 "";
1657
1658 if (
1659 /^Bearer\s+\S+/i.test(authorization)
1660 ) {
1661 const firstCapture =
1662 !page.__investingBearerToken;
1663
1664 page.__investingBearerToken =
1665 authorization;
1666
1667 if (sharedApiState) {
1668 sharedApiState.authorization =
1669 authorization;
1670
1671 sharedApiState.capturedAt =
1672 Date.now();
1673 }
1674
1675 if (firstCapture) {
1676 console.log(
1677 "[INV API] Captured Investing Bearer authorization from first-party app traffic."
1678 );
1679 }
1680 }
1681 }
1682 } catch {}
1683
1684 const type =
1685 request.resourceType();
1686
1687
1688
1689
1690
1691
1692
1693 let hostname = "";
1694 try {
1695 hostname = new URL(request.url()).hostname.toLowerCase();
1696 } catch {}
1697
1698 const firstParty =
1699 hostname === "investing.com" ||
1700 hostname.endsWith(".investing.com");
1701
1702 const obviousNoise =
1703 /doubleclick|googlesyndication|googleadservices|googletagmanager|google-analytics|scorecardresearch|doubleverify|pubmatic|admiral|prebid|amazon-adsystem/i.test(hostname);
1704
1705 const heavyVisual =
1706 type === "image" ||
1707 type === "media" ||
1708 type === "font";
1709
1710
1711
1712
1713
1714 if (!firstParty && (heavyVisual || obviousNoise)) {
1715 request.abort().catch(() => {});
1716 return;
1717 }
1718
1719 request.continue().catch(() => {});
1720 }
1721 );
1722
1723 page.on(
1724 "requestfailed",
1725 request => {
1726 try {
1727 const type = request.resourceType();
1728 const url = request.url();
1729 let hostname = "";
1730 try { hostname = new URL(url).hostname.toLowerCase(); } catch {}
1731 const firstParty = hostname === "investing.com" || hostname.endsWith(".investing.com");
1732 if (firstParty && ["document", "script", "stylesheet", "xhr", "fetch"].includes(type)) {
1733 console.log(
1734 `[INV ACCESS NET] FAILED type=${type} url=${url.slice(0, 260)} ` +
1735 `error=${request.failure()?.errorText || "UNKNOWN"}`
1736 );
1737 }
1738 } catch {}
1739 }
1740 );
1741
1742 page.on(
1743 "response",
1744 response => {
1745 try {
1746 const responseUrl =
1747 response.url();
1748
1749 try {
1750 const responseType = response.request().resourceType();
1751 const responseHost = new URL(responseUrl).hostname.toLowerCase();
1752 const firstPartyResponse = responseHost === "investing.com" || responseHost.endsWith(".investing.com");
1753 if (firstPartyResponse && response.status() >= 400 && ["document", "script", "stylesheet", "xhr", "fetch"].includes(responseType)) {
1754 console.log(
1755 `[INV ACCESS NET] HTTP_${response.status()} type=${responseType} url=${responseUrl.slice(0, 260)}`
1756 );
1757 }
1758 } catch {}
1759
1760 if (
1761 responseUrl
1762 .toLowerCase()
1763 .includes("endpoints.investing.com")
1764 ) {
1765 const headers =
1766 response.headers();
1767
1768 let path = "UNKNOWN";
1769
1770 try {
1771 path =
1772 new URL(responseUrl).pathname;
1773 } catch {}
1774
1775 invTrace(
1776 `endpoint_response status=${response.status()} path=${path} ` +
1777 `content_type=${headers["content-type"] || "UNKNOWN"}`
1778 );
1779
1780 if (
1781 response.status() === 200 &&
1782 String(headers["content-type"] || "")
1783 .toLowerCase()
1784 .includes("json")
1785 ) {
1786 page.__investingEndpoint200Seen = true;
1787 }
1788 }
1789 } catch {}
1790 }
1791 );
1792
1793 await installInvestingPassiveApiCapture(page);
1794
1795 return {
1796 browser,
1797 page
1798 };
1799
1800 }
1801
1802
1803 async function installInvestingPassiveApiCapture(page) {
1804 if (page.__investingPassiveApiCapture?.installed) return page.__investingPassiveApiCapture;
1805 const state = { installed: true, earningsPayloads: [], instrumentPayloads: [], requestMeta: new Map(), pendingReads: new Set(), cdpSession: null, installedAt: Date.now(), earningsGetRequestsSeen: 0 };
1806 page.__investingPassiveApiCapture = state;
1807
1808 const classifyUrl = url => {
1809 try {
1810 const parsed = new URL(url);
1811 const path = parsed.pathname.toLowerCase();
1812 if (parsed.hostname !== "endpoints.investing.com") return null;
1813 if (path === "/earnings/v1/instruments/earnings") {
1814 return { kind: "earnings", startDate: parsed.searchParams.get("start_date"), endDate: parsed.searchParams.get("end_date") };
1815 }
1816 if (path.includes("/instruments") && !path.includes("/earnings") && !path.includes("key-metrics") && parsed.searchParams.has("instrument_ids")) {
1817 return { kind: "instruments", startDate: null, endDate: null };
1818 }
1819 } catch {}
1820 return null;
1821 };
1822
1823 try {
1824 const cdp = await page.target().createCDPSession();
1825 state.cdpSession = cdp;
1826 await cdp.send("Network.enable");
1827 console.log("[INV WEEK2 PASSIVE] Persistent original-page CDP collector installed before calendar navigation.");
1828
1829 cdp.on("Network.requestWillBeSent", event => {
1830 try {
1831 const classification = classifyUrl(event.request?.url || "");
1832 if (!classification) return;
1833 const method = String(event.request?.method || "").toUpperCase();
1834 if (classification.kind === "earnings" && method === "GET") {
1835 state.earningsGetRequestsSeen++;
1836 }
1837 state.requestMeta.set(event.requestId, { ...classification, method, url: event.request.url, status: null });
1838 console.log(`[INV WEEK2 PASSIVE] request kind=${classification.kind} method=${method} range=${classification.startDate || "NONE"}..${classification.endDate || "NONE"}`);
1839 } catch {}
1840 });
1841
1842 cdp.on("Network.responseReceived", event => {
1843 const meta = state.requestMeta.get(event.requestId);
1844 if (!meta) return;
1845 meta.status = event.response?.status ?? null;
1846 console.log(`[INV WEEK2 PASSIVE] response kind=${meta.kind} status=${meta.status ?? "NONE"} method=${meta.method} range=${meta.startDate || "NONE"}..${meta.endDate || "NONE"}`);
1847 });
1848
1849 cdp.on("Network.loadingFinished", event => {
1850 const meta = state.requestMeta.get(event.requestId);
1851 if (!meta) return;
1852 if (meta.method !== "GET" || meta.status !== 200) {
1853 console.log(`[INV WEEK2 PASSIVE] body skipped kind=${meta.kind} method=${meta.method} status=${meta.status ?? "NONE"}`);
1854 state.requestMeta.delete(event.requestId);
1855 return;
1856 }
1857
1858 const readPromise = (async () => {
1859 const startedAt = Date.now();
1860 try {
1861 const bodyResult = await cdp.send("Network.getResponseBody", { requestId: event.requestId });
1862 const bodyText = bodyResult?.base64Encoded ? Buffer.from(bodyResult.body || "", "base64").toString("utf8") : String(bodyResult?.body || "");
1863 const payload = JSON.parse(bodyText);
1864 const record = { payload, url: meta.url, startDate: meta.startDate, endDate: meta.endDate, capturedAt: Date.now() };
1865 if (meta.kind === "earnings") {
1866 state.earningsPayloads.push(record);
1867 console.log(`[INV WEEK2 PASSIVE] CAPTURED earnings JSON bytes=${bodyText.length} range=${meta.startDate || "NONE"}..${meta.endDate || "NONE"} total=${state.earningsPayloads.length} elapsed=${Date.now() - startedAt}ms`);
1868 } else {
1869 state.instrumentPayloads.push(record);
1870 const rows = Array.isArray(payload) ? payload.length : Array.isArray(payload?.instruments) ? payload.instruments.length : 0;
1871 console.log(`[INV WEEK2 PASSIVE] CAPTURED instruments JSON bytes=${bodyText.length} rows=${rows} total=${state.instrumentPayloads.length} elapsed=${Date.now() - startedAt}ms`);
1872 }
1873 } catch (error) {
1874 console.log(`[INV WEEK2 PASSIVE] body capture failed kind=${meta.kind} range=${meta.startDate || "NONE"}..${meta.endDate || "NONE"}: ${error.message}`);
1875 } finally {
1876 state.requestMeta.delete(event.requestId);
1877 }
1878 })();
1879 state.pendingReads.add(readPromise);
1880 readPromise.finally(() => state.pendingReads.delete(readPromise));
1881 });
1882
1883 cdp.on("Network.loadingFailed", event => {
1884 const meta = state.requestMeta.get(event.requestId);
1885 if (!meta) return;
1886 console.log(`[INV WEEK2 PASSIVE] network failed kind=${meta.kind} method=${meta.method} range=${meta.startDate || "NONE"}..${meta.endDate || "NONE"} error=${event.errorText || "UNKNOWN"}`);
1887 state.requestMeta.delete(event.requestId);
1888 });
1889 } catch (error) {
1890 state.installError = error.message;
1891 console.log(`[INV WEEK2 PASSIVE] Collector installation failed safely: ${error.message}`);
1892 }
1893 return state;
1894 }
1895
1896 function getInvestingPassiveWeekMapNow(page, weekStart, weekEnd) {
1897 const state = page.__investingPassiveApiCapture;
1898 if (!state?.installed || state.installError) return new Map();
1899 if (state.earningsPayloads.length === 0 || state.instrumentPayloads.length === 0) return new Map();
1900
1901 const earningsRows = [];
1902 for (const record of state.earningsPayloads) {
1903 const payload = record?.payload;
1904 earningsRows.push(...(Array.isArray(payload) ? payload : Array.isArray(payload?.earnings) ? payload.earnings : []));
1905 }
1906
1907 const instrumentRows = [];
1908 for (const record of state.instrumentPayloads) {
1909 const payload = record?.payload;
1910 instrumentRows.push(...(Array.isArray(payload) ? payload : Array.isArray(payload?.instruments) ? payload.instruments : []));
1911 }
1912
1913 return parseInvestingApiResponses(
1914 { earnings: earningsRows },
1915 instrumentRows,
1916 weekStart,
1917 weekEnd
1918 );
1919 }
1920
1921 async function readInvestingPassiveWeekMap(page, weekStart, weekEnd, waitMs = 30000) {
1922 const state = page.__investingPassiveApiCapture;
1923 if (!state?.installed || state.installError) {
1924 console.log(`[INV WEEK2 PASSIVE] unavailable installed=${Boolean(state?.installed)} install_error=${state?.installError || "NONE"}`);
1925 return new Map();
1926 }
1927
1928
1929
1930
1931 const startedAt = Date.now();
1932 let lastSignature = "";
1933 while (Date.now() - startedAt <= waitMs) {
1934 const signature = `${state.earningsPayloads.length}/${state.instrumentPayloads.length}/${state.pendingReads.size}`;
1935 if (signature !== lastSignature) {
1936 lastSignature = signature;
1937 console.log(`[INV WEEK2 PASSIVE] harvest wait elapsed=${Date.now() - startedAt}ms earnings_payloads=${state.earningsPayloads.length} instrument_payloads=${state.instrumentPayloads.length} pending_reads=${state.pendingReads.size}`);
1938 }
1939 if (state.earningsPayloads.length > 0 && state.instrumentPayloads.length > 0) {
1940 const weekMap = getInvestingPassiveWeekMapNow(page, weekStart, weekEnd);
1941 console.log(`[INV WEEK2 PASSIVE] parse attempt verified_week_rows=${weekMap.size} target=${weekStart}..${weekEnd}`);
1942 if (weekMap.size > 0) {
1943 console.log(`✅ [INV WEEK2 PASSIVE] Original-page first-party traffic produced ${weekMap.size} verified Week-2 ticker/date pair(s).`);
1944 return weekMap;
1945 }
1946 }
1947 if (Date.now() - startedAt >= waitMs) break;
1948 await sleep(500);
1949 }
1950 console.log(`[INV WEEK2 PASSIVE] no verified Week-2 map after ${Date.now() - startedAt}ms; earnings_payloads=${state.earningsPayloads.length} instrument_payloads=${state.instrumentPayloads.length} pending_reads=${state.pendingReads.size}. Falling back to existing Week-1 routes.`);
1951 return new Map();
1952 }
1953
1954 async function testProxy(page) {
1955 try {
1956 const response =
1957 await page.goto(
1958 "https://api.ipify.org?format=json",
1959 {
1960 waitUntil:
1961 "domcontentloaded",
1962 timeout: 15000
1963 }
1964 );
1965
1966 const body =
1967 await page.evaluate(
1968 () =>
1969 document.body
1970 ?.innerText || ""
1971 );
1972
1973 console.log(
1974 "Proxy public IP test:",
1975 body.slice(0, 200)
1976 );
1977
1978 return Boolean(response);
1979
1980 } catch (error) {
1981
1982 console.log(
1983 "Proxy public IP test failed:",
1984 error.message
1985 );
1986
1987 return false;
1988 }
1989
1990 }
1991
1992 async function waitForInvestingCalendarMaterialization(page, reason = "thin calendar") {
1993 const startedAt = Date.now();
1994 let attempt = 0;
1995 let lastInfo = null;
1996
1997 console.log(
1998 `[INV ACCESS SETTLE] ${reason}; keeping the same browser/proxy for up to ${CALENDAR_ACCESS_SETTLE_MS}ms before rotation.`
1999 );
2000
2001 while (Date.now() - startedAt < CALENDAR_ACCESS_SETTLE_MS) {
2002 attempt++;
2003 await sleep(CALENDAR_ACCESS_REPROBE_MS);
2004
2005 try {
2006 lastInfo = await Promise.race([
2007 page.evaluate(() => {
2008 const bodyText = document.body?.innerText || "";
2009 return {
2010 url: window.location.href,
2011 title: document.title || "",
2012 htmlLength: document.documentElement?.innerHTML?.length || 0,
2013 bodyLength: bodyText.length,
2014 bodyText: bodyText.slice(0, 500),
2015 tableRows: document.querySelectorAll("table tr").length,
2016 roleRows: document.querySelectorAll("[role='row']").length
2017 };
2018 }),
2019 sleep(CALENDAR_ACCESS_SETTLE_MS).then(() => {
2020 throw new Error(`settle re-probe exceeded ${CALENDAR_ACCESS_SETTLE_MS}ms`);
2021 })
2022 ]);
2023
2024 const hasRows = lastInfo.tableRows > 0 || lastInfo.roleRows > 0;
2025 const hasContent = lastInfo.bodyLength >= 1000 && lastInfo.htmlLength >= 100000;
2026 const blocked = isBlocked(`${lastInfo.title || ""}\n${lastInfo.bodyText || ""}`);
2027
2028 console.log(
2029 `[INV ACCESS SETTLE] probe=${attempt} elapsed=${Date.now() - startedAt}ms ` +
2030 `body=${lastInfo.bodyLength} html=${lastInfo.htmlLength} ` +
2031 `tableRows=${lastInfo.tableRows} roleRows=${lastInfo.roleRows} ` +
2032 `bearer=${page.__investingBearerToken ? "YES" : "NO"} ` +
2033 `endpoint200=${page.__investingEndpoint200Seen ? "YES" : "NO"} ` +
2034 `blocked=${blocked ? "YES" : "NO"}`
2035 );
2036
2037 if (!blocked && (hasRows || hasContent)) {
2038 console.log("✅ [INV ACCESS SETTLE] Calendar materialized on the existing identity; proxy rotation avoided.");
2039 return { ready: true, info: lastInfo };
2040 }
2041
2042 if (blocked) {
2043 console.log("[INV ACCESS SETTLE] Visible block/challenge detected; ending settle early.");
2044 return { ready: false, info: lastInfo, blocked: true };
2045 }
2046 } catch (error) {
2047 console.log(
2048 `[INV ACCESS SETTLE] probe=${attempt} failed: ${error.message}; ` +
2049 `bearer=${page.__investingBearerToken ? "YES" : "NO"} endpoint200=${page.__investingEndpoint200Seen ? "YES" : "NO"}`
2050 );
2051
2052
2053
2054 if (page.__investingEndpoint200Seen) {
2055 console.log("✅ [INV ACCESS SETTLE] Keeping session because first-party JSON returned HTTP 200.");
2056 return { ready: true, info: lastInfo, apiEvidenceOverride: true };
2057 }
2058 }
2059 }
2060
2061 console.log(
2062 `[INV ACCESS SETTLE] exhausted after ${Date.now() - startedAt}ms; ` +
2063 `bearer=${page.__investingBearerToken ? "YES" : "NO"} ` +
2064 `endpoint200=${page.__investingEndpoint200Seen ? "YES" : "NO"} ` +
2065 `last=${JSON.stringify(lastInfo)}`
2066 );
2067
2068 if (page.__investingEndpoint200Seen) {
2069 console.log("✅ [INV ACCESS SETTLE] HTTP-200 JSON evidence override: retaining identity despite thin DOM.");
2070 return { ready: true, info: lastInfo, apiEvidenceOverride: true };
2071 }
2072
2073 return { ready: false, info: lastInfo };
2074 }
2075
2076 async function openCalendar(page, accessPath = "direct") {
2077 console.log(
2078 "Opening Investing.com earnings calendar..."
2079 );
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091 let response = null;
2092 let navigationError = null;
2093
2094 if (accessPath === "warmed") try {
2095 console.log(
2096 "[INV] Trying same-origin calendar navigation after warm-up..."
2097 );
2098
2099 await page.evaluate(
2100 url => {
2101 window.location.assign(url);
2102 },
2103 CALENDAR_URL
2104 );
2105
2106 await page.waitForFunction(
2107 () =>
2108 window.location.pathname
2109 .toLowerCase()
2110 .includes(
2111 "earnings-calendar"
2112 ),
2113 {
2114 timeout: CALENDAR_PATH_WAIT_MS
2115 }
2116 );
2117
2118
2119
2120
2121
2122
2123 await sleep(CALENDAR_RENDER_SETTLE_MS);
2124
2125 } catch (error) {
2126 if (/ERR_TUNNEL_CONNECTION_FAILED/i.test(String(error.message))) throw error;
2127 navigationError = error;
2128
2129 console.log(
2130 "[INV] Same-origin calendar navigation did not complete:",
2131 error.message
2132 );
2133 }
2134
2135
2136
2137
2138
2139
2140 const reachedCalendar =
2141 page.url()
2142 .toLowerCase()
2143 .includes(
2144 "earnings-calendar"
2145 );
2146
2147 if (!reachedCalendar) {
2148 console.log(
2149 "[INV] Falling back to direct calendar navigation..."
2150 );
2151
2152 try {
2153 response =
2154 await page.goto(
2155 CALENDAR_URL,
2156 {
2157 waitUntil:
2158 "domcontentloaded",
2159 timeout: CALENDAR_NAV_TIMEOUT_MS
2160 }
2161 );
2162
2163 await sleep(CALENDAR_RENDER_SETTLE_MS);
2164
2165 } catch (error) {
2166 if (/ERR_TUNNEL_CONNECTION_FAILED/i.test(String(error.message))) throw error;
2167 navigationError = error;
2168
2169 console.log(
2170 "Investing calendar direct navigation failed:",
2171 error.message
2172 );
2173 }
2174 }
2175
2176 const finalUrl =
2177 page.url();
2178
2179 const onCalendar =
2180 finalUrl
2181 .toLowerCase()
2182 .includes(
2183 "earnings-calendar"
2184 );
2185
2186 console.log(
2187 "Investing calendar URL:",
2188 finalUrl
2189 );
2190
2191
2192
2193
2194
2195
2196
2197
2198 const status =
2199 response
2200 ? response.status()
2201 : onCalendar
2202 ? 200
2203 : 0;
2204
2205 console.log(
2206 "Investing calendar HTTP status:",
2207 response
2208 ? status
2209 : onCalendar
2210 ? "IN-PAGE NAVIGATION"
2211 : "NO_RESPONSE"
2212 );
2213
2214 if (!onCalendar) {
2215 console.log(
2216 "⚠️ Investing calendar URL was not reached."
2217 );
2218
2219 return {
2220 accessible: false,
2221 blocked: false,
2222 status,
2223 info: null,
2224 navigationError:
2225 navigationError?.message ||
2226 null
2227 };
2228 }
2229
2230
2231
2232
2233
2234
2235 let info = null;
2236
2237 try {
2238 info =
2239 await Promise.race([
2240 page.evaluate(() => {
2241 const bodyText =
2242 document.body
2243 ?.innerText || "";
2244
2245 return {
2246 url:
2247 window.location.href,
2248
2249 title:
2250 document.title || "",
2251
2252 htmlLength:
2253 document.documentElement
2254 ?.innerHTML
2255 ?.length || 0,
2256
2257 bodyLength:
2258 bodyText.length,
2259
2260 bodyText:
2261 bodyText.slice(
2262 0,
2263 1200
2264 ),
2265
2266 tableRows:
2267 document.querySelectorAll(
2268 "table tr"
2269 ).length,
2270
2271 roleRows:
2272 document.querySelectorAll(
2273 "[role='row']"
2274 ).length
2275 };
2276 }),
2277 sleep(6000).then(() => {
2278 throw new Error(
2279 "lightweight calendar diagnostics exceeded 6000ms"
2280 );
2281 })
2282 ]);
2283
2284 } catch (error) {
2285 console.log(
2286 "[INV] Lightweight calendar diagnostics failed:",
2287 error.message
2288 );
2289
2290
2291
2292
2293
2294
2295 const settledAfterDiagnosticTimeout =
2296 await waitForInvestingCalendarMaterialization(
2297 page,
2298 "initial calendar diagnostic timed out"
2299 );
2300
2301 if (settledAfterDiagnosticTimeout.ready) {
2302 return {
2303 accessible: true,
2304 blocked: false,
2305 status,
2306 info: settledAfterDiagnosticTimeout.info,
2307 navigationError: error.message,
2308 accessSettled: true,
2309 apiEvidenceOverride: Boolean(settledAfterDiagnosticTimeout.apiEvidenceOverride)
2310 };
2311 }
2312
2313
2314
2315
2316
2317
2318
2319
2320 if (
2321 onCalendar &&
2322 page.__investingEndpoint200Seen
2323 ) {
2324 console.log(
2325 "✅ [INV] Keeping calendar session despite DOM diagnostic timeout: first-party Investing API already returned HTTP 200 JSON."
2326 );
2327
2328 return {
2329 accessible: true,
2330 blocked: false,
2331 status,
2332 info: null,
2333 navigationError:
2334 error.message,
2335 apiEvidenceOverride: true
2336 };
2337 }
2338
2339 return {
2340 accessible: false,
2341 blocked: false,
2342 status,
2343 info: null,
2344 navigationError:
2345 error.message
2346 };
2347 }
2348
2349 console.log(
2350 "Investing calendar title:",
2351 info.title
2352 );
2353
2354 console.log(
2355 "Investing calendar diagnostics:",
2356 JSON.stringify(info)
2357 );
2358
2359 const combined =
2360 [
2361 info.title || "",
2362 info.bodyText || ""
2363 ].join("\n");
2364
2365 const blocked =
2366 status === 401 ||
2367 status === 403 ||
2368 status === 407 ||
2369 status === 429 ||
2370 status >= 500 ||
2371 isBlocked(combined);
2372
2373
2374
2375
2376
2377
2378
2379 const hasCalendarRows =
2380 info.tableRows > 0 ||
2381 info.roleRows > 0;
2382
2383 const hasCalendarContent =
2384 info.bodyLength >= 1000 &&
2385 info.htmlLength >= 100000;
2386
2387 const suspiciouslyEmpty =
2388 !hasCalendarRows &&
2389 !hasCalendarContent;
2390
2391 let accessible =
2392 !blocked &&
2393 !suspiciouslyEmpty &&
2394 onCalendar;
2395
2396 let settledInfo = null;
2397 let accessSettled = false;
2398
2399 if (!blocked && suspiciouslyEmpty && onCalendar) {
2400 const settled =
2401 await waitForInvestingCalendarMaterialization(
2402 page,
2403 `calendar shell suspiciously empty (body=${info.bodyLength}, html=${info.htmlLength}, rows=${info.tableRows + info.roleRows})`
2404 );
2405
2406 if (settled.ready) {
2407 accessible = true;
2408 accessSettled = true;
2409 settledInfo = settled.info || null;
2410 }
2411 }
2412
2413 if (blocked) {
2414 console.log(
2415 "⚠️ Investing calendar appears blocked."
2416 );
2417 } else if (suspiciouslyEmpty && !accessible) {
2418 console.log(
2419 "⚠️ Investing calendar response remained suspiciously empty after bounded settle/re-probe."
2420 );
2421 } else if (accessSettled) {
2422 console.log(
2423 "✅ Investing calendar probe passed after bounded same-identity settle/re-probe."
2424 );
2425 } else {
2426 console.log(
2427 "✅ Investing calendar probe passed."
2428 );
2429 }
2430
2431 return {
2432 accessible,
2433 blocked,
2434 status,
2435 info: settledInfo || info,
2436 accessSettled,
2437 navigationError:
2438 navigationError?.message ||
2439 null
2440 };
2441
2442 }
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466 async function parseVisibleCalendarFast(page) {
2467 let raw;
2468
2469 try {
2470 raw = await Promise.race([
2471 page.evaluate(() => {
2472 const clean = value =>
2473 String(value || "").replace(/\s+/g, " ").trim();
2474
2475 const dateTextFrom = value => {
2476 const text = clean(value);
2477 if (!text) return "";
2478
2479 const patterns = [
2480 /\b20\d{2}[-/]\d{1,2}[-/]\d{1,2}\b/i,
2481 /\b\d{1,2}\/\d{1,2}\/20\d{2}\b/i,
2482 /\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)(?:day)?[,]?\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}(?:[,]?\s+20\d{2})?\b/i,
2483 /\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}(?:[,]?\s+20\d{2})?\b/i,
2484 /\b\d{1,2}\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?(?:\s+20\d{2})?\b/i
2485 ];
2486
2487 for (const pattern of patterns) {
2488 const match = text.match(pattern);
2489 if (match) return match[0];
2490 }
2491 return "";
2492 };
2493
2494 const tickerFromText = value => {
2495 const text = clean(value).replace(/^\$/, "");
2496 if (!text || text.length > 12) return "";
2497 if (!/^[A-Z0-9][A-Z0-9.\-^=]*$/i.test(text)) return "";
2498
2499 const upper = text.toUpperCase();
2500 const rejected = new Set([
2501 "SYMBOL", "TICKER", "COMPANY", "NAME", "DATE", "TIME",
2502 "EPS", "REVENUE", "ACTUAL", "FORECAST", "SURPRISE",
2503 "EARNINGS", "COUNTRY", "PERIOD", "BEFORE", "AFTER"
2504 ]);
2505 return rejected.has(upper) ? "" : upper;
2506 };
2507
2508 const rows = [];
2509 let currentDateText = "";
2510
2511
2512 const tables = Array.from(document.querySelectorAll("table"));
2513
2514 for (const table of tables) {
2515
2516 const tableDate =
2517 dateTextFrom(table.getAttribute("data-date")) ||
2518 dateTextFrom(table.getAttribute("aria-label")) ||
2519 dateTextFrom(table.querySelector("caption")?.innerText);
2520
2521 if (tableDate) currentDateText = tableDate;
2522
2523 for (const row of table.querySelectorAll("tr")) {
2524 const rowText = clean(row.innerText);
2525 if (!rowText) continue;
2526
2527 const explicitRowDate =
2528 dateTextFrom(row.getAttribute("data-date")) ||
2529 dateTextFrom(row.getAttribute("data-event-datetime")) ||
2530 dateTextFrom(row.getAttribute("datetime")) ||
2531 dateTextFrom(row.getAttribute("aria-label")) ||
2532 dateTextFrom(rowText);
2533
2534
2535 if (explicitRowDate) currentDateText = explicitRowDate;
2536
2537 let symbol = "";
2538
2539
2540
2541 for (const link of row.querySelectorAll("a")) {
2542 const candidate = tickerFromText(link.innerText);
2543 if (candidate) {
2544 symbol = candidate;
2545 break;
2546 }
2547 }
2548
2549 if (!symbol) {
2550 for (const cell of row.querySelectorAll("th,td")) {
2551 const candidate = tickerFromText(cell.innerText);
2552 if (candidate) {
2553 symbol = candidate;
2554 break;
2555 }
2556 }
2557 }
2558
2559 if (symbol && (explicitRowDate || currentDateText)) {
2560 rows.push({
2561 symbol,
2562 dateText: explicitRowDate || currentDateText
2563 });
2564 }
2565 }
2566 }
2567
2568 return {
2569 tableCount: tables.length,
2570 rowCount: document.querySelectorAll("table tr").length,
2571 rows
2572 };
2573 }),
2574 sleep(5000).then(() => {
2575 throw new Error("fast visible-calendar harvest exceeded 5000ms");
2576 })
2577 ]);
2578 } catch (error) {
2579 console.log(`[INV FAST DOM] Early harvest failed safely: ${error.message}`);
2580 return new Map();
2581 }
2582
2583 const map = new Map();
2584
2585 for (const row of raw.rows || []) {
2586 const ticker = normalizeTicker(row.symbol);
2587 const date = normalizeInvestingDate(row.dateText);
2588
2589 if (!looksLikeTicker(ticker) || !date) continue;
2590
2591 if (!map.has(ticker)) {
2592 map.set(ticker, date);
2593 }
2594 }
2595
2596 console.log(
2597 `[INV FAST DOM] Early visible harvest: tables=${raw.tableCount || 0}; ` +
2598 `tableRows=${raw.rowCount || 0}; rawPairs=${raw.rows?.length || 0}; ` +
2599 `verifiedPairs=${map.size}.`
2600 );
2601
2602 return map;
2603 }
2604
2605
2606 async function parseVisibleCalendar(
2607 page,
2608 fallbackDate = null
2609 ) {
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622 const raw =
2623 await page.evaluate(() => {
2624 function clean(value) {
2625 return String(
2626 value || ""
2627 )
2628 .replace(/\s+/g, " ")
2629 .trim();
2630 }
2631
2632 function tickerFromText(
2633 value
2634 ) {
2635 const text =
2636 clean(value)
2637 .replace(
2638 /^\$/,
2639 ""
2640 )
2641 .trim();
2642
2643 if (
2644 !text ||
2645 text.length > 12
2646 ) {
2647 return "";
2648 }
2649
2650 if (
2651 !/^[A-Z0-9][A-Z0-9.\-^=]*$/i.test(
2652 text
2653 )
2654 ) {
2655 return "";
2656 }
2657
2658 const upper =
2659 text.toUpperCase();
2660
2661 const rejected =
2662 new Set([
2663 "SYMBOL",
2664 "TICKER",
2665 "COMPANY",
2666 "NAME",
2667 "DATE",
2668 "TIME",
2669 "EPS",
2670 "REVENUE",
2671 "ACTUAL",
2672 "FORECAST",
2673 "SURPRISE",
2674 "EARNINGS",
2675 "COUNTRY",
2676 "PERIOD",
2677 "BEFORE",
2678 "AFTER"
2679 ]);
2680
2681 return rejected.has(
2682 upper
2683 )
2684 ? ""
2685 : upper;
2686 }
2687
2688 function dateTextFrom(
2689 value
2690 ) {
2691 const text =
2692 clean(value);
2693
2694 if (!text) {
2695 return "";
2696 }
2697
2698 const patterns = [
2699 /\b20\d{2}[-/]\d{1,2}[-/]\d{1,2}\b/i,
2700 /\b\d{1,2}\/\d{1,2}\/20\d{2}\b/i,
2701 /\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)(?:day)?[,]?\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}(?:[,]?\s+20\d{2})?\b/i,
2702 /\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2}(?:[,]?\s+20\d{2})?\b/i,
2703 /\b\d{1,2}\s+(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Sept|Oct|Nov|Dec)[a-z]*\.?(?:\s+20\d{2})?\b/i
2704 ];
2705
2706 for (
2707 const pattern
2708 of patterns
2709 ) {
2710 const match =
2711 text.match(
2712 pattern
2713 );
2714
2715 if (match) {
2716 return match[0];
2717 }
2718 }
2719
2720 return "";
2721 }
2722
2723 function findDateForRow(
2724 row
2725 ) {
2726
2727
2728
2729 const directValues = [
2730 row.getAttribute(
2731 "data-date"
2732 ),
2733 row.getAttribute(
2734 "data-event-datetime"
2735 ),
2736 row.getAttribute(
2737 "data-datetime"
2738 ),
2739 row.getAttribute(
2740 "datetime"
2741 ),
2742 row.getAttribute(
2743 "title"
2744 ),
2745 row.getAttribute(
2746 "aria-label"
2747 ),
2748 row.innerText
2749 ];
2750
2751 for (
2752 const value
2753 of directValues
2754 ) {
2755 const found =
2756 dateTextFrom(
2757 value
2758 );
2759
2760 if (found) {
2761 return found;
2762 }
2763 }
2764
2765
2766
2767
2768
2769 let node =
2770 row.previousElementSibling;
2771
2772 for (
2773 let i = 0;
2774 node && i < 20;
2775 i++,
2776 node =
2777 node.previousElementSibling
2778 ) {
2779 const found =
2780 dateTextFrom(
2781 node.innerText ||
2782 node.textContent ||
2783 node.getAttribute?.(
2784 "data-date"
2785 ) ||
2786 ""
2787 );
2788
2789 if (found) {
2790 return found;
2791 }
2792 }
2793
2794
2795
2796
2797
2798 let parent =
2799 row.parentElement;
2800
2801 for (
2802 let depth = 0;
2803 parent && depth < 6;
2804 depth++,
2805 parent =
2806 parent.parentElement
2807 ) {
2808 const ownDate =
2809 dateTextFrom(
2810 parent.getAttribute?.(
2811 "data-date"
2812 ) ||
2813 parent.getAttribute?.(
2814 "data-event-datetime"
2815 ) ||
2816 ""
2817 );
2818
2819 if (ownDate) {
2820 return ownDate;
2821 }
2822
2823 let sibling =
2824 parent.previousElementSibling;
2825
2826 for (
2827 let i = 0;
2828 sibling && i < 12;
2829 i++,
2830 sibling =
2831 sibling.previousElementSibling
2832 ) {
2833 const found =
2834 dateTextFrom(
2835 sibling.innerText ||
2836 sibling.textContent ||
2837 ""
2838 );
2839
2840 if (found) {
2841 return found;
2842 }
2843 }
2844 }
2845
2846 return "";
2847 }
2848
2849 const rows = [];
2850
2851 const tableRows =
2852 Array.from(
2853 document.querySelectorAll(
2854 "table tr"
2855 )
2856 );
2857
2858 for (
2859 const row
2860 of tableRows
2861 ) {
2862 const cells =
2863 Array.from(
2864 row.querySelectorAll(
2865 "th,td"
2866 )
2867 )
2868 .map(
2869 cell =>
2870 clean(
2871 cell.innerText
2872 )
2873 )
2874 .filter(Boolean);
2875
2876 if (!cells.length) {
2877 continue;
2878 }
2879
2880 let symbol = "";
2881
2882 for (
2883 const link
2884 of row.querySelectorAll(
2885 "a"
2886 )
2887 ) {
2888 const candidate =
2889 tickerFromText(
2890 link.innerText
2891 );
2892
2893 if (candidate) {
2894 symbol =
2895 candidate;
2896 break;
2897 }
2898 }
2899
2900 if (!symbol) {
2901 for (
2902 const cell
2903 of cells
2904 ) {
2905 const candidate =
2906 tickerFromText(
2907 cell
2908 );
2909
2910 if (candidate) {
2911 symbol =
2912 candidate;
2913 break;
2914 }
2915 }
2916 }
2917
2918 if (!symbol) {
2919 continue;
2920 }
2921
2922 rows.push({
2923 symbol,
2924 dateText:
2925 findDateForRow(
2926 row
2927 ),
2928 cells
2929 });
2930 }
2931
2932
2933
2934
2935
2936
2937 const pageDateTexts = [];
2938
2939 const dateNodes =
2940 Array.from(
2941 document.querySelectorAll(
2942 "time,[datetime],[data-date],h1,h2,h3,h4,h5,h6,caption,thead,th,button"
2943 )
2944 )
2945 .slice(
2946 0,
2947 500
2948 );
2949
2950 for (
2951 const node
2952 of dateNodes
2953 ) {
2954 const values = [
2955 node.getAttribute?.(
2956 "datetime"
2957 ),
2958 node.getAttribute?.(
2959 "data-date"
2960 ),
2961 node.innerText,
2962 node.textContent
2963 ];
2964
2965 for (
2966 const value
2967 of values
2968 ) {
2969 const found =
2970 dateTextFrom(
2971 value
2972 );
2973
2974 if (
2975 found &&
2976 !pageDateTexts.includes(
2977 found
2978 )
2979 ) {
2980 pageDateTexts.push(
2981 found
2982 );
2983 }
2984 }
2985 }
2986
2987 return {
2988 rows,
2989 pageDateTexts:
2990 pageDateTexts.slice(
2991 0,
2992 50
2993 )
2994 };
2995 });
2996
2997 function normalizeInvestingDate(
2998 value
2999 ) {
3000 if (!value) {
3001 return null;
3002 }
3003
3004 const direct =
3005 normalizeDate(
3006 value
3007 );
3008
3009 if (direct) {
3010 return direct;
3011 }
3012
3013 const text =
3014 String(value)
3015 .replace(
3016 /\bSept\b/i,
3017 "Sep"
3018 )
3019 .trim();
3020
3021
3022
3023
3024
3025 const monthNames = {
3026 jan: 0,
3027 feb: 1,
3028 mar: 2,
3029 apr: 3,
3030 may: 4,
3031 jun: 5,
3032 jul: 6,
3033 aug: 7,
3034 sep: 8,
3035 oct: 9,
3036 nov: 10,
3037 dec: 11
3038 };
3039
3040 let match =
3041 text.match(
3042 /\b(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)(?:day)?[,]?\s+([A-Za-z]+)\.?\s+(\d{1,2})(?:[,]?\s+(20\d{2}))?\b/i
3043 ) ||
3044 text.match(
3045 /\b([A-Za-z]+)\.?\s+(\d{1,2})(?:[,]?\s+(20\d{2}))?\b/i
3046 );
3047
3048 if (match) {
3049 const monthKey =
3050 match[1]
3051 .slice(0, 3)
3052 .toLowerCase();
3053
3054 const month =
3055 monthNames[
3056 monthKey
3057 ];
3058
3059 if (
3060 month !== undefined
3061 ) {
3062 const now =
3063 new Date();
3064
3065 let year =
3066 match[3]
3067 ? Number(
3068 match[3]
3069 )
3070 : now.getUTCFullYear();
3071
3072
3073
3074
3075 if (!match[3]) {
3076 const nowMonth =
3077 now.getUTCMonth();
3078
3079 if (
3080 nowMonth === 11 &&
3081 month === 0
3082 ) {
3083 year++;
3084 } else if (
3085 nowMonth === 0 &&
3086 month === 11
3087 ) {
3088 year--;
3089 }
3090 }
3091
3092 return (
3093 `${year}-` +
3094 `${String(month + 1).padStart(2, "0")}-` +
3095 `${String(Number(match[2])).padStart(2, "0")}`
3096 );
3097 }
3098 }
3099
3100 match =
3101 text.match(
3102 /\b(\d{1,2})\s+([A-Za-z]+)\.?(?:\s+(20\d{2}))?\b/i
3103 );
3104
3105 if (match) {
3106 const monthKey =
3107 match[2]
3108 .slice(0, 3)
3109 .toLowerCase();
3110
3111 const month =
3112 monthNames[
3113 monthKey
3114 ];
3115
3116 if (
3117 month !== undefined
3118 ) {
3119 const now =
3120 new Date();
3121
3122 let year =
3123 match[3]
3124 ? Number(
3125 match[3]
3126 )
3127 : now.getUTCFullYear();
3128
3129 return (
3130 `${year}-` +
3131 `${String(month + 1).padStart(2, "0")}-` +
3132 `${String(Number(match[1])).padStart(2, "0")}`
3133 );
3134 }
3135 }
3136
3137 return null;
3138 }
3139
3140 const normalizedPageDates =
3141 [
3142 ...new Set(
3143 raw.pageDateTexts
3144 .map(
3145 normalizeInvestingDate
3146 )
3147 .filter(Boolean)
3148 )
3149 ];
3150
3151 const map =
3152 new Map();
3153
3154 let datedRows = 0;
3155 let undatedRows = 0;
3156
3157 for (
3158 const row
3159 of raw.rows
3160 ) {
3161 const symbol =
3162 normalizeTicker(
3163 row.symbol
3164 );
3165
3166 if (
3167 !looksLikeTicker(
3168 symbol
3169 )
3170 ) {
3171 continue;
3172 }
3173
3174 let date =
3175 normalizeInvestingDate(
3176 row.dateText
3177 );
3178
3179
3180
3181
3182
3183 if (
3184 !date &&
3185 normalizedPageDates.length === 1
3186 ) {
3187 date =
3188 normalizedPageDates[0];
3189 }
3190
3191 if (
3192 !date &&
3193 fallbackDate &&
3194 normalizedPageDates.length === 0
3195 ) {
3196
3197
3198
3199
3200 date =
3201 fallbackDate;
3202 }
3203
3204 if (!date) {
3205 undatedRows++;
3206 continue;
3207 }
3208
3209 datedRows++;
3210
3211 if (
3212 !map.has(
3213 symbol
3214 )
3215 ) {
3216 map.set(
3217 symbol,
3218 date
3219 );
3220 }
3221 }
3222
3223 console.log(
3224 `[INV] Visible page dates: ${JSON.stringify(normalizedPageDates)}`
3225 );
3226
3227 console.log(
3228 `[INV] Parsed rows with real dates: ${datedRows}; skipped undated rows: ${undatedRows}`
3229 );
3230
3231 console.log(
3232 `[INV] Parsed ${map.size} ticker/date pairs from visible calendar.`
3233 );
3234
3235 return map;
3236
3237 }
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257 async function getCalendarState(
3258 page
3259 ) {
3260
3261
3262
3263
3264
3265
3266 return page.evaluate(() => {
3267 const table =
3268 document.querySelector(
3269 "table"
3270 );
3271
3272 const roleRows =
3273 document.querySelectorAll(
3274 "[role='row']"
3275 );
3276
3277 const tableRows =
3278 document.querySelectorAll(
3279 "table tr"
3280 );
3281
3282 const rowCount =
3283 tableRows.length ||
3284 roleRows.length;
3285
3286 let sample = "";
3287
3288 if (table) {
3289 sample =
3290 (
3291 table.innerText || ""
3292 )
3293 .slice(0, 2500);
3294 } else if (roleRows.length) {
3295 sample =
3296 Array.from(
3297 roleRows
3298 )
3299 .slice(0, 20)
3300 .map(
3301 row =>
3302 (
3303 row.innerText ||
3304 ""
3305 ).slice(0, 250)
3306 )
3307 .join("\n");
3308 } else {
3309 sample =
3310 (
3311 document.body
3312 ?.innerText || ""
3313 )
3314 .slice(-2500);
3315 }
3316
3317 return {
3318 url:
3319 location.href,
3320
3321 rowCount,
3322
3323 sample
3324 };
3325 });
3326
3327 }
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371 async function findNextDayControl(
3372 page
3373 ) {
3374
3375
3376
3377
3378 return page.evaluate(() => {
3379 const elements =
3380 Array.from(
3381 document.querySelectorAll(
3382 "button,a,[role='button']"
3383 )
3384 )
3385 .slice(0, 250);
3386
3387 const matches = [];
3388
3389 for (
3390 let index = 0;
3391 index < elements.length;
3392 index++
3393 ) {
3394 const element =
3395 elements[index];
3396
3397 const text =
3398 (
3399 element.innerText ||
3400 element.textContent ||
3401 ""
3402 )
3403 .replace(/\s+/g, " ")
3404 .trim()
3405 .toLowerCase();
3406
3407 const aria =
3408 (
3409 element.getAttribute(
3410 "aria-label"
3411 ) || ""
3412 )
3413 .trim()
3414 .toLowerCase();
3415
3416 const title =
3417 (
3418 element.getAttribute(
3419 "title"
3420 ) || ""
3421 )
3422 .trim()
3423 .toLowerCase();
3424
3425 const testId =
3426 (
3427 element.getAttribute(
3428 "data-testid"
3429 ) || ""
3430 )
3431 .trim()
3432 .toLowerCase();
3433
3434 const combined =
3435 `${text} ${aria} ${title} ${testId}`;
3436
3437 let score = 0;
3438
3439 if (
3440 combined.includes(
3441 "next day"
3442 ) ||
3443 combined.includes(
3444 "next-day"
3445 ) ||
3446 combined.includes(
3447 "next_day"
3448 )
3449 ) {
3450 score += 100;
3451 }
3452
3453 if (
3454 combined.includes(
3455 "next date"
3456 ) ||
3457 combined.includes(
3458 "next-date"
3459 ) ||
3460 combined.includes(
3461 "nextdate"
3462 )
3463 ) {
3464 score += 90;
3465 }
3466
3467 if (
3468 text === "next" ||
3469 aria === "next" ||
3470 title === "next"
3471 ) {
3472 score += 70;
3473 }
3474
3475 if (
3476 combined.includes(
3477 "chevron-right"
3478 ) ||
3479 combined.includes(
3480 "arrow-right"
3481 )
3482 ) {
3483 score += 45;
3484 }
3485
3486 if (
3487 score > 0
3488 ) {
3489 matches.push({
3490 index,
3491 score,
3492 text:
3493 text.slice(
3494 0,
3495 120
3496 ),
3497 aria:
3498 aria.slice(
3499 0,
3500 120
3501 ),
3502 title:
3503 title.slice(
3504 0,
3505 120
3506 ),
3507 testId:
3508 testId.slice(
3509 0,
3510 120
3511 )
3512 });
3513 }
3514 }
3515
3516 matches.sort(
3517 (a, b) =>
3518 b.score - a.score
3519 );
3520
3521 return matches.slice(
3522 0,
3523 5
3524 );
3525 });
3526
3527 }
3528
3529 async function clickNextDay(
3530 page
3531 ) {
3532
3533
3534
3535
3536
3537 const result =
3538 await page.evaluate(() => {
3539 const elements =
3540 Array.from(
3541 document.querySelectorAll(
3542 "button,a,[role='button']"
3543 )
3544 )
3545 .slice(0, 250);
3546
3547 let best =
3548 null;
3549
3550 let bestScore =
3551 0;
3552
3553 let bestInfo =
3554 null;
3555
3556 for (
3557 const element
3558 of elements
3559 ) {
3560 const text =
3561 (
3562 element.innerText ||
3563 element.textContent ||
3564 ""
3565 )
3566 .replace(
3567 /\s+/g,
3568 " "
3569 )
3570 .trim()
3571 .toLowerCase();
3572
3573 const aria =
3574 (
3575 element.getAttribute(
3576 "aria-label"
3577 ) || ""
3578 )
3579 .trim()
3580 .toLowerCase();
3581
3582 const title =
3583 (
3584 element.getAttribute(
3585 "title"
3586 ) || ""
3587 )
3588 .trim()
3589 .toLowerCase();
3590
3591 const testId =
3592 (
3593 element.getAttribute(
3594 "data-testid"
3595 ) || ""
3596 )
3597 .trim()
3598 .toLowerCase();
3599
3600 const combined =
3601 `${text} ${aria} ${title} ${testId}`;
3602
3603 let score = 0;
3604
3605 if (
3606 combined.includes(
3607 "next day"
3608 ) ||
3609 combined.includes(
3610 "next-day"
3611 ) ||
3612 combined.includes(
3613 "next_day"
3614 )
3615 ) {
3616 score += 100;
3617 }
3618
3619 if (
3620 combined.includes(
3621 "next date"
3622 ) ||
3623 combined.includes(
3624 "next-date"
3625 ) ||
3626 combined.includes(
3627 "nextdate"
3628 )
3629 ) {
3630 score += 90;
3631 }
3632
3633 if (
3634 text === "next" ||
3635 aria === "next" ||
3636 title === "next"
3637 ) {
3638 score += 70;
3639 }
3640
3641 if (
3642 combined.includes(
3643 "chevron-right"
3644 ) ||
3645 combined.includes(
3646 "arrow-right"
3647 )
3648 ) {
3649 score += 45;
3650 }
3651
3652 if (
3653 score > bestScore
3654 ) {
3655 bestScore =
3656 score;
3657
3658 best =
3659 element;
3660
3661 bestInfo = {
3662 score,
3663 text:
3664 text.slice(
3665 0,
3666 120
3667 ),
3668 aria:
3669 aria.slice(
3670 0,
3671 120
3672 ),
3673 title:
3674 title.slice(
3675 0,
3676 120
3677 ),
3678 testId:
3679 testId.slice(
3680 0,
3681 120
3682 )
3683 };
3684 }
3685 }
3686
3687 if (!best) {
3688 return {
3689 clicked: false,
3690 control: null
3691 };
3692 }
3693
3694 best.scrollIntoView({
3695 block: "center",
3696 inline: "center"
3697 });
3698
3699 best.click();
3700
3701 return {
3702 clicked: true,
3703 control:
3704 bestInfo
3705 };
3706 });
3707
3708 console.log(
3709 "[INV] Next-day result:",
3710 JSON.stringify(
3711 result
3712 )
3713 );
3714
3715 return Boolean(
3716 result?.clicked
3717 );
3718
3719 }
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745 async function waitForCalendarChange(
3746 page,
3747 previousState
3748 ) {
3749 const started =
3750 Date.now();
3751
3752 while (
3753 Date.now() -
3754 started <
3755 12000
3756 ) {
3757 await sleep(
3758 1000
3759 );
3760
3761 try {
3762 const state =
3763 await getCalendarState(
3764 page
3765 );
3766
3767 if (
3768 state.url !==
3769 previousState.url
3770 ) {
3771 return true;
3772 }
3773
3774 if (
3775 state.rowCount !==
3776 previousState.rowCount
3777 ) {
3778 return true;
3779 }
3780
3781 if (
3782 state.sample !==
3783 previousState.sample
3784 ) {
3785 return true;
3786 }
3787
3788 } catch (
3789 error
3790 ) {
3791 console.log(
3792 "[INV] Calendar-change check failed:",
3793 error.message
3794 );
3795
3796 return false;
3797 }
3798 }
3799
3800 return false;
3801
3802 }
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814 async function clickInvestingRangeButton(
3815 page,
3816 wantedText,
3817 expectedStartDate = null,
3818 expectedEndDate = null,
3819 onClickAttempted = null,
3820 onApiPayloads = null,
3821 onEarlyComplete = null
3822 ) {
3823 const marker =
3824 "data-inv-range-target";
3825
3826 try {
3827 const found =
3828 await page.evaluate(
3829 ({ wantedText, marker }) => {
3830 const wanted =
3831 String(wantedText || "")
3832 .trim()
3833 .toLowerCase();
3834
3835 const elements =
3836 Array.from(
3837 document.querySelectorAll(
3838 "button,a,[role='button']"
3839 )
3840 );
3841
3842 let exactTextMatches = 0;
3843
3844 for (
3845 let index = 0;
3846 index < elements.length;
3847 index++
3848 ) {
3849 const element =
3850 elements[index];
3851
3852 const text =
3853 String(
3854 element.innerText ||
3855 element.textContent ||
3856 ""
3857 )
3858 .replace(/\s+/g, " ")
3859 .trim()
3860 .toLowerCase();
3861
3862 const aria =
3863 String(
3864 element.getAttribute("aria-label") || ""
3865 )
3866 .replace(/\s+/g, " ")
3867 .trim()
3868 .toLowerCase();
3869
3870 const title =
3871 String(
3872 element.getAttribute("title") || ""
3873 )
3874 .replace(/\s+/g, " ")
3875 .trim()
3876 .toLowerCase();
3877
3878 const exactText =
3879 text === wanted;
3880
3881 const exactAccessibleName =
3882 aria === wanted ||
3883 title === wanted;
3884
3885 if (exactText) {
3886 exactTextMatches++;
3887 }
3888
3889 if (
3890 exactText ||
3891 exactAccessibleName
3892 ) {
3893 element.setAttribute(marker, "1");
3894 element.scrollIntoView({
3895 block: "center",
3896 inline: "center"
3897 });
3898
3899 const rect =
3900 element.getBoundingClientRect();
3901
3902 return {
3903 found: true,
3904 text,
3905 aria,
3906 title,
3907 tag: element.tagName,
3908 index,
3909 candidateCount:
3910 elements.length,
3911 exactTextMatches,
3912 disabled:
3913 Boolean(
3914 element.disabled ||
3915 element.getAttribute("aria-disabled") === "true"
3916 ),
3917 x: rect.left + rect.width / 2,
3918 y: rect.top + rect.height / 2,
3919 width: rect.width,
3920 height: rect.height
3921 };
3922 }
3923 }
3924
3925 return {
3926 found: false,
3927 text: null,
3928 candidateCount:
3929 elements.length,
3930 exactTextMatches
3931 };
3932 },
3933 {
3934 wantedText,
3935 marker
3936 }
3937 );
3938
3939 console.log(
3940 `[INV] Range control "${wantedText}":`,
3941 JSON.stringify(found)
3942 );
3943
3944 if (
3945 !found?.found ||
3946 found.disabled
3947 ) {
3948 return false;
3949 }
3950
3951 if (
3952 !Number.isFinite(found.x) ||
3953 !Number.isFinite(found.y) ||
3954 found.width <= 0 ||
3955 found.height <= 0
3956 ) {
3957 console.log(
3958 `[INV] Range control "${wantedText}" has no usable click coordinates.`
3959 );
3960
3961 return false;
3962 }
3963
3964
3965
3966
3967
3968
3969
3970 let earningsJson = null;
3971 let instrumentsJson = null;
3972 let apiPayloadsPublished = false;
3973 const networkDiagnostics = [];
3974 let earlyCompleteValue = null;
3975
3976 const checkEarlyComplete = () => {
3977 if (earlyCompleteValue || typeof onEarlyComplete !== "function") return Boolean(earlyCompleteValue);
3978 try {
3979 const value = onEarlyComplete();
3980 if (value && (!(value instanceof Map) || value.size > 0)) {
3981 earlyCompleteValue = value;
3982 const size = value instanceof Map ? value.size : "READY";
3983 console.log(`[INV WEEK2] Passive API already complete (${size} verified row(s)); ending Next Week capture wait early.`);
3984 return true;
3985 }
3986 } catch (error) {
3987 console.log(`[INV WEEK2] Passive early-complete check failed safely: ${error.message}`);
3988 }
3989 return false;
3990 };
3991
3992
3993
3994
3995
3996
3997
3998
3999 const publishCapturedApiIfReady = () => {
4000 if (
4001 apiPayloadsPublished ||
4002 !earningsJson ||
4003 !instrumentsJson ||
4004 typeof onApiPayloads !== "function"
4005 ) {
4006 return;
4007 }
4008
4009 apiPayloadsPublished = true;
4010
4011 try {
4012 onApiPayloads(
4013 earningsJson,
4014 instrumentsJson
4015 );
4016 } catch (error) {
4017 console.log(
4018 `[INV MAP] Live Week-2 API publish failed safely: ${error.message}`
4019 );
4020 }
4021 };
4022
4023
4024
4025
4026
4027
4028 const pendingApiReads = new Set();
4029
4030 const trackApiRead = promise => {
4031 pendingApiReads.add(promise);
4032 promise.finally(() => pendingApiReads.delete(promise));
4033 return promise;
4034 };
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047 let cdpSession = null;
4048 const cdpResponses = new Map();
4049
4050 const isExpectedEarningsUrl = url => {
4051 try {
4052 const parsed = new URL(url);
4053
4054 if (
4055 parsed.hostname !== "endpoints.investing.com" ||
4056 parsed.pathname !== "/earnings/v1/instruments/earnings"
4057 ) {
4058 return false;
4059 }
4060
4061 if (!expectedStartDate) {
4062 return true;
4063 }
4064
4065 const expectedStartIso =
4066 `${expectedStartDate}T00:00:00.000Z`;
4067
4068 const expectedEndIso =
4069 expectedEndDate
4070 ? `${expectedEndDate}T00:00:00.000Z`
4071 : null;
4072
4073 return (
4074 parsed.searchParams.get("start_date") ===
4075 expectedStartIso &&
4076 (
4077 !expectedEndIso ||
4078 parsed.searchParams.get("end_date") ===
4079 expectedEndIso
4080 )
4081 );
4082 } catch {
4083 return false;
4084 }
4085 };
4086
4087 const isInstrumentMetadataUrl = url => {
4088 try {
4089 const parsed = new URL(url);
4090 const path = parsed.pathname.toLowerCase();
4091
4092 return (
4093 parsed.hostname === "endpoints.investing.com" &&
4094 path.includes("/instruments") &&
4095 !path.includes("/earnings") &&
4096 !path.includes("key-metrics") &&
4097 parsed.searchParams.has("instrument_ids")
4098 );
4099 } catch {
4100 return false;
4101 }
4102 };
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112 const summarizeInstrumentPayload = payload => {
4113 const rows = Array.isArray(payload)
4114 ? payload
4115 : Array.isArray(payload?.instruments)
4116 ? payload.instruments
4117 : [];
4118
4119 const validRows = rows.filter(row => {
4120 if (!row || typeof row !== "object") {
4121 return false;
4122 }
4123
4124 const instrumentId =
4125 row.instrument_id ??
4126 row.instrumentId ??
4127 row.id;
4128
4129 return (
4130 instrumentId !== undefined &&
4131 instrumentId !== null &&
4132 String(row.display_symbol || "").trim()
4133 );
4134 });
4135
4136 const sample = validRows[0] || null;
4137
4138 return {
4139 shape: Array.isArray(payload) ? "array" : typeof payload,
4140 rows: rows.length,
4141 validRows: validRows.length,
4142 sampleId:
4143 sample?.instrument_id ??
4144 sample?.instrumentId ??
4145 sample?.id ??
4146 null,
4147 sampleSymbol: sample?.display_symbol ?? null
4148 };
4149 };
4150
4151 const acceptInstrumentPayload = (payload, sourceLabel) => {
4152 const summary = summarizeInstrumentPayload(payload);
4153
4154 console.log(
4155 `[INV INSTRUMENTS] ${sourceLabel} shape=${summary.shape} rows=${summary.rows} ` +
4156 `with_instrument_id_and_display_symbol=${summary.validRows} ` +
4157 `sample_id=${summary.sampleId ?? "NONE"} sample_symbol=${summary.sampleSymbol ?? "NONE"}`
4158 );
4159
4160 if (summary.validRows === 0) {
4161
4162
4163
4164
4165
4166
4167 const diagnosticRows = Array.isArray(payload)
4168 ? payload
4169 : Array.isArray(payload?.instruments)
4170 ? payload.instruments
4171 : [];
4172
4173 const firstRow =
4174 diagnosticRows.find(
4175 row =>
4176 row &&
4177 typeof row === "object"
4178 ) || null;
4179
4180 if (firstRow) {
4181 const topLevelKeys =
4182 Object.keys(firstRow)
4183 .slice(0, 80);
4184
4185 console.log(
4186 `[INV INSTRUMENTS DIAG] ${sourceLabel} first_row_keys=${JSON.stringify(topLevelKeys)}`
4187 );
4188
4189 try {
4190 const sampleJson =
4191 JSON.stringify(firstRow);
4192
4193 console.log(
4194 `[INV INSTRUMENTS DIAG] ${sourceLabel} first_row_sample=${sampleJson.slice(0, 4000)}`
4195 );
4196 } catch (error) {
4197 console.log(
4198 `[INV INSTRUMENTS DIAG] ${sourceLabel} first_row_sample stringify failed: ${error.message}`
4199 );
4200 }
4201
4202 const nestedObjectSummary = [];
4203
4204 for (
4205 const [key, value]
4206 of Object.entries(firstRow)
4207 ) {
4208 if (
4209 value &&
4210 typeof value === "object"
4211 ) {
4212 nestedObjectSummary.push({
4213 key,
4214 type:
4215 Array.isArray(value)
4216 ? "array"
4217 : "object",
4218 keys:
4219 Array.isArray(value)
4220 ? (
4221 value[0] &&
4222 typeof value[0] === "object"
4223 ? Object.keys(value[0]).slice(0, 40)
4224 : []
4225 )
4226 : Object.keys(value).slice(0, 40)
4227 });
4228 }
4229 }
4230
4231 console.log(
4232 `[INV INSTRUMENTS DIAG] ${sourceLabel} nested_objects=${JSON.stringify(nestedObjectSummary).slice(0, 4000)}`
4233 );
4234 } else {
4235 console.log(
4236 `[INV INSTRUMENTS DIAG] ${sourceLabel} no object row available for structural sample.`
4237 );
4238 }
4239
4240 console.log(
4241 `[INV INSTRUMENTS] Rejected ${sourceLabel}: no id/instrument_id + display_symbol mappings.`
4242 );
4243 return false;
4244 }
4245
4246 instrumentsJson = payload;
4247 publishCapturedApiIfReady();
4248
4249 console.log(
4250 `[INV INSTRUMENTS] Accepted ${sourceLabel}.`
4251 );
4252
4253 return true;
4254 };
4255
4256
4257
4258
4259
4260
4261
4262 const cdpRequests = new Map();
4263
4264 try {
4265 cdpSession =
4266 await page.target().createCDPSession();
4267
4268 await cdpSession.send(
4269 "Network.enable"
4270 );
4271
4272 cdpSession.on(
4273 "Network.requestWillBeSent",
4274 event => {
4275 try {
4276 const url =
4277 event.request?.url || "";
4278
4279 const method =
4280 String(
4281 event.request?.method || ""
4282 ).toUpperCase();
4283
4284 if (
4285 isExpectedEarningsUrl(url) ||
4286 isInstrumentMetadataUrl(url)
4287 ) {
4288 cdpRequests.set(
4289 event.requestId,
4290 {
4291 url,
4292 method,
4293 kind:
4294 isExpectedEarningsUrl(url)
4295 ? "earnings"
4296 : "instruments"
4297 }
4298 );
4299
4300 let range = "";
4301
4302 try {
4303 const parsed =
4304 new URL(url);
4305
4306 range =
4307 ` range=${parsed.searchParams.get("start_date") || "NONE"}..${parsed.searchParams.get("end_date") || "NONE"}`;
4308 } catch {}
4309
4310 invTrace(
4311 `cdp_request_seen id=${event.requestId} kind=${cdpRequests.get(event.requestId).kind} ` +
4312 `method=${method || "UNKNOWN"}${range}`
4313 );
4314 }
4315 } catch {}
4316 }
4317 );
4318
4319 cdpSession.on(
4320 "Network.responseReceived",
4321 event => {
4322 try {
4323 const tracked =
4324 cdpRequests.get(
4325 event.requestId
4326 );
4327
4328 if (!tracked) {
4329 return;
4330 }
4331
4332 const status =
4333 event.response?.status || 0;
4334
4335 const contentType =
4336 event.response?.mimeType ||
4337 event.response?.headers?.["content-type"] ||
4338 event.response?.headers?.["Content-Type"] ||
4339 "UNKNOWN";
4340
4341 invTrace(
4342 `cdp_response_seen id=${event.requestId} kind=${tracked.kind} ` +
4343 `method=${tracked.method || "UNKNOWN"} status=${status} content_type=${contentType}`
4344 );
4345
4346
4347
4348
4349
4350
4351 if (
4352 tracked.method !== "GET" ||
4353 status !== 200
4354 ) {
4355 if (tracked.method === "OPTIONS") {
4356 invTrace(
4357 `cdp_response_ignored id=${event.requestId} kind=${tracked.kind} reason=OPTIONS_PREFLIGHT`
4358 );
4359 }
4360
4361 return;
4362 }
4363
4364 cdpResponses.set(
4365 event.requestId,
4366 {
4367 ...tracked,
4368 status,
4369 contentType
4370 }
4371 );
4372 } catch {}
4373 }
4374 );
4375
4376 cdpSession.on(
4377 "Network.loadingFinished",
4378 event => {
4379 const tracked =
4380 cdpRequests.get(
4381 event.requestId
4382 );
4383
4384 const captured =
4385 cdpResponses.get(
4386 event.requestId
4387 );
4388
4389 if (tracked) {
4390 invTrace(
4391 `cdp_loading_finished id=${event.requestId} kind=${tracked.kind} ` +
4392 `method=${tracked.method || "UNKNOWN"} eligible=${captured ? "YES" : "NO"}`
4393 );
4394 }
4395
4396 cdpRequests.delete(
4397 event.requestId
4398 );
4399
4400 if (!captured) {
4401 return;
4402 }
4403
4404 cdpResponses.delete(
4405 event.requestId
4406 );
4407
4408 trackApiRead(
4409 (async () => {
4410 try {
4411 invTrace(
4412 `cdp_body_read_start id=${event.requestId} kind=${captured.kind} method=${captured.method}`
4413 );
4414
4415 const result =
4416 await cdpSession.send(
4417 "Network.getResponseBody",
4418 {
4419 requestId:
4420 event.requestId
4421 }
4422 );
4423
4424 const bodyText =
4425 result.base64Encoded
4426 ? Buffer.from(
4427 result.body || "",
4428 "base64"
4429 ).toString("utf8")
4430 : String(
4431 result.body || ""
4432 );
4433
4434 invTrace(
4435 `cdp_body_received id=${event.requestId} kind=${captured.kind} ` +
4436 `method=${captured.method} status=${captured.status} ` +
4437 `content_type=${captured.contentType} bytes=${bodyText.length}`
4438 );
4439
4440 const payload =
4441 JSON.parse(
4442 bodyText
4443 );
4444
4445 if (
4446 captured.kind ===
4447 "earnings"
4448 ) {
4449 earningsJson =
4450 payload;
4451
4452 publishCapturedApiIfReady();
4453
4454 console.log(
4455 "[INV] Captured exact-week 200 GET earnings API JSON via CDP Network body."
4456 );
4457 } else {
4458 if (acceptInstrumentPayload(
4459 payload,
4460 "CDP /instruments response"
4461 )) {
4462 console.log(
4463 "[INV] Captured validated 200 GET instrument metadata API JSON via CDP Network body."
4464 );
4465 }
4466 }
4467 } catch (error) {
4468 console.log(
4469 `[INV] CDP ${captured.kind} GET body capture warning ` +
4470 `(requestId=${event.requestId}): ${error.message}`
4471 );
4472 }
4473 })()
4474 );
4475 }
4476 );
4477
4478 console.log(
4479 "[INV] CDP Network response-body capture armed (GET-only; OPTIONS preflights ignored)."
4480 );
4481 } catch (error) {
4482 console.log(
4483 "[INV] Could not arm CDP Network body capture; Puppeteer response capture remains available:",
4484 error.message
4485 );
4486
4487 cdpSession = null;
4488 }
4489
4490 const relevantRequest = request => {
4491 try {
4492 const url = request.url();
4493 const lower = url.toLowerCase();
4494
4495 if (
4496 lower.includes(
4497 "endpoints.investing.com/earnings/v1/instruments/earnings"
4498 )
4499 ) {
4500 networkDiagnostics.push({
4501 method: request.method(),
4502 url: url.slice(0, 700)
4503 });
4504
4505 if (networkDiagnostics.length > 12) {
4506 networkDiagnostics.shift();
4507 }
4508 }
4509 } catch {}
4510 };
4511
4512 const relevantResponse = response => {
4513 try {
4514 const url = response.url();
4515 const lower = url.toLowerCase();
4516
4517 if (response.status() !== 200) {
4518 return;
4519 }
4520
4521 if (
4522 response.request().method() === "GET" &&
4523 lower.includes(
4524 "endpoints.investing.com/earnings/v1/instruments/earnings"
4525 )
4526 ) {
4527
4528
4529
4530
4531 if (expectedStartDate) {
4532 const parsedUrl =
4533 new URL(url);
4534
4535 if (
4536 parsedUrl.searchParams.get("start_date") !==
4537 `${expectedStartDate}T00:00:00.000Z` ||
4538 (
4539 expectedEndDate &&
4540 parsedUrl.searchParams.get("end_date") !==
4541 `${expectedEndDate}T00:00:00.000Z`
4542 )
4543 ) {
4544 return;
4545 }
4546 }
4547
4548 trackApiRead(
4549 (async () => {
4550 try {
4551 earningsJson =
4552 await response.json();
4553
4554 publishCapturedApiIfReady();
4555
4556 console.log(
4557 "[INV] Captured 200 Next Week earnings API JSON."
4558 );
4559 } catch (error) {
4560 console.log(
4561 "[INV] Earnings API body capture warning:",
4562 error.message
4563 );
4564 }
4565 })()
4566 );
4567
4568 return;
4569 }
4570
4571 if (
4572 lower.includes("endpoints.investing.com") &&
4573 lower.includes("/instruments") &&
4574 lower.includes("instrument_ids=") &&
4575 !lower.includes("/earnings") &&
4576 !lower.includes("key-metrics")
4577 ) {
4578 trackApiRead(
4579 (async () => {
4580 try {
4581 const payload =
4582 await response.json();
4583
4584 if (acceptInstrumentPayload(
4585 payload,
4586 "Puppeteer /instruments response"
4587 )) {
4588 console.log(
4589 "[INV] Captured validated 200 instrument metadata API JSON."
4590 );
4591 }
4592 } catch (error) {
4593 console.log(
4594 "[INV] Instrument API body capture warning:",
4595 error.message
4596 );
4597 }
4598 })()
4599 );
4600 }
4601 } catch (error) {
4602 console.log(
4603 "[INV] API response capture warning:",
4604 error.message
4605 );
4606 }
4607 };
4608
4609 page.on(
4610 "request",
4611 relevantRequest
4612 );
4613
4614 page.on(
4615 "response",
4616 relevantResponse
4617 );
4618
4619 let clickMethod = null;
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629 try {
4630 const domClickResult =
4631 await Promise.race([
4632 page.evaluate(
4633 marker => {
4634 const element =
4635 document.querySelector(
4636 `[${marker}="1"]`
4637 );
4638
4639 if (!element) {
4640 return {
4641 clicked: false,
4642 reason:
4643 "marked range control disappeared"
4644 };
4645 }
4646
4647 const disabled =
4648 Boolean(
4649 element.disabled ||
4650 element.getAttribute(
4651 "aria-disabled"
4652 ) === "true"
4653 );
4654
4655 if (disabled) {
4656 return {
4657 clicked: false,
4658 reason:
4659 "marked range control became disabled"
4660 };
4661 }
4662
4663 element.scrollIntoView({
4664 block: "center",
4665 inline: "center"
4666 });
4667
4668
4669
4670
4671
4672
4673
4674
4675 setTimeout(() => {
4676 try {
4677 element.click();
4678 } catch {}
4679 }, 0);
4680
4681 return {
4682 clicked: true,
4683 scheduled: true,
4684 tag:
4685 element.tagName,
4686 text:
4687 String(
4688 element.innerText ||
4689 element.textContent ||
4690 ""
4691 )
4692 .replace(/\s+/g, " ")
4693 .trim()
4694 .slice(0, 120)
4695 };
4696 },
4697 marker
4698 ),
4699 sleep(2500).then(() => {
4700 throw new Error(
4701 "DOM range click exceeded 2500ms"
4702 );
4703 })
4704 ]);
4705
4706 if (domClickResult?.clicked) {
4707 clickMethod =
4708 "DOM";
4709
4710 console.log(
4711 `[INV] DOM click sent to "${wantedText}": ${JSON.stringify(domClickResult)}`
4712 );
4713 } else {
4714 console.log(
4715 `[INV] DOM click for "${wantedText}" was not delivered: ${domClickResult?.reason || "unknown reason"}`
4716 );
4717 }
4718
4719 } catch (error) {
4720 console.log(
4721 `[INV] DOM click for "${wantedText}" failed fast: ${error.message}`
4722 );
4723 }
4724
4725
4726
4727
4728
4729
4730 if (!clickMethod) {
4731 try {
4732 await Promise.race([
4733 page.mouse.click(
4734 found.x,
4735 found.y,
4736 { delay: 80 }
4737 ),
4738 sleep(2500).then(() => {
4739 throw new Error(
4740 "native mouse click exceeded 2500ms"
4741 );
4742 })
4743 ]);
4744
4745 clickMethod =
4746 "NATIVE_MOUSE";
4747
4748 console.log(
4749 `[INV] Native trusted mouse click sent to "${wantedText}" at (${Math.round(found.x)}, ${Math.round(found.y)}).`
4750 );
4751
4752 } catch (error) {
4753
4754
4755
4756
4757
4758
4759 clickMethod =
4760 "ATTEMPTED_TIMEOUT";
4761
4762 console.log(
4763 `[INV] Both DOM and native click calls timed out for "${wantedText}": ${error.message}. ` +
4764 "Keeping Page A alive for the delayed first-party Week-2 request."
4765 );
4766 }
4767 }
4768
4769
4770
4771
4772
4773
4774 if (typeof onClickAttempted === "function") {
4775 try {
4776 onClickAttempted();
4777 } catch (error) {
4778 console.log(
4779 `[INV TICKER] Could not start concurrent fallback after Next Week click attempt: ${error.message}`
4780 );
4781 }
4782 }
4783
4784
4785
4786
4787
4788 const captureStarted =
4789 Date.now();
4790
4791 while (
4792 Date.now() - captureStarted < 12000 &&
4793 !(
4794 earningsJson &&
4795 instrumentsJson
4796 ) &&
4797 !checkEarlyComplete()
4798 ) {
4799 await sleep(250);
4800 }
4801
4802
4803
4804
4805
4806
4807 if (
4808 pendingApiReads.size > 0 &&
4809 !(
4810 earningsJson &&
4811 instrumentsJson
4812 ) &&
4813 !earlyCompleteValue
4814 ) {
4815 console.log(
4816 `[INV] Waiting for ${pendingApiReads.size} in-flight Next Week API body read(s).`
4817 );
4818
4819 await Promise.race([
4820 Promise.allSettled([...pendingApiReads]),
4821 sleep(3000)
4822 ]);
4823 }
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833 if (
4834 !(
4835 earningsJson &&
4836 instrumentsJson
4837 ) &&
4838 !earlyCompleteValue
4839 ) {
4840 console.log(
4841 `[INV] Extending Next Week CDP capture by up to ${NEXT_WEEK_CDP_EXTRA_WAIT_MS}ms for delayed browser API responses.`
4842 );
4843
4844 const extendedCaptureStarted =
4845 Date.now();
4846
4847 while (
4848 Date.now() - extendedCaptureStarted <
4849 NEXT_WEEK_CDP_EXTRA_WAIT_MS &&
4850 !(
4851 earningsJson &&
4852 instrumentsJson
4853 ) &&
4854 !checkEarlyComplete()
4855 ) {
4856 await sleep(250);
4857 }
4858
4859 invTrace(
4860 `cdp_extended_wait_done elapsed_ms=${Date.now() - extendedCaptureStarted} ` +
4861 `earnings_json=${earningsJson ? "YES" : "NO"} ` +
4862 `instruments_json=${instrumentsJson ? "YES" : "NO"}`
4863 );
4864 }
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876 const hasRelevantCdpWorkInFlight = () => {
4877 if (earningsJson && instrumentsJson) {
4878 return false;
4879 }
4880
4881 const missingKind =
4882 !earningsJson
4883 ? "earnings"
4884 : !instrumentsJson
4885 ? "instruments"
4886 : null;
4887
4888 if (!missingKind) {
4889 return false;
4890 }
4891
4892 const requestStillActive =
4893 [...cdpRequests.values()].some(
4894 item =>
4895 item?.method === "GET" &&
4896 item?.kind === missingKind
4897 );
4898
4899 const responseStillActive =
4900 [...cdpResponses.values()].some(
4901 item =>
4902 item?.method === "GET" &&
4903 item?.kind === missingKind
4904 );
4905
4906 return (
4907 pendingApiReads.size > 0 ||
4908 requestStillActive ||
4909 responseStillActive
4910 );
4911 };
4912
4913 if (
4914 !(earningsJson && instrumentsJson) &&
4915 !earlyCompleteValue &&
4916 hasRelevantCdpWorkInFlight()
4917 ) {
4918
4919
4920
4921 const drainStarted = Date.now();
4922 const FINAL_OBSERVED_BODY_DRAIN_MS = 5000;
4923
4924 console.log(
4925 `[INV] Exact-week API traffic is still in flight after the capture ceiling; final observed-work drain capped at ${FINAL_OBSERVED_BODY_DRAIN_MS}ms.`
4926 );
4927
4928 while (
4929 Date.now() - drainStarted < FINAL_OBSERVED_BODY_DRAIN_MS &&
4930 !(earningsJson && instrumentsJson) &&
4931 hasRelevantCdpWorkInFlight()
4932 ) {
4933 await sleep(250);
4934 }
4935
4936 invTrace(
4937 `cdp_inflight_drain_done elapsed_ms=${Date.now() - drainStarted} ` +
4938 `pending_reads=${pendingApiReads.size} ` +
4939 `earnings_json=${earningsJson ? "YES" : "NO"} ` +
4940 `instruments_json=${instrumentsJson ? "YES" : "NO"}`
4941 );
4942 }
4943
4944
4945
4946
4947
4948 const graceStarted =
4949 Date.now();
4950
4951 while (
4952 Date.now() - graceStarted < 1500 &&
4953 !(
4954 earningsJson &&
4955 instrumentsJson
4956 ) &&
4957 !earlyCompleteValue
4958 ) {
4959 await sleep(100);
4960 }
4961
4962 if (
4963 pendingApiReads.size > 0 &&
4964 !(
4965 earningsJson &&
4966 instrumentsJson
4967 ) &&
4968 !earlyCompleteValue
4969 ) {
4970 await Promise.race([
4971 Promise.allSettled([...pendingApiReads]),
4972 sleep(1500)
4973 ]);
4974 }
4975
4976 page.off(
4977 "request",
4978 relevantRequest
4979 );
4980
4981 page.off(
4982 "response",
4983 relevantResponse
4984 );
4985
4986 if (cdpSession) {
4987
4988
4989
4990 try {
4991 await Promise.race([
4992 cdpSession.send("Network.disable"),
4993 sleep(1500).then(() => { throw new Error("Network.disable cleanup exceeded 1500ms"); })
4994 ]);
4995 } catch (error) {
4996 console.log(`[INV] CDP Network.disable cleanup skipped/timeout: ${error.message}`);
4997 }
4998
4999 try {
5000 await Promise.race([
5001 cdpSession.detach(),
5002 sleep(1500).then(() => { throw new Error("CDP detach cleanup exceeded 1500ms"); })
5003 ]);
5004 } catch (error) {
5005 console.log(`[INV] CDP detach cleanup skipped/timeout: ${error.message}`);
5006 }
5007 }
5008
5009 console.log(
5010 `[INV] Range-click network diagnostics for "${wantedText}": ${JSON.stringify(networkDiagnostics)}`
5011 );
5012
5013 console.log(
5014 `[INV] Next Week API capture: earnings=${earningsJson ? "YES" : "NO"} instruments=${instrumentsJson ? "YES" : "NO"}.`
5015 );
5016
5017 if (expectedStartDate) {
5018 console.log(
5019 `[INV] "${wantedText}" click delivered; expected target week begins ${expectedStartDate}. API verification follows.`
5020 );
5021 }
5022
5023 return {
5024 clicked: true,
5025 clickMethod,
5026 earningsJson,
5027 instrumentsJson,
5028 passiveWeekMap: earlyCompleteValue instanceof Map ? earlyCompleteValue : null
5029 };
5030
5031 } catch (error) {
5032 console.log(
5033 `[INV] Could not click range button "${wantedText}":`,
5034 error.message
5035 );
5036
5037 return false;
5038 }
5039
5040 }
5041
5042 function mergeCalendarMapsInRange({
5043 target,
5044 source,
5045 rangeStart,
5046 rangeEnd
5047 }) {
5048 let added = 0;
5049
5050 for (
5051 const [
5052 symbol,
5053 date
5054 ]
5055 of source
5056 ) {
5057 if (
5058 date >= rangeStart &&
5059 date <= rangeEnd
5060 ) {
5061 if (
5062 !target.has(
5063 symbol
5064 ) ||
5065 target.get(
5066 symbol
5067 ) !== date
5068 ) {
5069 target.set(
5070 symbol,
5071 date
5072 );
5073
5074 added++;
5075 }
5076 }
5077 }
5078
5079 return added;
5080
5081 }
5082
5083 function calendarCoverage(
5084 calendarMap
5085 ) {
5086 const dates =
5087 [
5088 ...new Set(
5089 [
5090 ...calendarMap.values()
5091 ]
5092 )
5093 ]
5094 .filter(Boolean)
5095 .sort();
5096
5097 return {
5098 dates,
5099 firstDate:
5100 dates[0] ||
5101 null,
5102 lastDate:
5103 dates[
5104 dates.length - 1
5105 ] ||
5106 null
5107 };
5108
5109 }
5110
5111 async function waitForNextWeekRows(
5112 page,
5113 previousCoverageEnd
5114 ) {
5115
5116
5117
5118
5119
5120
5121
5122
5123 console.log(
5124 `[INV] Next Week clicked. Settling ${NEXT_WEEK_SETTLE_MS / 1000} seconds without CDP polling; previous coverage ended ${previousCoverageEnd || "UNKNOWN"}.`
5125 );
5126
5127 await sleep(
5128 NEXT_WEEK_SETTLE_MS
5129 );
5130
5131 return true;
5132 }
5133
5134 function parseInvestingApiResponses(
5135 earningsPayload,
5136 instrumentsPayload,
5137 rangeStart,
5138 rangeEnd
5139 ) {
5140 const map =
5141 new Map();
5142
5143 const instrumentSymbols =
5144 new Map();
5145
5146 const seenInstrumentObjects =
5147 new Set();
5148
5149 function firstValue(object, keys) {
5150 for (const key of keys) {
5151 const value =
5152 object?.[key];
5153
5154 if (
5155 value !== undefined &&
5156 value !== null &&
5157 value !== ""
5158 ) {
5159 return value;
5160 }
5161 }
5162
5163 return null;
5164 }
5165
5166 function instrumentIdFrom(object) {
5167 return firstValue(
5168 object,
5169 [
5170 "instrument_id",
5171 "instrumentId",
5172 "id"
5173 ]
5174 );
5175 }
5176
5177 function symbolFrom(object) {
5178 const candidates = [
5179 firstValue(
5180 object,
5181 [
5182 "display_symbol",
5183 "symbol",
5184 "ticker",
5185 "stock_symbol",
5186 "instrument_symbol"
5187 ]
5188 ),
5189 firstValue(
5190 object?.instrument,
5191 [
5192 "symbol",
5193 "ticker",
5194 "stock_symbol"
5195 ]
5196 ),
5197 firstValue(
5198 object?.security,
5199 [
5200 "symbol",
5201 "ticker"
5202 ]
5203 ),
5204 firstValue(
5205 object?.company,
5206 [
5207 "symbol",
5208 "ticker"
5209 ]
5210 )
5211 ];
5212
5213 for (const value of candidates) {
5214 const symbol =
5215 normalizeTicker(
5216 value
5217 );
5218
5219 if (
5220 looksLikeTicker(
5221 symbol
5222 )
5223 ) {
5224 return symbol;
5225 }
5226 }
5227
5228 return "";
5229 }
5230
5231 function collectInstrumentSymbols(value) {
5232 if (
5233 !value ||
5234 typeof value !== "object"
5235 ) {
5236 return;
5237 }
5238
5239 if (
5240 seenInstrumentObjects.has(
5241 value
5242 )
5243 ) {
5244 return;
5245 }
5246
5247 seenInstrumentObjects.add(
5248 value
5249 );
5250
5251 if (!Array.isArray(value)) {
5252 const id =
5253 instrumentIdFrom(
5254 value
5255 );
5256
5257 const symbol =
5258 symbolFrom(
5259 value
5260 );
5261
5262 if (
5263 id !== null &&
5264 id !== undefined &&
5265 symbol
5266 ) {
5267 instrumentSymbols.set(
5268 String(id),
5269 symbol
5270 );
5271 }
5272 }
5273
5274 for (
5275 const child
5276 of Object.values(value)
5277 ) {
5278 if (
5279 child &&
5280 typeof child === "object"
5281 ) {
5282 collectInstrumentSymbols(
5283 child
5284 );
5285 }
5286 }
5287 }
5288
5289 collectInstrumentSymbols(
5290 instrumentsPayload
5291 );
5292
5293 console.log(
5294 `[INV] Instrument API mapping produced ${instrumentSymbols.size} instrument-id/ticker pairs.`
5295 );
5296
5297 const seenEarningsObjects =
5298 new Set();
5299
5300 function visitEarnings(value) {
5301 if (
5302 !value ||
5303 typeof value !== "object"
5304 ) {
5305 return;
5306 }
5307
5308 if (
5309 seenEarningsObjects.has(
5310 value
5311 )
5312 ) {
5313 return;
5314 }
5315
5316 seenEarningsObjects.add(
5317 value
5318 );
5319
5320 if (!Array.isArray(value)) {
5321 const date =
5322 normalizeDate(
5323 firstValue(
5324 value,
5325 [
5326 "date",
5327 "earnings_date",
5328 "earningsDate",
5329 "report_date",
5330 "reportDate",
5331 "release_date",
5332 "releaseDate",
5333 "event_date",
5334 "eventDate"
5335 ]
5336 )
5337 );
5338
5339 const id =
5340 firstValue(
5341 value,
5342 [
5343 "instrument_id",
5344 "instrumentId"
5345 ]
5346 );
5347
5348 const directSymbol =
5349 symbolFrom(
5350 value
5351 );
5352
5353 const mappedSymbol =
5354 id !== null &&
5355 id !== undefined
5356 ? instrumentSymbols.get(
5357 String(id)
5358 ) || ""
5359 : "";
5360
5361 const symbol =
5362 directSymbol ||
5363 mappedSymbol;
5364
5365 if (
5366 symbol &&
5367 date &&
5368 date >= rangeStart &&
5369 date <= rangeEnd
5370 ) {
5371 map.set(
5372 symbol,
5373 date
5374 );
5375 }
5376 }
5377
5378 for (
5379 const child
5380 of Object.values(value)
5381 ) {
5382 if (
5383 child &&
5384 typeof child === "object"
5385 ) {
5386 visitEarnings(
5387 child
5388 );
5389 }
5390 }
5391 }
5392
5393 visitEarnings(
5394 earningsPayload
5395 );
5396
5397 console.log(
5398 `[INV] Earnings API parsing produced ${map.size} verified ticker/date pairs.`
5399 );
5400
5401 return map;
5402 }
5403
5404 async function fetchObservedInvestingInstrumentMetadata(
5405 page
5406 ) {
5407 const authorization =
5408 page.__investingBearerToken ||
5409 null;
5410
5411 const templateUrl =
5412 page.__investingInstrumentTemplate ||
5413 null;
5414
5415 if (
5416 !authorization ||
5417 !templateUrl
5418 ) {
5419 console.log(
5420 `[INV API] Observed instrument metadata replay unavailable: bearer=${authorization ? "YES" : "NO"} template=${templateUrl ? "YES" : "NO"}.`
5421 );
5422
5423 return null;
5424 }
5425
5426 try {
5427 const result =
5428 await Promise.race([
5429 page.evaluate(
5430 async ({ url, authorization }) => {
5431 const response =
5432 await fetch(
5433 url,
5434 {
5435 method: "GET",
5436 headers: {
5437 accept: "*/*",
5438 authorization
5439 },
5440 credentials: "include"
5441 }
5442 );
5443
5444 const text =
5445 await response.text();
5446
5447 return {
5448 ok: response.ok,
5449 status: response.status,
5450 contentType:
5451 response.headers.get("content-type") || "",
5452 text
5453 };
5454 },
5455 {
5456 url: templateUrl,
5457 authorization
5458 }
5459 ),
5460 sleep(10000).then(() => {
5461 throw new Error(
5462 "browser-context instrument metadata fetch exceeded 10000ms"
5463 );
5464 })
5465 ]);
5466
5467 console.log(
5468 `[INV API] Observed instrument metadata response: status=${result?.status ?? "NONE"} ` +
5469 `contentType=${result?.contentType || "UNKNOWN"} bytes=${result?.text?.length || 0}.`
5470 );
5471
5472 if (
5473 !result?.ok ||
5474 !result.text
5475 ) {
5476 return null;
5477 }
5478
5479 return JSON.parse(
5480 result.text
5481 );
5482 } catch (error) {
5483 console.log(
5484 `[INV API] Observed instrument metadata replay failed safely: ${error.message}`
5485 );
5486
5487 return null;
5488 }
5489 }
5490
5491
5492 async function fetchInvestingWeekViaApi(
5493 page,
5494 weekStart,
5495 weekEnd,
5496 rangeStart,
5497 rangeEnd
5498 ) {
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512 const authorization =
5513 page.__investingBearerToken ||
5514 null;
5515
5516 if (!authorization) {
5517 invTrace(
5518 `week_api_browser_skip requested=${weekStart}..${weekEnd} reason=NO_BEARER`
5519 );
5520
5521 console.log(
5522 `[INV API] Browser-context week fetch unavailable for ${weekStart} through ${weekEnd}: Bearer not captured yet.`
5523 );
5524
5525 return new Map();
5526 }
5527
5528 const startIso =
5529 `${weekStart}T00:00:00.000Z`;
5530
5531
5532
5533
5534
5535
5536 const endIso =
5537 `${weekEnd}T00:00:00.000Z`;
5538
5539 const endpoint =
5540 "https://endpoints.investing.com/earnings/v1/instruments/earnings" +
5541 `?start_date=${encodeURIComponent(startIso)}` +
5542 `&end_date=${encodeURIComponent(endIso)}` +
5543 "&country_ids=5&limit=200&deduplicate=true";
5544
5545 invTrace(
5546 `week_api_browser_start requested=${weekStart}..${weekEnd} bearer=YES`
5547 );
5548
5549 let result = null;
5550
5551 try {
5552 result =
5553 await Promise.race([
5554 page.evaluate(
5555 async ({ url, authorization }) => {
5556 const response =
5557 await fetch(
5558 url,
5559 {
5560 method: "GET",
5561 headers: {
5562 accept: "*/*",
5563 authorization
5564 },
5565 credentials: "include"
5566 }
5567 );
5568
5569 const text =
5570 await response.text();
5571
5572 return {
5573 ok: response.ok,
5574 status: response.status,
5575 contentType:
5576 response.headers.get("content-type") || "",
5577 text
5578 };
5579 },
5580 {
5581 url: endpoint,
5582 authorization
5583 }
5584 ),
5585 sleep(5000).then(() => {
5586 throw new Error(
5587 "browser-context earnings API fetch exceeded 5000ms"
5588 );
5589 })
5590 ]);
5591 } catch (error) {
5592 console.log(
5593 `[INV API] Browser-context week fetch failed for ${weekStart} through ${weekEnd}: ${error.message}`
5594 );
5595
5596 invTrace(
5597 `week_api_browser_error requested=${weekStart}..${weekEnd} error=${JSON.stringify(error.message)}`
5598 );
5599
5600 return new Map();
5601 }
5602
5603 console.log(
5604 `[INV API] Browser-context earnings response: status=${result?.status ?? "NONE"} ` +
5605 `contentType=${result?.contentType || "UNKNOWN"} bytes=${result?.text?.length || 0} ` +
5606 `range=${weekStart}..${weekEnd}.`
5607 );
5608
5609 if (
5610 !result?.ok ||
5611 !result.text
5612 ) {
5613 return new Map();
5614 }
5615
5616 let earningsPayload = null;
5617
5618 try {
5619 earningsPayload =
5620 JSON.parse(
5621 result.text
5622 );
5623 } catch (error) {
5624 console.log(
5625 `[INV API] Browser-context earnings JSON parse failed: ${error.message}`
5626 );
5627
5628 return new Map();
5629 }
5630
5631 const instrumentsPayload =
5632 await fetchObservedInvestingInstrumentMetadata(
5633 page
5634 );
5635
5636 const parsed =
5637 parseInvestingApiResponses(
5638 earningsPayload,
5639 instrumentsPayload,
5640 rangeStart,
5641 rangeEnd
5642 );
5643
5644 invTrace(
5645 `week_api_browser_done requested=${weekStart}..${weekEnd} rows=${parsed.size}`
5646 );
5647
5648 console.log(
5649 `[INV API] Browser-context exact-week fetch parsed ${parsed.size} ticker/date pair(s).`
5650 );
5651
5652 return parsed;
5653
5654 }
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673 async function resolveInvestingUsEquityUrl(
5674 page,
5675 ticker,
5676 signal
5677 ) {
5678 throwIfAborted(signal);
5679
5680 const normalizedTicker =
5681 normalizeTicker(ticker);
5682
5683 if (!normalizedTicker) {
5684 return null;
5685 }
5686
5687 const searchUrl =
5688 `https://www.investing.com/search?q=${encodeURIComponent(normalizedTicker.toLowerCase())}`;
5689
5690 console.log(
5691 `[INV TICKER:${normalizedTicker}] Searching Investing for exact U.S. equity result.`
5692 );
5693
5694 try {
5695 await page.goto(
5696 searchUrl,
5697 {
5698 waitUntil:
5699 "domcontentloaded",
5700 timeout:
5701 TICKER_SEARCH_NAV_TIMEOUT_MS
5702 }
5703 );
5704 } catch (error) {
5705 console.log(
5706 `[INV TICKER:${normalizedTicker}] Search navigation warning: ${error.message}`
5707 );
5708 }
5709
5710 throwIfAborted(signal);
5711
5712 await sleep(
5713 TICKER_PAGE_SETTLE_MS
5714 );
5715
5716 let result = null;
5717
5718 try {
5719 result =
5720 await Promise.race([
5721 page.evaluate(
5722 wantedTicker => {
5723 const clean = value =>
5724 String(value || "")
5725 .replace(/\s+/g, " ")
5726 .trim();
5727
5728 const upperTicker =
5729 wantedTicker.toUpperCase();
5730
5731 const links =
5732 Array.from(
5733 document.querySelectorAll(
5734 'a[href*="/equities/"]'
5735 )
5736 );
5737
5738 const candidates = [];
5739
5740 for (
5741 const link
5742 of links
5743 ) {
5744 const href =
5745 link.href || "";
5746
5747 if (!href) {
5748 continue;
5749 }
5750
5751 let container =
5752 link;
5753
5754 for (
5755 let depth = 0;
5756 container &&
5757 depth < 6;
5758 depth++,
5759 container =
5760 container.parentElement
5761 ) {
5762 const text =
5763 clean(
5764 container.innerText ||
5765 container.textContent ||
5766 ""
5767 );
5768
5769 if (
5770 !text ||
5771 text.length > 1200
5772 ) {
5773 continue;
5774 }
5775
5776
5777
5778
5779
5780
5781 const tokens =
5782 text
5783 .toUpperCase()
5784 .split(/[^A-Z0-9.\-^=]+/)
5785 .filter(Boolean);
5786
5787 if (
5788 !tokens.includes(
5789 upperTicker
5790 )
5791 ) {
5792 continue;
5793 }
5794
5795 const lower =
5796 text.toLowerCase();
5797
5798 const isEquity =
5799 /\bstock\b/.test(lower) ||
5800 lower.includes("equities");
5801
5802 const isNonStock =
5803 /\betf\b/.test(lower) ||
5804 lower.includes("exchange traded fund") ||
5805 lower.includes("fund ");
5806
5807
5808
5809
5810
5811
5812 const isUsExchange =
5813 /\bnyse\b/i.test(text) ||
5814 /\bnasdaq\b/i.test(text) ||
5815 /\bnyse american\b/i.test(text) ||
5816 /\bamex\b/i.test(text);
5817
5818 const isForeignExchange =
5819 /\bnse\b/i.test(text) ||
5820 /\bbse\b/i.test(text) ||
5821 /\bxetra\b/i.test(text) ||
5822 /\btsx\b/i.test(text) ||
5823 /\btsxv\b/i.test(text) ||
5824 /\bfrankfurt\b/i.test(text) ||
5825 /\btradegate\b/i.test(text) ||
5826 /\bswitzerland\b/i.test(text);
5827
5828 let score = 0;
5829
5830 if (isEquity) {
5831 score += 100;
5832 }
5833
5834 if (isUsExchange) {
5835 score += 200;
5836 }
5837
5838 if (isForeignExchange) {
5839 score -= 200;
5840 }
5841
5842 if (isNonStock) {
5843 score -= 500;
5844 }
5845
5846
5847
5848
5849
5850 score +=
5851 Math.max(
5852 0,
5853 100 -
5854 Math.min(
5855 100,
5856 Math.floor(
5857 text.length / 5
5858 )
5859 )
5860 );
5861
5862 candidates.push({
5863 href,
5864 text:
5865 text.slice(
5866 0,
5867 500
5868 ),
5869 score,
5870 isEquity,
5871 isUsExchange,
5872 isNonStock
5873 });
5874
5875 break;
5876 }
5877 }
5878
5879 candidates.sort(
5880 (a, b) =>
5881 b.score - a.score
5882 );
5883
5884 const best =
5885 candidates.find(
5886 item =>
5887 item.isEquity &&
5888 item.isUsExchange &&
5889 !item.isNonStock &&
5890 item.score > 0
5891 );
5892
5893 return {
5894 best:
5895 best || null,
5896 candidates:
5897 candidates
5898 .slice(0, 5)
5899 .map(
5900 item => ({
5901 href:
5902 item.href,
5903 text:
5904 item.text,
5905 score:
5906 item.score,
5907 isEquity:
5908 item.isEquity,
5909 isUsExchange:
5910 item.isUsExchange,
5911 isNonStock:
5912 item.isNonStock
5913 })
5914 )
5915 };
5916 },
5917 normalizedTicker
5918 ),
5919 sleep(5000).then(
5920 () => {
5921 throw new Error(
5922 "ticker search result parsing exceeded 5000ms"
5923 );
5924 }
5925 )
5926 ]);
5927 } catch (error) {
5928 console.log(
5929 `[INV TICKER:${normalizedTicker}] Search result parsing failed: ${error.message}`
5930 );
5931
5932 return null;
5933 }
5934
5935 const best =
5936 result?.best || null;
5937
5938 console.log(
5939 `[INV TICKER:${normalizedTicker}] Search candidates: ${JSON.stringify(result?.candidates || [])}`
5940 );
5941
5942 if (
5943 !best ||
5944 !best.href
5945 ) {
5946 console.log(
5947 `[INV TICKER:${normalizedTicker}] No exact U.S. stock/equity search result found.`
5948 );
5949
5950 return null;
5951 }
5952
5953 console.log(
5954 `[INV TICKER:${normalizedTicker}] Resolved U.S. equity page: ${best.href}`
5955 );
5956
5957 return best.href;
5958
5959 }
5960
5961
5962 async function extractNextEarningsDateFromEquityPage(
5963 page,
5964 ticker,
5965 equityUrl,
5966 rangeStart,
5967 rangeEnd,
5968 signal
5969 ) {
5970 throwIfAborted(signal);
5971
5972 const normalizedTicker =
5973 normalizeTicker(ticker);
5974
5975 console.log(
5976 `[INV TICKER:${normalizedTicker}] Opening resolved equity page for Next Earnings Date.`
5977 );
5978
5979 try {
5980 await page.goto(
5981 equityUrl,
5982 {
5983 waitUntil:
5984 "domcontentloaded",
5985 timeout:
5986 TICKER_EQUITY_NAV_TIMEOUT_MS
5987 }
5988 );
5989 } catch (error) {
5990 console.log(
5991 `[INV TICKER:${normalizedTicker}] Equity navigation warning: ${error.message}`
5992 );
5993 }
5994
5995 throwIfAborted(signal);
5996
5997 await sleep(
5998 TICKER_PAGE_SETTLE_MS
5999 );
6000
6001 let extracted = null;
6002
6003 try {
6004 extracted =
6005 await Promise.race([
6006 page.evaluate(() => {
6007 const clean = value =>
6008 String(value || "")
6009 .replace(/\s+/g, " ")
6010 .trim();
6011
6012 const nodes =
6013 Array.from(
6014 document.querySelectorAll(
6015 "dt,div,span,p,td,th"
6016 )
6017 );
6018
6019 for (
6020 const label
6021 of nodes
6022 ) {
6023 const labelText =
6024 clean(
6025 label.innerText ||
6026 label.textContent ||
6027 ""
6028 );
6029
6030 if (
6031 labelText.toLowerCase() !==
6032 "next earnings date"
6033 ) {
6034 continue;
6035 }
6036
6037 const candidates = [];
6038
6039 if (
6040 label.nextElementSibling
6041 ) {
6042 candidates.push(
6043 label.nextElementSibling
6044 );
6045 }
6046
6047 if (
6048 label.parentElement
6049 ) {
6050 candidates.push(
6051 ...Array.from(
6052 label.parentElement.children
6053 ).filter(
6054 child =>
6055 child !== label
6056 )
6057 );
6058 }
6059
6060 for (
6061 const candidate
6062 of candidates
6063 ) {
6064 const link =
6065 candidate.matches?.("a")
6066 ? candidate
6067 : candidate.querySelector?.(
6068 'a[data-test="link-key-info"],a[href*="-earnings"],a'
6069 );
6070
6071 const text =
6072 clean(
6073 link?.innerText ||
6074 candidate.innerText ||
6075 candidate.textContent ||
6076 ""
6077 );
6078
6079 if (text) {
6080 return {
6081 label:
6082 labelText,
6083 value:
6084 text,
6085 href:
6086 link?.href ||
6087 null,
6088 method:
6089 "LABEL_SIBLING"
6090 };
6091 }
6092 }
6093 }
6094
6095
6096
6097
6098
6099
6100
6101 const keyInfoLinks =
6102 Array.from(
6103 document.querySelectorAll(
6104 'a[data-test="link-key-info"],a[href*="-earnings"]'
6105 )
6106 );
6107
6108 for (
6109 const link
6110 of keyInfoLinks
6111 ) {
6112 let container =
6113 link.parentElement;
6114
6115 for (
6116 let depth = 0;
6117 container &&
6118 depth < 4;
6119 depth++,
6120 container =
6121 container.parentElement
6122 ) {
6123 const text =
6124 clean(
6125 container.innerText ||
6126 container.textContent ||
6127 ""
6128 );
6129
6130 if (
6131 text
6132 .toLowerCase()
6133 .includes(
6134 "next earnings date"
6135 )
6136 ) {
6137 return {
6138 label:
6139 "Next Earnings Date",
6140 value:
6141 clean(
6142 link.innerText ||
6143 link.textContent ||
6144 ""
6145 ),
6146 href:
6147 link.href ||
6148 null,
6149 method:
6150 "KEY_INFO_LINK"
6151 };
6152 }
6153 }
6154 }
6155
6156 return null;
6157 }),
6158 sleep(5000).then(
6159 () => {
6160 throw new Error(
6161 "Next Earnings Date extraction exceeded 5000ms"
6162 );
6163 }
6164 )
6165 ]);
6166 } catch (error) {
6167 console.log(
6168 `[INV TICKER:${normalizedTicker}] Next Earnings Date extraction failed: ${error.message}`
6169 );
6170
6171 return {
6172 date: null,
6173 conclusiveNoDate: false,
6174 reason: "EXTRACTION_FAILED"
6175 };
6176 }
6177
6178 if (
6179 !extracted ||
6180 !extracted.value
6181 ) {
6182 let verification = null;
6183
6184 try {
6185 verification =
6186 await Promise.race([
6187 page.evaluate(
6188 expectedUrl => {
6189 const bodyText =
6190 String(
6191 document.body?.innerText ||
6192 ""
6193 );
6194
6195 const currentUrl =
6196 String(
6197 window.location.href ||
6198 ""
6199 );
6200
6201 let expectedPath = "";
6202 let currentPath = "";
6203
6204 try {
6205 expectedPath =
6206 new URL(expectedUrl).pathname
6207 .replace(/\/+$/, "")
6208 .toLowerCase();
6209
6210 currentPath =
6211 new URL(currentUrl).pathname
6212 .replace(/\/+$/, "")
6213 .toLowerCase();
6214 } catch {}
6215
6216 const correctEquityPage =
6217 Boolean(
6218 expectedPath &&
6219 currentPath ===
6220 expectedPath
6221 );
6222
6223 const usableBody =
6224 bodyText.length >= 1000;
6225
6226 const blocked =
6227 /captcha|cloudflare|verify you are human|access denied|just a moment|unusual traffic|pardon our interruption/i.test(
6228 bodyText
6229 );
6230
6231
6232
6233
6234
6235
6236 const earningsEvidence =
6237 /earnings/i.test(
6238 bodyText
6239 ) &&
6240 (
6241 /earnings\s+(?:date|calendar|report|release|estimates?)/i.test(
6242 bodyText
6243 ) ||
6244 /next\s+earnings/i.test(
6245 bodyText
6246 ) ||
6247 /eps/i.test(
6248 bodyText
6249 )
6250 );
6251
6252 return {
6253 currentUrl,
6254 bodyLength:
6255 bodyText.length,
6256 correctEquityPage,
6257 usableBody,
6258 blocked,
6259 earningsEvidence
6260 };
6261 },
6262 equityUrl
6263 ),
6264 sleep(3000).then(
6265 () => {
6266 throw new Error(
6267 "equity-page negative verification exceeded 3000ms"
6268 );
6269 }
6270 )
6271 ]);
6272
6273 } catch (error) {
6274 console.log(
6275 `[INV TICKER:${normalizedTicker}] NO_DATE verification failed safely: ${error.message}`
6276 );
6277 }
6278
6279 const conclusiveNoDate =
6280 Boolean(
6281 verification?.correctEquityPage &&
6282 verification?.usableBody &&
6283 !verification?.blocked &&
6284 verification?.earningsEvidence
6285 );
6286
6287 if (conclusiveNoDate) {
6288 console.log(
6289 `[INV TICKER:${normalizedTicker}] CONFIRMED_NO_DATE — verified resolved equity page has earnings evidence but no Next Earnings Date field.`
6290 );
6291 } else {
6292 console.log(
6293 `[INV TICKER:${normalizedTicker}] Next Earnings Date field not found, but page was not strong enough to prove NO_DATE.`
6294 );
6295 }
6296
6297 return {
6298 date: null,
6299 conclusiveNoDate,
6300 reason:
6301 conclusiveNoDate
6302 ? "VERIFIED_EQUITY_PAGE_NO_NEXT_EARNINGS_DATE"
6303 : "UNVERIFIED_NO_DATE"
6304 };
6305 }
6306
6307 const normalizedDate =
6308 normalizeInvestingDate(
6309 extracted.value
6310 );
6311
6312 console.log(
6313 `[INV TICKER:${normalizedTicker}] Next Earnings Date raw=${JSON.stringify(extracted.value)} ` +
6314 `normalized=${normalizedDate || "NONE"} method=${extracted.method || "UNKNOWN"}.`
6315 );
6316
6317 if (!normalizedDate) {
6318 return {
6319 date: null,
6320 conclusiveNoDate: false,
6321 reason: "UNPARSEABLE_NEXT_EARNINGS_DATE"
6322 };
6323 }
6324
6325 if (
6326 normalizedDate < rangeStart
6327 ) {
6328 console.log(
6329 `[INV TICKER:${normalizedTicker}] Verified next earnings date ${normalizedDate} is before ` +
6330 `today (${rangeStart}); rejecting stale/past date.`
6331 );
6332
6333 return {
6334 date: null,
6335 conclusiveNoDate: true,
6336 reason: "VERIFIED_NEXT_EARNINGS_DATE_IS_PAST"
6337 };
6338 }
6339
6340 console.log(
6341 `✅ [INV TICKER:${normalizedTicker}] Verified fallback earnings date: ${normalizedDate}`
6342 );
6343
6344 return {
6345 date: normalizedDate,
6346 conclusiveNoDate: false,
6347 reason: "DATE_FOUND"
6348 };
6349
6350 }
6351
6352
6353 async function fillMissingRequestedTickersFromEquityPages({
6354 page,
6355 requestedTickers,
6356 calendarMap,
6357 rangeStart,
6358 rangeEnd,
6359 signal,
6360 negativeCache = {},
6361 positiveTickerCache = new Map()
6362 }) {
6363 for (const ticker of requestedTickers) {
6364 const cachedPositiveDate =
6365 positiveTickerCache.get(normalizeTicker(ticker));
6366
6367 if (cachedPositiveDate && !calendarMap.has(normalizeTicker(ticker))) {
6368 calendarMap.set(normalizeTicker(ticker), cachedPositiveDate);
6369 console.log(
6370 `[INV POS CACHE:${normalizeTicker(ticker)}] Fresh successful direct result reused; skipping targeted fallback.`
6371 );
6372 }
6373 }
6374
6375 const missing =
6376 requestedTickers
6377 .map(
6378 normalizeTicker
6379 )
6380 .filter(
6381 ticker =>
6382 ticker &&
6383 !calendarMap.has(
6384 ticker
6385 )
6386 );
6387
6388 if (!missing.length) {
6389 console.log(
6390 "[INV TICKER] Calendar resolved every requested ticker; targeted fallback not needed."
6391 );
6392
6393 return {
6394 attempted: 0,
6395 added: 0
6396 };
6397 }
6398
6399 console.log(
6400 `[INV TICKER] Starting targeted fallback for ${missing.length} calendar miss(es): ${missing.join(", ")}`
6401 );
6402
6403 let attempted = 0;
6404 let added = 0;
6405 const fallbackStartedAt = Date.now();
6406
6407 for (
6408 const ticker
6409 of missing
6410 ) {
6411 const fallbackElapsedMs = Date.now() - fallbackStartedAt;
6412
6413 if (fallbackElapsedMs >= TICKER_FALLBACK_TOTAL_BUDGET_MS) {
6414 console.log(
6415 `[INV TICKER] Targeted fallback total budget reached after ${fallbackElapsedMs}ms ` +
6416 `(budget=${TICKER_FALLBACK_TOTAL_BUDGET_MS}ms); stopping before ${ticker}.`
6417 );
6418 break;
6419 }
6420
6421 if (isAborted(signal)) {
6422 console.log(
6423 "[INV TICKER] Production deadline reached; stopping targeted fallback."
6424 );
6425 break;
6426 }
6427
6428
6429
6430
6431
6432
6433 if (calendarMap.has(ticker)) {
6434 console.log(
6435 `[INV TICKER:${ticker}] Skipping fallback; calendar resolved ticker while it was queued.`
6436 );
6437
6438 if (negativeCache[ticker]) {
6439 await clearInvestingNegative(ticker);
6440 delete negativeCache[ticker];
6441 }
6442
6443 continue;
6444 }
6445
6446 if (
6447 isFreshInvestingNegative(
6448 negativeCache,
6449 ticker
6450 )
6451 ) {
6452 console.log(
6453 `[INV NEG CACHE:${ticker}] Fresh confirmed NO_DATE; skipping targeted ticker fallback (48h TTL).`
6454 );
6455 continue;
6456 }
6457
6458 attempted++;
6459
6460 try {
6461 const equityUrl =
6462 await resolveInvestingUsEquityUrl(
6463 page,
6464 ticker,
6465 signal
6466 );
6467
6468 if (!equityUrl) {
6469 continue;
6470 }
6471
6472 const directResult =
6473 await extractNextEarningsDateFromEquityPage(
6474 page,
6475 ticker,
6476 equityUrl,
6477 formatDate(new Date()),
6478 rangeEnd,
6479 signal
6480 );
6481
6482 if (directResult?.date) {
6483 calendarMap.set(
6484 ticker,
6485 directResult.date
6486 );
6487
6488 positiveTickerCache.set(
6489 ticker,
6490 directResult.date
6491 );
6492
6493 await saveInvestingPositiveTicker(
6494 ticker,
6495 directResult.date
6496 );
6497
6498 if (negativeCache[ticker]) {
6499 await clearInvestingNegative(
6500 ticker
6501 );
6502
6503 delete negativeCache[ticker];
6504 }
6505
6506 added++;
6507
6508 } else if (
6509 directResult?.conclusiveNoDate ===
6510 true
6511 ) {
6512 await saveInvestingConfirmedNegative(
6513 ticker,
6514 directResult.reason
6515 );
6516
6517 negativeCache[ticker] = {
6518 status:
6519 "NO_DATE",
6520 reason:
6521 directResult.reason,
6522 confirmedAt:
6523 new Date().toISOString()
6524 };
6525 }
6526
6527 } catch (error) {
6528 if (
6529 error?.name ===
6530 "AbortError"
6531 ) {
6532 throw error;
6533 }
6534
6535 console.log(
6536 `[INV TICKER:${ticker}] Targeted fallback failed safely: ${error.message}`
6537 );
6538 }
6539 }
6540
6541 console.log(
6542 `[INV TICKER] Targeted fallback complete: attempted=${attempted}; added=${added}; unresolved=${Math.max(0, missing.length - added)}.`
6543 );
6544
6545 return {
6546 attempted,
6547 added
6548 };
6549
6550 }
6551
6552
6553 async function configureInvestingFallbackPage(
6554 fallbackPage,
6555 calendarPage
6556 ) {
6557 await fallbackPage.setUserAgent(USER_AGENT);
6558
6559 if (calendarPage.__investingProxyAuth) {
6560 await fallbackPage.authenticate(
6561 calendarPage.__investingProxyAuth
6562 );
6563 }
6564
6565 await fallbackPage.setViewport({
6566 width: 1440,
6567 height: 1000,
6568 deviceScaleFactor: 1
6569 });
6570
6571 await fallbackPage.setExtraHTTPHeaders({
6572 "Accept-Language": "en-US,en;q=0.9"
6573 });
6574
6575 await fallbackPage.setRequestInterception(true);
6576
6577
6578
6579
6580
6581
6582
6583
6584 fallbackPage.__investingBearerToken =
6585 calendarPage.__investingBearerToken ||
6586 null;
6587
6588 fallbackPage.__investingInstrumentTemplate =
6589 calendarPage.__investingInstrumentTemplate ||
6590 null;
6591
6592 fallbackPage.__investingEndpoint200Seen = false;
6593
6594 fallbackPage.on("request", request => {
6595 try {
6596 const requestUrl = request.url();
6597 const lowerUrl = requestUrl.toLowerCase();
6598
6599 if (lowerUrl.includes("endpoints.investing.com")) {
6600 const headers = request.headers();
6601 const authorization =
6602 headers.authorization ||
6603 headers.Authorization ||
6604 "";
6605
6606 if (/^Bearer\s+\S+/i.test(authorization)) {
6607 const firstCapture =
6608 !fallbackPage.__investingBearerToken;
6609
6610 fallbackPage.__investingBearerToken =
6611 authorization;
6612
6613 if (firstCapture) {
6614 console.log(
6615 "[INV TAB2 API] Captured Bearer authorization from Tab 2 first-party traffic."
6616 );
6617 }
6618 }
6619
6620 if (
6621 lowerUrl.includes("/instruments") &&
6622 lowerUrl.includes("instrument_ids=") &&
6623 !lowerUrl.includes("/earnings") &&
6624 !lowerUrl.includes("key-metrics")
6625 ) {
6626 fallbackPage.__investingInstrumentTemplate =
6627 requestUrl;
6628
6629 invTrace(
6630 "tab2_instrument_template_captured"
6631 );
6632 }
6633 }
6634 } catch {}
6635
6636 const type = request.resourceType();
6637
6638
6639
6640
6641 let hostname = "";
6642 try {
6643 hostname = new URL(request.url()).hostname.toLowerCase();
6644 } catch {}
6645
6646 const firstParty =
6647 hostname === "investing.com" ||
6648 hostname.endsWith(".investing.com");
6649
6650 const obviousNoise =
6651 /doubleclick|googlesyndication|googleadservices|googletagmanager|google-analytics|scorecardresearch|doubleverify|pubmatic|admiral|prebid|amazon-adsystem/i.test(hostname);
6652
6653 const heavyVisual =
6654 type === "image" ||
6655 type === "media" ||
6656 type === "font";
6657
6658 if (!firstParty && (heavyVisual || obviousNoise)) {
6659 request.abort().catch(() => {});
6660 return;
6661 }
6662
6663 request.continue().catch(() => {});
6664 });
6665
6666 fallbackPage.on("response", response => {
6667 try {
6668 const responseUrl =
6669 response.url().toLowerCase();
6670
6671 if (
6672 responseUrl.includes("endpoints.investing.com") &&
6673 response.status() === 200 &&
6674 String(
6675 response.headers()["content-type"] ||
6676 ""
6677 ).toLowerCase().includes("json")
6678 ) {
6679 fallbackPage.__investingEndpoint200Seen = true;
6680 }
6681 } catch {}
6682 });
6683 }
6684
6685
6686 function startConcurrentInvestingTickerFallback({
6687 calendarPage,
6688 requestedTickers,
6689 calendarMap,
6690 rangeStart,
6691 rangeEnd,
6692 signal
6693 }) {
6694 const initialMissing =
6695 requestedTickers
6696 .map(normalizeTicker)
6697 .filter(
6698 ticker =>
6699 ticker &&
6700 !calendarMap.has(ticker)
6701 );
6702
6703 if (!initialMissing.length) {
6704 console.log(
6705 "[INV TICKER] No current misses when concurrent fallback was offered."
6706 );
6707
6708 return Promise.resolve({
6709 attempted: 0,
6710 added: 0,
6711 concurrent: true
6712 });
6713 }
6714
6715 console.log(
6716 `[INV TICKER] Starting SECOND-TAB concurrent fallback for ${initialMissing.length} current miss(es), one ticker at a time: ${initialMissing.join(", ")}`
6717 );
6718
6719 return (async () => {
6720 let fallbackPage = null;
6721
6722 try {
6723 fallbackPage =
6724 await calendarPage.browser().newPage();
6725
6726 await configureInvestingFallbackPage(
6727 fallbackPage,
6728 calendarPage
6729 );
6730
6731 return await fillMissingRequestedTickersFromEquityPages({
6732 page: fallbackPage,
6733 requestedTickers: initialMissing,
6734 calendarMap,
6735 rangeStart,
6736 rangeEnd,
6737 signal
6738 });
6739 } catch (error) {
6740 if (error?.name === "AbortError") {
6741 throw error;
6742 }
6743
6744 console.log(
6745 `[INV TICKER] Second-tab concurrent fallback failed safely: ${error.message}`
6746 );
6747
6748 return {
6749 attempted: 0,
6750 added: 0,
6751 concurrent: true
6752 };
6753 } finally {
6754 if (fallbackPage) {
6755 try {
6756 await fallbackPage.close();
6757 } catch {}
6758 }
6759 }
6760 })();
6761 }
6762
6763
6764 async function waitForInvestingWeekApiReadiness(
6765 page,
6766 label,
6767 timeoutMs = 90000
6768 ) {
6769 const startedAt = Date.now();
6770
6771 while (
6772 Date.now() - startedAt < timeoutMs
6773 ) {
6774 if (
6775 page.__investingBearerToken &&
6776 page.__investingInstrumentTemplate
6777 ) {
6778 invTrace(
6779 `${label}_api_ready wait_ms=${Date.now() - startedAt} bearer=YES instruments=YES`
6780 );
6781
6782 return true;
6783 }
6784
6785 await sleep(500);
6786 }
6787
6788 invTrace(
6789 `${label}_api_ready_timeout wait_ms=${Date.now() - startedAt} ` +
6790 `bearer=${page.__investingBearerToken ? "YES" : "NO"} ` +
6791 `instruments=${page.__investingInstrumentTemplate ? "YES" : "NO"}`
6792 );
6793
6794 return Boolean(
6795 page.__investingBearerToken
6796 );
6797 }
6798
6799
6800 async function scanCalendar(
6801 page,
6802 startDate,
6803 daysToScan,
6804 requestedTickers = [],
6805 signal
6806 ) {
6807
6808
6809
6810
6811
6812
6813
6814
6815
6816
6817
6818 const calendarMap = new Map();
6819
6820 const base = new Date(startDate.getTime());
6821 base.setUTCHours(0, 0, 0, 0);
6822
6823
6824
6825
6826
6827
6828
6829
6830
6831
6832
6833
6834
6835
6836 const passiveState = page.__investingPassiveApiCapture;
6837 const observedCurrentRanges = [];
6838
6839 for (const record of passiveState?.earningsPayloads || []) {
6840 if (record?.startDate && record?.endDate) {
6841 observedCurrentRanges.push({
6842 startDate: record.startDate,
6843 endDate: record.endDate,
6844 source: "captured-body"
6845 });
6846 }
6847 }
6848
6849 for (const meta of passiveState?.requestMeta?.values?.() || []) {
6850 if (
6851 meta?.kind === "earnings" &&
6852 meta?.startDate &&
6853 meta?.endDate
6854 ) {
6855 observedCurrentRanges.push({
6856 startDate: meta.startDate,
6857 endDate: meta.endDate,
6858 source: "observed-request"
6859 });
6860 }
6861 }
6862
6863 let weekStart = null;
6864 let weekEnd = null;
6865
6866 if (observedCurrentRanges.length > 0) {
6867 const currentRange = observedCurrentRanges[observedCurrentRanges.length - 1];
6868 const currentEnd = new Date(`${currentRange.endDate}T00:00:00.000Z`);
6869
6870 if (!Number.isNaN(currentEnd.getTime())) {
6871 weekStart = formatDate(addDays(currentEnd, 1));
6872 weekEnd = formatDate(addDays(currentEnd, 7));
6873
6874 console.log(
6875 `[INV WEEK2 RANGE] Derived from Investing current-week ${currentRange.source}: ` +
6876 `${currentRange.startDate}..${currentRange.endDate} -> Next Week ${weekStart}..${weekEnd}`
6877 );
6878 }
6879 }
6880
6881 if (!weekStart || !weekEnd) {
6882
6883
6884 const dow = base.getUTCDay();
6885 const daysUntilNextMonday = dow === 0 ? 8 : 8 - dow;
6886 const nextMonday = addDays(base, daysUntilNextMonday);
6887 weekStart = formatDate(nextMonday);
6888 weekEnd = formatDate(addDays(nextMonday, 6));
6889
6890 console.log(
6891 `[INV WEEK2 RANGE] Current-week API boundary not observed; ` +
6892 `using next Monday-Sunday fallback ${weekStart}..${weekEnd}.`
6893 );
6894 }
6895
6896 console.log(`[INV WEEK2] Standalone next-week worker: ${weekStart}..${weekEnd}`);
6897
6898
6899
6900 const attachWeek2Range = () => {
6901 calendarMap.__investingWeek2Range = {
6902 start: weekStart,
6903 end: weekEnd
6904 };
6905 return calendarMap;
6906 };
6907
6908 attachWeek2Range();
6909
6910 if (isAborted(signal)) return attachWeek2Range();
6911
6912 let apiRows = 0;
6913 let domRows = 0;
6914 let fastDomRows = 0;
6915
6916
6917
6918
6919
6920
6921
6922
6923 try {
6924 const clickResult = await clickInvestingRangeButton(
6925 page,
6926 "Next Week",
6927 weekStart,
6928 weekEnd,
6929 null,
6930 null,
6931 () => getInvestingPassiveWeekMapNow(page, weekStart, weekEnd)
6932 );
6933
6934 console.log(
6935 `[INV WEEK2] Next Week action returned; clicked=${Boolean(clickResult && (clickResult === true || clickResult.clicked))}.`
6936 );
6937
6938 if (clickResult?.passiveWeekMap?.size > 0) {
6939 mergeCalendarMapsInRange({
6940 target: calendarMap,
6941 source: clickResult.passiveWeekMap,
6942 rangeStart: weekStart,
6943 rangeEnd: weekEnd
6944 });
6945 apiRows = Math.max(apiRows, clickResult.passiveWeekMap.size);
6946 console.log(`✅ [INV WEEK2] Passive API completed during Next Week capture; accepted ${clickResult.passiveWeekMap.size} verified row(s) without waiting for the remaining capture window.`);
6947 calendarMap.__investingTickerFallbackResult = {
6948 attempted: 0, added: 0, splitWorker: "week2",
6949 week2VerifiedRows: clickResult.passiveWeekMap.size
6950 };
6951 return attachWeek2Range();
6952 }
6953
6954 if (clickResult?.earningsJson && clickResult?.instrumentsJson) {
6955 const clickMap = parseInvestingApiResponses(
6956 clickResult.earningsJson,
6957 clickResult.instrumentsJson,
6958 weekStart,
6959 weekEnd
6960 );
6961 apiRows = Math.max(apiRows, clickMap.size);
6962 mergeCalendarMapsInRange({
6963 target: calendarMap,
6964 source: clickMap,
6965 rangeStart: weekStart,
6966 rangeEnd: weekEnd
6967 });
6968 if (clickMap.size > 0) {
6969 console.log(`✅ [INV WEEK2] Next Week click captured ${clickMap.size} verified API row(s).`);
6970 calendarMap.__investingTickerFallbackResult = {
6971 attempted: 0, added: 0, splitWorker: "week2",
6972 week2VerifiedRows: clickMap.size
6973 };
6974 return attachWeek2Range();
6975 }
6976 }
6977 } catch (error) {
6978 if (error?.name === "AbortError") throw error;
6979 console.log(`[INV WEEK2] Next Week action failed safely: ${error.message}`);
6980 }
6981
6982
6983
6984
6985 try {
6986
6987
6988
6989 let passiveMap = await readInvestingPassiveWeekMap(
6990 page, weekStart, weekEnd, 3000
6991 );
6992
6993
6994
6995
6996
6997
6998
6999
7000 if (passiveMap.size === 0) {
7001 const stillUsable = await waitForInvestingCalendarMaterialization(
7002 page,
7003 "Next Week dispatch delayed; same-identity recovery"
7004 );
7005
7006 if (stillUsable.ready) {
7007 console.log(
7008 "[INV WEEK2 ACCESS] Calendar identity still healthy; re-triggering Next Week once before API fallback/rotation."
7009 );
7010
7011 const passiveState = page.__investingPassiveApiCapture;
7012 const earningsRequestsBeforeRetry =
7013 passiveState?.earningsGetRequestsSeen || 0;
7014
7015 const retryAction = await clickInvestingRangeButton(
7016 page,
7017 "Next Week",
7018 weekStart
7019 );
7020
7021 console.log(
7022 `[INV WEEK2 ACCESS] Same-identity Next Week retry returned; clicked=${Boolean(retryAction?.clicked)}.`
7023 );
7024
7025 const earningsRequestsAfterRetry =
7026 passiveState?.earningsGetRequestsSeen || 0;
7027
7028 if (
7029 earningsRequestsBeforeRetry === 0 &&
7030 earningsRequestsAfterRetry === 0
7031 ) {
7032 console.log(
7033 "[INV WEEK2 ACCESS] Two Next Week actions produced no earnings GET request; abandoning this healthy-but-stalled identity immediately so the outer runner can rotate."
7034 );
7035 calendarMap.__investingForceRotate = true;
7036 calendarMap.__investingTickerFallbackResult = {
7037 attempted: 0,
7038 added: 0,
7039 splitWorker: "week2",
7040 week2VerifiedRows: 0,
7041 reason: "NEXT_WEEK_NO_EARNINGS_REQUEST_AFTER_RETRY"
7042 };
7043 return attachWeek2Range();
7044 }
7045
7046 passiveMap = await readInvestingPassiveWeekMap(
7047 page,
7048 weekStart,
7049 weekEnd,
7050 NEXT_WEEK_SAME_IDENTITY_RETRY_WAIT_MS
7051 );
7052
7053 if (passiveMap.size > 0) {
7054 console.log(
7055 `✅ [INV WEEK2 ACCESS] Same-identity retry recovered ${passiveMap.size} verified Week-2 ticker/date pair(s).`
7056 );
7057 }
7058 } else {
7059 console.log(
7060 "[INV WEEK2 ACCESS] Existing identity no longer looks usable; continuing to normal fallback/rotation."
7061 );
7062 }
7063 }
7064
7065 apiRows = Math.max(apiRows, passiveMap.size);
7066 mergeCalendarMapsInRange({
7067 target: calendarMap,
7068 source: passiveMap,
7069 rangeStart: weekStart,
7070 rangeEnd: weekEnd
7071 });
7072 if (passiveMap.size > 0) {
7073 console.log(`✅ [INV WEEK2] Passive first-party API produced ${passiveMap.size} verified next-week row(s).`);
7074 calendarMap.__investingTickerFallbackResult = {
7075 attempted: 0, added: 0, splitWorker: "week2",
7076 week2VerifiedRows: passiveMap.size
7077 };
7078 return attachWeek2Range();
7079 }
7080 } catch (error) {
7081 console.log(`[INV WEEK2] Passive API failed safely: ${error.message}`);
7082 }
7083
7084 try {
7085 const retryMap = await fetchInvestingWeekViaApi(
7086 page, weekStart, weekEnd, weekStart, weekEnd
7087 );
7088 apiRows = Math.max(apiRows, retryMap.size);
7089 mergeCalendarMapsInRange({
7090 target: calendarMap,
7091 source: retryMap,
7092 rangeStart: weekStart,
7093 rangeEnd: weekEnd
7094 });
7095 if (retryMap.size > 0) {
7096 console.log(`✅ [INV WEEK2] Post-transition exact API succeeded with ${retryMap.size} verified row(s).`);
7097 calendarMap.__investingTickerFallbackResult = {
7098 attempted: 0, added: 0, splitWorker: "week2",
7099 week2VerifiedRows: retryMap.size
7100 };
7101 return attachWeek2Range();
7102 }
7103 } catch (error) {
7104 console.log(`[INV WEEK2] Post-transition API failed safely: ${error.message}`);
7105 }
7106
7107
7108
7109
7110
7111 const latePassiveMap = getInvestingPassiveWeekMapNow(
7112 page, weekStart, weekEnd
7113 );
7114 if (latePassiveMap.size > 0) {
7115 mergeCalendarMapsInRange({
7116 target: calendarMap,
7117 source: latePassiveMap,
7118 rangeStart: weekStart,
7119 rangeEnd: weekEnd
7120 });
7121 console.log(`✅ [INV WEEK2] Late passive API pair produced ${latePassiveMap.size} verified next-week row(s).`);
7122 calendarMap.__investingTickerFallbackResult = {
7123 attempted: 0, added: 0, splitWorker: "week2",
7124 week2VerifiedRows: latePassiveMap.size
7125 };
7126 return attachWeek2Range();
7127 }
7128
7129 try {
7130 const fastVisibleMap = await parseVisibleCalendarFast(page);
7131 const fastWeekMap = new Map(
7132 [...fastVisibleMap.entries()].filter(
7133 ([, date]) => date >= weekStart && date <= weekEnd
7134 )
7135 );
7136 fastDomRows = fastWeekMap.size;
7137 domRows = fastDomRows;
7138 mergeCalendarMapsInRange({
7139 target: calendarMap,
7140 source: fastWeekMap,
7141 rangeStart: weekStart,
7142 rangeEnd: weekEnd
7143 });
7144 if (fastWeekMap.size > 0) {
7145 console.log(`✅ [INV WEEK2] Fast visible DOM produced ${fastWeekMap.size} verified row(s).`);
7146 }
7147 } catch (error) {
7148 console.log(`[INV WEEK2] Fast DOM failed safely: ${error.message}`);
7149 }
7150
7151 if (calendarMap.size === 0) {
7152 try {
7153 const visibleMap = await Promise.race([
7154 parseVisibleCalendar(page, null),
7155 sleep(4000).then(() => {
7156 throw new Error("standalone Week-2 DOM parse exceeded 4000ms");
7157 })
7158 ]);
7159 const weekMap = new Map(
7160 [...visibleMap.entries()].filter(
7161 ([, date]) => date >= weekStart && date <= weekEnd
7162 )
7163 );
7164 domRows = Math.max(domRows, weekMap.size);
7165 mergeCalendarMapsInRange({
7166 target: calendarMap,
7167 source: weekMap,
7168 rangeStart: weekStart,
7169 rangeEnd: weekEnd
7170 });
7171 } catch (error) {
7172 console.log(`[INV WEEK2] Last-resort DOM failed safely: ${error.message}`);
7173 }
7174 }
7175
7176 calendarMap.__investingTickerFallbackResult = {
7177 attempted: 0, added: 0, splitWorker: "week2",
7178 week2VerifiedRows: calendarMap.size,
7179 week2ApiRows: apiRows,
7180 week2DomRows: domRows,
7181 week2FastDomRows: fastDomRows
7182 };
7183
7184
7185
7186
7187 calendarMap.__investingWeek2Range = {
7188 start: weekStart,
7189 end: weekEnd
7190 };
7191
7192 return calendarMap;
7193 }
7194
7195
7196
7197
7198
7199
7200
7201
7202
7203
7204
7205 function buildIncidentalResults(
7206 calendarMap,
7207 rangeStart,
7208 rangeEnd,
7209 sourceLabel = "Investing.com"
7210 ) {
7211 const results = [];
7212
7213 for (
7214 const [ticker, date]
7215 of calendarMap || []
7216 ) {
7217 if (
7218 ticker &&
7219 date &&
7220 date >= rangeStart
7221 ) {
7222 results.push({
7223 ticker,
7224 date,
7225 source: sourceLabel
7226 });
7227 }
7228 }
7229
7230 results.sort(
7231 (a, b) =>
7232 a.date.localeCompare(b.date) ||
7233 a.ticker.localeCompare(b.ticker)
7234 );
7235
7236 return results;
7237 }
7238
7239 async function run({
7240 tickers = [],
7241 proxyConfiguration,
7242 signal,
7243 includeIncidental = false
7244 } = {}) {
7245
7246 console.log(
7247 "Launching Investing.com WEEK2 standalone scraper..."
7248 );
7249
7250 const requestedTickers =
7251 Array.isArray(tickers)
7252 ? [
7253 ...new Set(
7254 tickers
7255 .map(
7256 normalizeTicker
7257 )
7258 .filter(Boolean)
7259 )
7260 ]
7261 : [];
7262
7263 console.log(
7264 "Requested tickers:",
7265 requestedTickers
7266 );
7267
7268 if (
7269 !requestedTickers.length
7270 ) {
7271 return [];
7272 }
7273
7274 const DAYS_TO_SCAN =
7275 7;
7276
7277 const currentRangeStartDate =
7278 new Date();
7279
7280 currentRangeStartDate.setUTCHours(
7281 0,
7282 0,
7283 0,
7284 0
7285 );
7286
7287 const currentRangeStart =
7288 formatDate(
7289 currentRangeStartDate
7290 );
7291
7292 const currentRangeEnd =
7293 formatDate(
7294 addDays(
7295 currentRangeStartDate,
7296 DAYS_TO_SCAN - 1
7297 )
7298 );
7299
7300
7301
7302
7303
7304
7305
7306
7307
7308
7309 const negativeCache =
7310 await loadInvestingNegativeCache();
7311
7312 const positiveTickerCache =
7313 await loadInvestingPositiveTickerCache(currentRangeStart);
7314
7315 const cached =
7316 await loadInvestingCache();
7317
7318 if (cached) {
7319 console.log(
7320 "[INV CACHE] Fresh CSV found. Skipping Investing.com scrape and proxy use."
7321 );
7322
7323 const cachedWeek2Start =
7324 normalizeDate(cached.meta?.rangeStart) || currentRangeStart;
7325 const cachedWeek2End =
7326 normalizeDate(cached.meta?.rangeEnd) || currentRangeEnd;
7327
7328 console.log(
7329 `[INV WEEK2 RANGE] Fresh cache authoritative range: ${cachedWeek2Start}..${cachedWeek2End}.`
7330 );
7331
7332 for (const ticker of requestedTickers) {
7333 const positiveDate = positiveTickerCache.get(ticker);
7334 if (positiveDate) {
7335 cached.calendarMap.set(ticker, positiveDate);
7336 }
7337
7338 if (
7339 cached.calendarMap.has(ticker) &&
7340 negativeCache[ticker]
7341 ) {
7342 await clearInvestingNegative(
7343 ticker
7344 );
7345
7346 delete negativeCache[ticker];
7347 }
7348 }
7349
7350 const results =
7351 buildRequestedResults(
7352 requestedTickers,
7353 cached.calendarMap,
7354 DAYS_TO_SCAN,
7355 "Investing.com",
7356 currentRangeStart,
7357 cachedWeek2End,
7358 cachedWeek2Start,
7359 cachedWeek2End
7360 );
7361
7362 console.log(
7363 "Investing cached results:",
7364 JSON.stringify(
7365 results
7366 )
7367 );
7368
7369 if (includeIncidental) {
7370 const allCachedResults =
7371 buildIncidentalResults(
7372 cached.calendarMap,
7373 currentRangeStart,
7374 cachedWeek2End,
7375 "Investing.com"
7376 );
7377
7378 console.log(
7379 `[INV INCIDENTAL] Returning ${allCachedResults.length} total future rows from fresh cache ` +
7380 `(${results.length} requested result(s); incidental calendar rows included).`
7381 );
7382
7383 const combinedCachedResults =
7384 new Map(
7385 allCachedResults.map(
7386 row => [normalizeTicker(row.ticker), row]
7387 )
7388 );
7389
7390 for (const row of results) {
7391 combinedCachedResults.set(
7392 normalizeTicker(row.ticker),
7393 row
7394 );
7395 }
7396
7397 return [...combinedCachedResults.values()]
7398 .sort(
7399 (a, b) =>
7400 a.date.localeCompare(b.date) ||
7401 a.ticker.localeCompare(b.ticker)
7402 );
7403 }
7404
7405 return results;
7406 }
7407
7408 console.log(
7409 "[INV CACHE] No fresh CSV available. Performing one Investing.com refresh."
7410 );
7411
7412 console.log(
7413 `[INV] Fail-fast bounds: protocol=${PROTOCOL_TIMEOUT_MS}ms; warm-nav=${WARM_NAV_TIMEOUT_MS}ms; calendar-nav=${CALENDAR_NAV_TIMEOUT_MS}ms; path-wait=${CALENDAR_PATH_WAIT_MS}ms; access-settle=${CALENDAR_ACCESS_SETTLE_MS}ms; next-week-settle=${NEXT_WEEK_SETTLE_MS}ms.`
7414 );
7415
7416
7417
7418
7419
7420
7421
7422
7423
7424
7425
7426
7427
7428 const MAX_PROXY_ATTEMPTS = 3;
7429
7430
7431
7432
7433
7434
7435
7436 const sharedApiState = {
7437 authorization: null,
7438 capturedAt: null
7439 };
7440
7441
7442
7443 const accessPhaseStartedAt = Date.now();
7444
7445 for (
7446 let attempt = 1;
7447 attempt <= MAX_PROXY_ATTEMPTS;
7448 attempt++
7449 ) {
7450
7451 const accessAttemptStartedAt = Date.now();
7452
7453 if (isAborted(signal)) {
7454 console.log(
7455 "[INV] Production deadline reached before next proxy attempt."
7456 );
7457 return [];
7458 }
7459
7460 const accessElapsedMs = Date.now() - accessPhaseStartedAt;
7461 if (accessElapsedMs >= INV_ACCESS_TOTAL_BUDGET_MS) {
7462 console.log(
7463 `[INV WEEK2 ACCESS] Total access budget exhausted after ${accessElapsedMs}ms; ` +
7464 `stopping before proxy attempt ${attempt}.`
7465 );
7466 break;
7467 }
7468
7469 console.log(
7470 `[INV WEEK2 ACCESS] access_budget elapsed=${accessElapsedMs}ms ` +
7471 `remaining=${Math.max(0, INV_ACCESS_TOTAL_BUDGET_MS - accessElapsedMs)}ms`
7472 );
7473
7474 let browser = null;
7475
7476 const abortHandler = () => {
7477 if (browser) {
7478 browser.close().catch(() => {});
7479 }
7480 };
7481
7482 if (signal) {
7483 signal.addEventListener(
7484 "abort",
7485 abortHandler,
7486 { once: true }
7487 );
7488 }
7489
7490 const sessionId =
7491 `inv_${Date.now()}_${attempt}_${Math.random()
7492 .toString(36)
7493 .slice(2, 10)}`;
7494
7495 console.log(
7496 "\n================================================"
7497 );
7498
7499 console.log(
7500 `INV ACCESS ATTEMPT ${attempt}/${MAX_PROXY_ATTEMPTS}`
7501 );
7502
7503 console.log(
7504 "Fresh proxy session:",
7505 sessionId
7506 );
7507
7508 console.log(
7509 "================================================"
7510 );
7511
7512 try {
7513
7514 throwIfAborted(signal);
7515
7516 const launched =
7517 await launchBrowser(
7518 proxyConfiguration,
7519 sessionId,
7520 sharedApiState
7521 );
7522
7523 browser =
7524 launched.browser;
7525
7526 const page =
7527 launched.page;
7528
7529
7530
7531
7532
7533
7534
7535 console.log(
7536 "[INV] Production mode: skipping ipify proxy test to save residential proxy traffic."
7537 );
7538
7539
7540
7541
7542
7543
7544
7545 const accessPath = "direct";
7546 console.log(`[INV WEEK2 ACCESS] path=${accessPath} session=${sessionId}; proxy gateway address may stay the same across residential exits.`);
7547
7548 if (accessPath === "warmed") {
7549 try {
7550 const warmResponse = await page.goto("https://www.investing.com/", {
7551 waitUntil: "domcontentloaded",
7552 timeout: WARM_NAV_TIMEOUT_MS
7553 });
7554 const warmStatus = warmResponse?.status() ?? 0;
7555 const warmUrl = page.url();
7556 const warmTitle = await Promise.race([
7557 page.title().catch(() => ""),
7558 sleep(800).then(() => "")
7559 ]);
7560 const warmBlocked =
7561 [401, 403, 407, 429].includes(warmStatus) ||
7562 /__cf_chl|cdn-cgi\/challenge|\/cdn-cgi\//i.test(warmUrl) ||
7563 isBlocked(warmTitle);
7564 console.log(
7565 `[INV WEEK2 ACCESS] warm-up status=${warmStatus} ` +
7566 `blocked=${warmBlocked ? "YES" : "NO"} url=${warmUrl}`
7567 );
7568 if (warmBlocked) {
7569 const blockedError = new Error("Investing warm-up challenged; rotating before calendar navigation");
7570 blockedError.code = "INV_ACCESS_BLOCKED";
7571 throw blockedError;
7572 }
7573 } catch (error) {
7574 if (error?.code === "INV_ACCESS_BLOCKED" ||
7575 /ERR_TUNNEL_CONNECTION_FAILED/i.test(String(error.message))) throw error;
7576 if (/Navigation timeout/i.test(String(error.message))) {
7577 console.log("[INV WEEK2 ACCESS] Warm-up timed out; rotating rather than navigating from a half-loaded page.");
7578 throw error;
7579 }
7580 console.log(`[INV WEEK2 ACCESS] Warm-up incomplete: ${error.message}; probing calendar.`);
7581 }
7582 }
7583
7584
7585 let probe = await openCalendar(page, accessPath);
7586
7587 const accessAttemptElapsedAfterProbe =
7588 Date.now() - accessAttemptStartedAt;
7589
7590 if (
7591 !probe.accessible &&
7592 accessAttemptElapsedAfterProbe >= INV_ACCESS_FAILURE_CAP_MS
7593 ) {
7594 console.log(
7595 `[INV WEEK2 ACCESS] Per-session failure cap reached after ` +
7596 `${accessAttemptElapsedAfterProbe}ms ` +
7597 `(cap=${INV_ACCESS_FAILURE_CAP_MS}ms); skipping further same-identity recovery.`
7598 );
7599 }
7600
7601
7602
7603
7604
7605
7606
7607
7608
7609
7610
7611 if (!probe.accessible && !probe.blocked &&
7612 Date.now() - accessAttemptStartedAt < INV_ACCESS_FAILURE_CAP_MS &&
7613 page.url().toLowerCase().includes("earnings-calendar") &&
7614 !isAborted(signal)) {
7615
7616 const hasApiEvidence =
7617 Boolean(page.__investingBearerToken) ||
7618 Boolean(page.__investingEndpoint200Seen);
7619
7620 if (!hasApiEvidence) {
7621 console.log(
7622 "[INV WEEK2 ACCESS] Empty calendar shell with bearer=NO endpoint200=NO; " +
7623 "skipping same-identity reload and rotating immediately."
7624 );
7625 } else {
7626 console.log(
7627 `[INV WEEK2 ACCESS] Empty shell but API evidence exists ` +
7628 `(bearer=${page.__investingBearerToken ? "YES" : "NO"} ` +
7629 `endpoint200=${page.__investingEndpoint200Seen ? "YES" : "NO"}); ` +
7630 "trying one short same-identity reload."
7631 );
7632
7633 try {
7634 await page.reload({
7635 waitUntil: "domcontentloaded",
7636 timeout: CALENDAR_EMPTY_SHELL_RELOAD_TIMEOUT_MS
7637 });
7638 } catch (reloadError) {
7639 console.log(
7640 `[INV WEEK2 ACCESS] Short recovery reload incomplete: ${reloadError.message}`
7641 );
7642 }
7643
7644 if (
7645 !isAborted(signal) &&
7646 Date.now() - accessAttemptStartedAt < INV_ACCESS_FAILURE_CAP_MS
7647 ) {
7648 const recovery = await waitForInvestingCalendarMaterialization(
7649 page,
7650 "post-reload calendar shell"
7651 );
7652
7653 if (recovery.ready) {
7654 probe = {
7655 ...probe,
7656 accessible: true,
7657 blocked: false,
7658 recovery: "API_EVIDENCE_RELOAD"
7659 };
7660 console.log(
7661 "[INV WEEK2 ACCESS] Same-identity recovery succeeded; retaining session."
7662 );
7663 } else {
7664 console.log(
7665 "[INV WEEK2 ACCESS] Same-identity recovery produced no usable calendar/API evidence."
7666 );
7667 }
7668 }
7669 }
7670 }
7671
7672 if (
7673 !probe.accessible
7674 ) {
7675
7676 console.log(
7677 `⚠️ INV access attempt ${attempt} failed calendar probe.`
7678 );
7679
7680 if (
7681 attempt <
7682 MAX_PROXY_ATTEMPTS
7683 ) {
7684 console.log(
7685 "Closing blocked session and rotating proxy..."
7686 );
7687 }
7688
7689 continue;
7690 }
7691
7692 console.log(
7693 `✅ INV access attempt ${attempt} passed. Starting calendar scan.`
7694 );
7695
7696
7697
7698
7699
7700
7701
7702 const today =
7703 new Date();
7704
7705 today.setUTCHours(
7706 0,
7707 0,
7708 0,
7709 0
7710 );
7711
7712 const daysToScan =
7713 DAYS_TO_SCAN;
7714
7715 const endDate =
7716 addDays(
7717 today,
7718 daysToScan - 1
7719 );
7720
7721 console.log(
7722 "\n========================================"
7723 );
7724
7725 console.log(
7726 "INVESTING CALENDAR SCAN"
7727 );
7728
7729 console.log(
7730 "========================================"
7731 );
7732
7733 console.log(
7734 "Start date:",
7735 formatDate(today)
7736 );
7737
7738 console.log(
7739 "End date:",
7740 formatDate(endDate)
7741 );
7742
7743 console.log(
7744 "Days scanned:",
7745 daysToScan
7746 );
7747
7748 console.log(
7749 "Requested tickers:",
7750 requestedTickers
7751 );
7752
7753
7754
7755
7756 const calendarMap =
7757 await scanCalendar(
7758 page,
7759 today,
7760 daysToScan,
7761 requestedTickers,
7762 signal
7763 );
7764
7765 if (calendarMap.__investingForceRotate) {
7766 console.log(
7767 "[INV WEEK2 ACCESS] Week-2 dispatch stalled after two Next Week actions; rotating immediately to a fresh proxy/browser identity."
7768 );
7769 continue;
7770 }
7771
7772 const authoritativeWeek2Start =
7773 normalizeDate(calendarMap.__investingWeek2Range?.start) || formatDate(today);
7774 const authoritativeWeek2End =
7775 normalizeDate(calendarMap.__investingWeek2Range?.end) || formatDate(endDate);
7776
7777 console.log(
7778 `[INV WEEK2 RANGE] Outer runner authoritative target: ${authoritativeWeek2Start}..${authoritativeWeek2End}.`
7779 );
7780
7781 console.log(
7782 "\n========================================"
7783 );
7784
7785 console.log(
7786 "INVESTING CALENDAR SCAN COMPLETE"
7787 );
7788
7789 console.log(
7790 "Unique tickers found:",
7791 calendarMap.size
7792 );
7793
7794 console.log(
7795 "========================================"
7796 );
7797
7798
7799
7800
7801
7802
7803
7804
7805
7806
7807
7808
7809
7810 const tickerFallback =
7811 calendarMap.__investingTickerFallbackResult ||
7812 await fillMissingRequestedTickersFromEquityPages({
7813 page,
7814 requestedTickers,
7815 calendarMap,
7816 rangeStart:
7817 authoritativeWeek2Start,
7818 rangeEnd:
7819 authoritativeWeek2End,
7820 signal,
7821 negativeCache,
7822 positiveTickerCache
7823 });
7824
7825 console.log(
7826 `[INV] Combined routes: calendar/ticker map=${calendarMap.size}; ` +
7827 `tickerFallbackAttempted=${tickerFallback.attempted}; ` +
7828 `tickerFallbackAdded=${tickerFallback.added}.`
7829 );
7830
7831 for (const ticker of requestedTickers) {
7832 if (
7833 calendarMap.has(ticker) &&
7834 negativeCache[ticker]
7835 ) {
7836 await clearInvestingNegative(
7837 ticker
7838 );
7839
7840 delete negativeCache[ticker];
7841 }
7842 }
7843
7844
7845
7846
7847
7848
7849
7850
7851 if (
7852 calendarMap.size === 0 &&
7853 attempt <
7854 MAX_PROXY_ATTEMPTS
7855 ) {
7856
7857 console.log(
7858 "⚠️ Investing scan collected zero tickers."
7859 );
7860
7861 console.log(
7862 "Rotating to another proxy/browser session..."
7863 );
7864
7865 continue;
7866 }
7867
7868
7869
7870
7871
7872
7873
7874 const snapshotCoverage =
7875 calendarCoverage(
7876 calendarMap
7877 );
7878
7879 const expectedRangeEnd =
7880 authoritativeWeek2End;
7881
7882 const completeSnapshot =
7883 Boolean(
7884 calendarMap.size > 0 &&
7885 snapshotCoverage.firstDate &&
7886 snapshotCoverage.lastDate
7887 );
7888
7889
7890
7891
7892
7893
7894
7895
7896
7897
7898
7899
7900
7901
7902
7903
7904
7905 let mapToSave =
7906 calendarMap;
7907
7908 console.log(
7909 `[INV DIAG] Fresh calendar map: ${calendarMap.size} row(s); ` +
7910 `coverage=${snapshotCoverage.firstDate || "NONE"}..${snapshotCoverage.lastDate || "NONE"}; ` +
7911 `target=${authoritativeWeek2Start}..${expectedRangeEnd}; ` +
7912 `snapshot=${completeSnapshot ? "COMPLETE" : "PARTIAL"}.`
7913 );
7914
7915 if (!completeSnapshot) {
7916 console.log(
7917 `[INV COVERAGE DIAG] partialReason=LAST_OBSERVED_EARNINGS_BEFORE_TARGET_END ` +
7918 `lastObserved=${snapshotCoverage.lastDate || "NONE"} ` +
7919 `targetEnd=${expectedRangeEnd} ` +
7920 `note="Current completeness rule uses the last observed earnings date; ` +
7921 `this alone cannot distinguish incomplete calendar access from no earnings on the target end date."`
7922 );
7923 }
7924
7925 if (
7926 calendarMap.size > 0
7927 ) {
7928
7929 if (
7930 !completeSnapshot
7931 ) {
7932 const existing =
7933 await loadExistingInvestingSnapshotForMerge();
7934
7935 mapToSave =
7936 mergeFreshInvestingSnapshot({
7937 existingMap:
7938 existing.calendarMap,
7939 freshMap:
7940 calendarMap,
7941 freshCoverageStart:
7942 snapshotCoverage.firstDate,
7943 freshCoverageEnd:
7944 snapshotCoverage.lastDate,
7945 rangeStart:
7946 currentRangeStart,
7947 rangeEnd:
7948 expectedRangeEnd,
7949 requestedTickers
7950 });
7951
7952 console.log(
7953 `[INV CACHE] Fresh scrape is PARTIAL but valid: ${calendarMap.size} verified ticker/date pairs covering ${snapshotCoverage.firstDate || "UNKNOWN"} through ${snapshotCoverage.lastDate || "UNKNOWN"}.`
7954 );
7955
7956 console.log(
7957 `[INV CACHE] Saving merged CSV with ${mapToSave.size} ticker/date pairs. Fresh verified coverage overwrites stale cached rows in that coverage.`
7958 );
7959
7960 console.log(
7961 "[INV CACHE] Snapshot metadata marked PARTIAL; next run will refresh again instead of using the 24-hour shortcut."
7962 );
7963 } else {
7964 console.log(
7965 `[INV CACHE] Verified Week-2 capture succeeded. Saving COMPLETE snapshot for the 24-hour cache shortcut.`
7966 );
7967 }
7968
7969 console.log(
7970 `[INV DIAG] Cache map selected: ${mapToSave.size} row(s); ` +
7971 `source=${completeSnapshot ? "FRESH_COMPLETE" : "MERGED_PARTIAL"}.`
7972 );
7973
7974 const cacheSaved =
7975 await saveInvestingCache({
7976 calendarMap:
7977 mapToSave,
7978 rangeStart:
7979 authoritativeWeek2Start,
7980 rangeEnd:
7981 expectedRangeEnd,
7982 complete:
7983 completeSnapshot,
7984 freshCoverageStart:
7985 snapshotCoverage.firstDate,
7986 freshCoverageEnd:
7987 snapshotCoverage.lastDate,
7988 freshRowCount:
7989 calendarMap.size
7990 });
7991
7992 console.log(
7993 `[INV DIAG] Cache save result: ${cacheSaved ? "SUCCESS" : "FAILED"}.`
7994 );
7995
7996 } else {
7997 console.log(
7998 "[INV CACHE] Fresh scrape produced zero verified rows. Existing CSV is left untouched."
7999 );
8000 }
8001
8002 const results =
8003 buildRequestedResults(
8004 requestedTickers,
8005 calendarMap,
8006 daysToScan,
8007 "Investing.com",
8008 currentRangeStart,
8009 expectedRangeEnd,
8010 authoritativeWeek2Start,
8011 authoritativeWeek2End
8012 );
8013
8014 console.log(
8015 "Investing results:",
8016 JSON.stringify(
8017 results
8018 )
8019 );
8020
8021 console.log(
8022 `[INV DIAG] Requested matches: ${results.length}/${requestedTickers.length}; ` +
8023 `includeIncidental=${includeIncidental}.`
8024 );
8025
8026 if (includeIncidental) {
8027 const incidentalSourceMap =
8028 calendarMap.size > 0
8029 ? (
8030 completeSnapshot
8031 ? calendarMap
8032 : mapToSave
8033 )
8034 : calendarMap;
8035
8036 const allResults =
8037 buildIncidentalResults(
8038 incidentalSourceMap,
8039 currentRangeStart,
8040 expectedRangeEnd,
8041 "Investing.com"
8042 );
8043
8044 const incidentalCount =
8045 Math.max(
8046 0,
8047 allResults.length - results.length
8048 );
8049
8050 console.log(
8051 `[INV INCIDENTAL] Returning ${allResults.length} total future rows ` +
8052 `(${results.length} requested result(s); ${incidentalCount} incidental row(s)).`
8053 );
8054
8055 console.log(
8056 `[INV SUMMARY] access=OK freshRows=${calendarMap.size} ` +
8057 `savedRows=${mapToSave.size} requestedMatches=${results.length} ` +
8058 `returnedRows=${allResults.length} snapshot=${completeSnapshot ? "COMPLETE" : "PARTIAL"} ` +
8059 `diagnosis=SUCCESS.`
8060 );
8061
8062 const combinedResults =
8063 new Map(
8064 allResults.map(
8065 row => [normalizeTicker(row.ticker), row]
8066 )
8067 );
8068
8069 for (const row of results) {
8070 combinedResults.set(
8071 normalizeTicker(row.ticker),
8072 row
8073 );
8074 }
8075
8076 return [...combinedResults.values()]
8077 .sort(
8078 (a, b) =>
8079 a.date.localeCompare(b.date) ||
8080 a.ticker.localeCompare(b.ticker)
8081 );
8082 }
8083
8084 console.log(
8085 `[INV SUMMARY] access=OK freshRows=${calendarMap.size} ` +
8086 `savedRows=${mapToSave.size} requestedMatches=${results.length} ` +
8087 `returnedRows=${results.length} snapshot=${completeSnapshot ? "COMPLETE" : "PARTIAL"} ` +
8088 `diagnosis=SUCCESS.`
8089 );
8090
8091 return results;
8092
8093 } catch (
8094 error
8095 ) {
8096
8097 console.log(
8098 `Investing.com attempt ${attempt} failed:`,
8099 error.message
8100 );
8101
8102 console.log(
8103 error.stack || ""
8104 );
8105
8106 if (
8107 attempt <
8108 MAX_PROXY_ATTEMPTS
8109 ) {
8110 console.log(
8111 "Rotating to a fresh proxy/browser session..."
8112 );
8113 }
8114
8115 } finally {
8116
8117 if (signal) {
8118 signal.removeEventListener(
8119 "abort",
8120 abortHandler
8121 );
8122 }
8123
8124 if (
8125 browser
8126 ) {
8127 try {
8128 await browser.close();
8129 } catch {}
8130 }
8131 }
8132
8133
8134
8135
8136 if (
8137 isAborted(signal)
8138 ) {
8139 console.log(
8140 "[INV] Production deadline reached; stopping proxy rotation."
8141 );
8142 return [];
8143 }
8144
8145 if (
8146 attempt <
8147 MAX_PROXY_ATTEMPTS
8148 ) {
8149 const elapsedBeforeBackoff = Date.now() - accessPhaseStartedAt;
8150 const remainingAccessMs = INV_ACCESS_TOTAL_BUDGET_MS - elapsedBeforeBackoff;
8151
8152 if (remainingAccessMs <= 0) {
8153 console.log(
8154 `[INV WEEK2 ACCESS] Total access budget exhausted after ${elapsedBeforeBackoff}ms; skipping backoff.`
8155 );
8156 break;
8157 }
8158
8159 await sleep(
8160 Math.min(500 * attempt, remainingAccessMs)
8161 );
8162 }
8163 }
8164
8165 console.log(
8166 `❌ Investing.com failed after ${MAX_PROXY_ATTEMPTS} fresh proxy sessions.`
8167 );
8168
8169 return [];
8170 }
8171
8172 export {
8173 run
8174 };
