This Actor is under maintenance.
This actor is under maintenance and it may unreliable.
.eslintrc
1{
2 "extends": "@apify"
3}
4.gitignore
1apify_storage
2node_modules.npmignore
1# This file tells Git which files shouldn't be added to source control
2
3.idea
4node_modules
5Dockerfile
1# Dockerfile contains instructions how to build a Docker image that will contain
2# all the code and configuration needed to run your actor. For a full
3# Dockerfile reference, see https://docs.docker.com/engine/reference/builder/
4
5# First, specify the base Docker image. Apify provides the following base images
6# for your convenience:
7# apify/actor-node-basic (Node.js 10 on Alpine Linux, small and fast image)
8# apify/actor-node-chrome (Node.js 10 + Chrome on Debian)
9# apify/actor-node-chrome-xvfb (Node.js 10 + Chrome + Xvfb on Debian)
10# For more information, see https://apify.com/docs/actor#base-images
11# Note that you can use any other image from Docker Hub.
12FROM apify/actor-node-basic
13
14# Second, copy just package.json and package-lock.json since they are the only files
15# that affect NPM install in the next step
16COPY package*.json ./
17
18# Install NPM packages, skip optional and development dependencies to keep the
19# image small. Avoid logging too much and print the dependency tree for debugging
20RUN npm --quiet set progress=false \
21 && npm install --only=prod --no-optional \
22 && echo "Installed NPM packages:" \
23 && npm list \
24 && echo "Node.js version:" \
25 && node --version \
26 && echo "NPM version:" \
27 && npm --version
28
29# Next, copy the remaining files and directories with the source code.
30# Since we do this after NPM install, quick build will be really fast
31# for simple source file changes.
32COPY . ./
33
34# Specify how to run the source code
35CMD npm start
36INPUT_SCHEMA.json
1{
2 "title": "Firestore Import input",
3 "description": "Imports dataset to Firestore DB",
4 "type": "object",
5 "schemaVersion": 1,
6 "properties": {
7 "datasetId": {
8 "title": "Dataset",
9 "type": "string",
10 "description": "Dataset ID of dataset you want to import to Firestore",
11 "editor": "textfield"
12 },
13 "apiKey": {
14 "title": "Api key",
15 "type": "string",
16 "description": "Firestore API key",
17 "editor": "textfield"
18 },
19 "authDomain": {
20 "title": "Auth domain",
21 "type": "string",
22 "description": "Firestore authentication domain",
23 "editor": "textfield"
24 },
25 "projectId": {
26 "title": "Project ID",
27 "type": "string",
28 "description": "Firestore project ID",
29 "editor": "textfield"
30 },
31 "collectionName": {
32 "title": "Collection name",
33 "type": "string",
34 "description": "Firestore collection name",
35 "editor": "textfield"
36 }
37 }
38}
39apify.json
1{
2 "name": "firestore-import",
3 "version": "0.0",
4 "buildTag": "latest",
5 "env": null,
6 "template": "hello_world"
7}
8main.js
1const Apify = require('apify');
2const firebase = require('firebase');
3const fetch = require('node-fetch'); // Import the fetch function
4
5function fetchMultipleUrls2(urls) {
6 try {
7 const promises = urls.map(url => fetch(url).then(response => response.json()));
8 return Promise.all(promises);
9 } catch (error) {
10 console.error('Error fetching data:', error);
11 }
12}
13
14function fetchMultipleUrls(urls) {
15 const promises = urls.map(url =>
16 fetch(url)
17 .then(response => {
18 if (response.status === 404 || response.status === 400) {
19 //console.warn(`URL ${url} returned a ${response.status} status.`);
20 return {}; // Return an empty object for 404 and 400 errors
21 }
22 if (!response.ok) {
23 // Handle non-OK responses here (e.g., other error status codes)
24 //console.error(`Failed to fetch ${url}. Status: ${response.status}`);
25 //throw new Error(`Failed to fetch ${url}. Status: ${response.status}`);
26 return {};
27 }
28 return response.json();
29 })
30 .catch(error => {
31 console.error(`Error while fetching ${url}: ${error}`);
32 return {}; // Return an empty object in case of an error
33 })
34 );
35
36 return Promise.all(promises);
37}
38
39async function startFetchingNseIndicesData() {
40 try {
41 // console.log("calling fetchMultipleUrls function");
42
43
44 var urls =[
45 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%2050.json",
46 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20NEXT%2050.json",
47 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20100.json",
48 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20200.json",
49 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20500.json",
50 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDCAP%20150.json",
51 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDCAP%2050.json",
52 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDCAP%20SELECT.json",
53 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDCAP%20100.json",
54 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SMALLCAP%20250.json",
55 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SMALLCAP%2050.json",
56 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SMALLCAP%20100.json",
57 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20LARGEMIDCAP%20250.json",
58 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDSMALLCAP%20400.json",
59 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20AUTO.json",
60 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20BANK.json",
61 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20FINANCIAL%20SERVICES.json",
62 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20FMCG.json",
63 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20IT.json",
64 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MEDIA.json",
65 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20METAL.json",
66 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20PHARMA.json",
67 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20PRIVATE%20BANK.json",
68 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20PSU%20BANK.json",
69 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20REALTY.json",
70 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20ADITYA%20BIRLA%20GROUP.json",
71 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20COMMODITIES.json",
72 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20SHARIAH.json",
73 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20CPSE.json",
74 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20ENERGY.json",
75 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20INDIA%20CONSUMPTION.json",
76 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20INFRASTRUCTURE.json",
77 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20INDIA%20MANUFACTURING.json",
78 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MAHINDRA%20GROUP.json",
79 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDCAP%20LIQUID%2015.json",
80 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MNC.json",
81 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20PSE.json",
82 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SERVICES%20SECTOR.json",
83 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SHARIAH%2025.json",
84 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20TATA%20GROUP.json",
85 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20TATA%20GROUP%2025%%20CAP.json",
86 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY100%20LIQUID%2015.json",
87 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20SHARIAH.json",
88 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY500%20SHARIAH.json",
89 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SME%20EMERGE.json",
90 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY100%20EQUAL%20WEIGHT.json",
91 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY100%20LOW%20VOLATILITY%2030.json",
92 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20ALPHA%2050.json",
93 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20EQUITY%20SAVINGS.json",
94 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%2050%20ARBITRAGE.json",
95 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%2050%20FUTURES%20PR.json",
96 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%2050%20FUTURES%20TR.json",
97 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20DIVIDEND%20OPPORTUNITIES%2050.json",
98 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20HIGH%20BETA%2050.json",
99 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20LOW%20VOLATILITY%2050.json",
100 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY100%20QUALITY%2030.json",
101 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20GROWTH%20SECTORS%2015.json",
102 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20DIVIDEND%20POINTS.json",
103 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20EQUAL%20WEIGHT.json",
104 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20PR%201X%20INVERSE.json",
105 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20PR%202X%20LEVERAGE.json",
106 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20TR%201X%20INVERSE.json",
107 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20TR%202X%20LEVERAGE.json",
108 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20USD.json",
109 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY50%20VALUE%2020.json",
110 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20AAA%20CORPORATE%20BOND.json",
111 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%201D%20RATE%20INDEX.json",
112 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%2010%20YEAR%20SDL%20INDEX.json",
113 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY100%20ALPHA%2030.json",
114 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY500%20VALUE%2050.json",
115 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY200%20QUALITY%2030.json",
116 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDCAP150%20QUALITY%2050.json",
117 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20CONSUMER%20DURABLES.json",
118 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20OIL%20&%20GAS.json",
119 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY200%20MOMENTUM%2030.json",
120 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20CPSE%20BOND%20PLUS%20SDL%20SEP%202024%2050:50%20INDEX.json",
121 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20HEALTHCARE.json",
122 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY500%20MULTICAP%2050:25:25.json",
123 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20AAA%20BOND%20PLUS%20SDL%20APR%202026%2050:50%20INDEX.json",
124 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20PSU%20BOND%20PLUS%20SDL%20APR%202026%2050:50%20INDEX.json",
125 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20PSU%20BOND%20PLUS%20SDL%20APR%202027%2050:50%20INDEX.json",
126 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20APR%202026%20TOP%2020%20EQUAL%20WEIGHT%20INDEX.json",
127 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MICROCAP%20250.json",
128 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY100%20ENHANCED%20ESG.json",
129 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20PLUS%20PSU%20BOND%20SEP%202026%2060:40%20INDEX.json",
130 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20PSU%20BOND%20PLUS%20SDL%20SEP%202027%2040:60%20INDEX.json",
131 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20TOTAL%20MARKET.json",
132 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20AAA%20BOND%20PLUS%20SDL%20APR%202026%2070:30%20INDEX.json",
133 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20AAA%20BOND%20PLUS%20SDL%20APR%202031%2070:30%20INDEX.json",
134 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MOBILITY.json",
135 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20INDIA%20DIGITAL.json",
136 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20CPSE%20BOND%20PLUS%20SDL%20SEP%202026%2050:50%20INDEX.json",
137 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20APR%202027%20INDEX.json",
138 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20INDIA%20DEFENCE.json",
139 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20APR%202027%20TOP%2012%20EQUAL%20WEIGHT%20INDEX.json",
140 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20APR%202032%20TOP%2012%20EQUAL%20WEIGHT%20INDEX.json",
141 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20HOUSING.json",
142 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20TRANSPORTATION%20&%20LOGISTICS.json",
143 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20PLUS%20AAA%20PSU%20BOND%20DEC%202027%2060:40%20INDEX.json",
144 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20JUN%202027%20INDEX.json",
145 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20SEP%202027%20INDEX.json",
146 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20AAA%20CPSE%20BOND%20PLUS%20SDL%20APR%202027%2060:40%20INDEX.json",
147 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapCOMMERCIAL%20PAPERS%20INDICES.json",
148 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20FIXED%20INCOME%20PRC%20INDICES.json",
149 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY200%20ALPHA%2030.json",
150 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDSMALL%20FINANCIAL%20SERVICES.json",
151 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDSMALL%20HEALTHCARE.json",
152 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDSMALL%20IT%20&%20TELECOM.json",
153 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDCAP150%20MOMENTUM%2050.json",
154 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20SEP%202025%20INDEX.json",
155 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20PLUS%20AAA%20PSU%20BOND%20JUL%202028%2060:40%20INDEX.json",
156 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20DEC%202028%20INDEX.json",
157 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20AAA%20PSU%20BOND%20PLUS%20SDL%20APR%202026%2050:50%20INDEX.json",
158 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20PLUS%20AAA%20PSU%20BOND%20JUL%202033%2060:40%20INDEX.json",
159 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20AAA%20PSU%20BOND%20PLUS%20SDL%20SEP%202026%2050:50%20INDEX.json",
160 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20MIDSMALL%20INDIA%20CONSUMPTION.json",
161 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20SEP%202026%20INDEX.json",
162 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20SEP%202026%20V1%20INDEX.json",
163 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20JUL%202026%20INDEX.json",
164 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20DEC%202026%20INDEX.json",
165 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20PLUS%20AAA%20PSU%20BOND%20APR%202026%2075:25%20INDEX.json",
166 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20JUL%202033%20INDEX.json",
167 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20OCT%202026%20INDEX.json",
168 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20INDIA%20MUNICIPAL%20BOND.json",
169 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20PLUS%20AAA%20PSU%20BOND%20APR%202028%2075:25.json",
170 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20INDIA%20SOVEREIGN%20GREEN%20BOND%20JAN%202033.json",
171 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20INDIA%20SOVEREIGN%20GREEN%20BOND%20JAN%202028.json",
172 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20JUL%202028.json",
173 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SMALLCAP250%20QUALITY%2050.json",
174 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20SDL%20JUNE%202028.json",
175 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20REITS%20&%20INVITS.json",
176 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20CORE%20HOUSING.json",
177 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%205%20YEAR%20SDL.json",
178 "https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%203%20YEAR%20SDL.json"
179]
180 fetchedDataArray = await fetchMultipleUrls(urls);
181
182 // Loop through and print the fetched data
183 fetchedDataArray.forEach((data, index) => {
184 if( Object.keys(data).length>0 )
185 {
186 urlCount = urlCount + 1
187 var fetchedUrl = urls[index]
188 var fetchedData = data
189 var returnedDataObj = handleFetchedData( fetchedUrl, fetchedData )
190
191 var pathOfDocument = returnedDataObj['id'] ;
192
193 onlyForSectorsStocksObj["data"][ returnedDataObj['id'] ] = {
194 "data": returnedDataObj['data']
195 }
196 onlyForSectorsStocksObj["data"][ returnedDataObj['id'] ]["timeStamp"] = returnedDataObj['nseDate']
197 onlyForSectorsStocksObj["data"][ returnedDataObj['id'] ]["id"] = returnedDataObj['id']
198 onlyForSectorsStocksObj['dates'].push( returnedDataObj['nseDate'] )
199
200 /*
201 console.log(`Result for URL ${urls[index]}:`);
202 console.log(data[0]['sector']);
203 for(var i=0;i<data.length;i++ )
204 {
205 allSybmolsObj [ data[i]['symbol'] ] =1
206 }
207 */
208
209 }
210 });
211 //console.log( urls.length )
212 //console.log( "urlCount",urlCount)
213 //console.log( Object.keys(allSybmolsObj).length )
214 //console.log( allSybmolsObj )
215 //console.log("done");
216
217 console.log("onlyForSectorsStocksObj", onlyForSectorsStocksObj);
218 } catch (error) {
219 console.error('Error accessing fetched data:', error);
220 }
221}
222
223function formatDate(inputDateStr) {
224 // Parse the input date string into a Date object
225 var inputDate = new Date(inputDateStr);
226
227 // Define months in abbreviated form
228 var months = [
229 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
230 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'
231 ];
232
233 // Get the day, month, and year components from the Date object
234 var day = String(inputDate.getDate()).padStart(2, '0');
235 var month = months[inputDate.getMonth()];
236 var year = inputDate.getFullYear();
237
238 // Construct the desired output date string
239 var outputDateStr = `${day}-${month}-${year}`;
240
241 // Return the formatted date string
242 return outputDateStr;
243}
244
245function handleFetchedData( fetchedUrl, fetchedData )
246{
247
248
249 var dateOptions = {
250 timeZone: 'Asia/Kolkata',
251 year: 'numeric',
252 month: 'short', // Specify 'long' for full month name
253 day: '2-digit',
254};
255
256
257 //"https://iislliveblob.niftyindices.com/jsonfiles/HeatmapDetail/FinalHeatmapNIFTY%20NEXT%2050.json".split('FinalHeatmap')[1].replace('.json','').replace(/%20/g,' ')
258 var sectorName = fetchedUrl.split('FinalHeatmap')[1].replace('.json','').replace(/%20/g,' ').replace(/&/g,'&').replace(/&/g,'AND')
259 //var sectorName = fetchedUrl.split('FinalHeatmap')[1].replace('.json','').replace(/%20/g,' ')
260 var sectors_identifiers = {"NIFTY 50": {"index": "NIFTY 50", "indexSymbol": "NIFTY 50", "key": "BROAD MARKET INDICES" }, "NIFTY NEXT 50": {"index": "NIFTY NEXT 50", "indexSymbol": "NIFTY NEXT 50", "key": "BROAD MARKET INDICES" }, "NIFTY 100": {"index": "NIFTY 100", "indexSymbol": "NIFTY 100", "key": "BROAD MARKET INDICES" }, "NIFTY 200": {"index": "NIFTY 200", "indexSymbol": "NIFTY 200", "key": "BROAD MARKET INDICES" }, "NIFTY 500": {"index": "NIFTY 500", "indexSymbol": "NIFTY 500", "key": "BROAD MARKET INDICES" }, "NIFTY MIDCAP 50": {"index": "NIFTY MIDCAP 50", "indexSymbol": "NIFTY MIDCAP 50", "key": "BROAD MARKET INDICES" }, "NIFTY MIDCAP 100": {"index": "NIFTY MIDCAP 100", "indexSymbol": "NIFTY MIDCAP 100", "key": "BROAD MARKET INDICES" }, "NIFTY SMALLCAP 100": {"index": "NIFTY SMALLCAP 100", "indexSymbol": "NIFTY SMLCAP 100", "key": "BROAD MARKET INDICES" }, "INDIA VIX": {"index": "INDIA VIX", "indexSymbol": "INDIA VIX", "key": "BROAD MARKET INDICES" }, "NIFTY MIDCAP 150": {"index": "NIFTY MIDCAP 150", "indexSymbol": "NIFTY MIDCAP 150", "key": "BROAD MARKET INDICES" }, "NIFTY SMALLCAP 50": {"index": "NIFTY SMALLCAP 50", "indexSymbol": "NIFTY SMLCAP 50", "key": "BROAD MARKET INDICES" }, "NIFTY SMALLCAP 250": {"index": "NIFTY SMALLCAP 250", "indexSymbol": "NIFTY SMLCAP 250", "key": "BROAD MARKET INDICES" }, "NIFTY MIDSMALLCAP 400": {"index": "NIFTY MIDSMALLCAP 400", "indexSymbol": "NIFTY MIDSML 400", "key": "BROAD MARKET INDICES" }, "NIFTY500 MULTICAP 50:25:25": {"index": "NIFTY500 MULTICAP 50:25:25", "indexSymbol": "NIFTY500 MULTICAP", "key": "BROAD MARKET INDICES" }, "NIFTY LARGEMIDCAP 250": {"index": "NIFTY LARGEMIDCAP 250", "indexSymbol": "NIFTY LARGEMID250", "key": "BROAD MARKET INDICES" }, "NIFTY MIDCAP SELECT": {"index": "NIFTY MIDCAP SELECT", "indexSymbol": "NIFTY MID SELECT", "key": "BROAD MARKET INDICES" }, "NIFTY TOTAL MARKET": {"index": "NIFTY TOTAL MARKET", "indexSymbol": "NIFTY TOTAL MKT", "key": "BROAD MARKET INDICES" }, "NIFTY MICROCAP 250": {"index": "NIFTY MICROCAP 250", "indexSymbol": "NIFTY MICROCAP250", "key": "BROAD MARKET INDICES" }, "NIFTY BANK": {"index": "NIFTY BANK", "indexSymbol": "NIFTY BANK", "key": "SECTORAL INDICES" }, "NIFTY AUTO": {"index": "NIFTY AUTO", "indexSymbol": "NIFTY AUTO", "key": "SECTORAL INDICES" }, "NIFTY FINANCIAL SERVICES": {"index": "NIFTY FINANCIAL SERVICES", "indexSymbol": "NIFTY FIN SERVICE", "key": "SECTORAL INDICES" }, "NIFTY FINANCIAL SERVICES 25/50": {"index": "NIFTY FINANCIAL SERVICES 25/50", "indexSymbol": "NIFTY FINSRV25 50", "key": "SECTORAL INDICES" }, "NIFTY FMCG": {"index": "NIFTY FMCG", "indexSymbol": "NIFTY FMCG", "key": "SECTORAL INDICES" }, "NIFTY IT": {"index": "NIFTY IT", "indexSymbol": "NIFTY IT", "key": "SECTORAL INDICES" }, "NIFTY MEDIA": {"index": "NIFTY MEDIA", "indexSymbol": "NIFTY MEDIA", "key": "SECTORAL INDICES" }, "NIFTY METAL": {"index": "NIFTY METAL", "indexSymbol": "NIFTY METAL", "key": "SECTORAL INDICES" }, "NIFTY PHARMA": {"index": "NIFTY PHARMA", "indexSymbol": "NIFTY PHARMA", "key": "SECTORAL INDICES" }, "NIFTY PSU BANK": {"index": "NIFTY PSU BANK", "indexSymbol": "NIFTY PSU BANK", "key": "SECTORAL INDICES" }, "NIFTY PRIVATE BANK": {"index": "NIFTY PRIVATE BANK", "indexSymbol": "NIFTY PVT BANK", "key": "SECTORAL INDICES" }, "NIFTY REALTY": {"index": "NIFTY REALTY", "indexSymbol": "NIFTY REALTY", "key": "SECTORAL INDICES" }, "NIFTY HEALTHCARE INDEX": {"index": "NIFTY HEALTHCARE INDEX", "indexSymbol": "NIFTY HEALTHCARE", "key": "SECTORAL INDICES" }, "NIFTY CONSUMER DURABLES": {"index": "NIFTY CONSUMER DURABLES", "indexSymbol": "NIFTY CONSR DURBL", "key": "SECTORAL INDICES" }, "NIFTY OIL &GAS": {"index": "NIFTY OIL &GAS", "indexSymbol": "NIFTY OIL AND GAS", "key": "SECTORAL INDICES" },"NIFTY OIL & GAS": {"index": "NIFTY OIL &GAS", "indexSymbol": "NIFTY OIL AND GAS", "key": "SECTORAL INDICES" }, "NIFTY DIVIDEND OPPORTUNITIES 50": {"index": "NIFTY DIVIDEND OPPORTUNITIES 50", "indexSymbol": "NIFTY DIV OPPS 50", "key": "STRATEGY INDICES" }, "NIFTY GROWTH SECTORS 15": {"index": "NIFTY GROWTH SECTORS 15", "indexSymbol": "NIFTY GROWSECT 15", "key": "STRATEGY INDICES" }, "NIFTY100 QUALITY 30": {"index": "NIFTY100 QUALITY 30", "indexSymbol": "NIFTY100 QUALTY30", "key": "STRATEGY INDICES" }, "NIFTY50 VALUE 20": {"index": "NIFTY50 VALUE 20", "indexSymbol": "NIFTY50 VALUE 20", "key": "STRATEGY INDICES" }, "NIFTY50 TR 2X LEVERAGE": {"index": "NIFTY50 TR 2X LEVERAGE", "indexSymbol": "NIFTY50 TR 2X LEV", "key": "STRATEGY INDICES" }, "NIFTY50 PR 2X LEVERAGE": {"index": "NIFTY50 PR 2X LEVERAGE", "indexSymbol": "NIFTY50 PR 2X LEV", "key": "STRATEGY INDICES" }, "NIFTY50 TR 1X INVERSE": {"index": "NIFTY50 TR 1X INVERSE", "indexSymbol": "NIFTY50 TR 1X INV", "key": "STRATEGY INDICES" }, "NIFTY50 PR 1X INVERSE": {"index": "NIFTY50 PR 1X INVERSE", "indexSymbol": "NIFTY50 PR 1X INV", "key": "STRATEGY INDICES" }, "NIFTY50 DIVIDEND POINTS": {"index": "NIFTY50 DIVIDEND POINTS", "indexSymbol": "NIFTY50 DIV POINT", "key": "STRATEGY INDICES" }, "NIFTY ALPHA 50": {"index": "NIFTY ALPHA 50", "indexSymbol": "NIFTY ALPHA 50", "key": "STRATEGY INDICES" }, "NIFTY50 EQUAL WEIGHT": {"index": "NIFTY50 EQUAL WEIGHT", "indexSymbol": "NIFTY50 EQL WGT", "key": "STRATEGY INDICES" }, "NIFTY100 EQUAL WEIGHT": {"index": "NIFTY100 EQUAL WEIGHT", "indexSymbol": "NIFTY100 EQL WGT", "key": "STRATEGY INDICES" }, "NIFTY100 LOW VOLATILITY 30": {"index": "NIFTY100 LOW VOLATILITY 30", "indexSymbol": "NIFTY100 LOWVOL30", "key": "STRATEGY INDICES" }, "NIFTY200 QUALITY 30": {"index": "NIFTY200 QUALITY 30", "indexSymbol": "NIFTY200 QUALTY30", "key": "STRATEGY INDICES" }, "NIFTY ALPHA LOW-VOLATILITY 30": {"index": "NIFTY ALPHA LOW-VOLATILITY 30", "indexSymbol": "NIFTY ALPHALOWVOL", "key": "STRATEGY INDICES" }, "NIFTY200 MOMENTUM 30": {"index": "NIFTY200 MOMENTUM 30", "indexSymbol": "NIFTY200MOMENTM30", "key": "STRATEGY INDICES" }, "NIFTY MIDCAP150 QUALITY 50": {"index": "NIFTY MIDCAP150 QUALITY 50", "indexSymbol": "NIFTY M150 QLTY50", "key": "STRATEGY INDICES" }, "NIFTY COMMODITIES": {"index": "NIFTY COMMODITIES", "indexSymbol": "NIFTY COMMODITIES", "key": "THEMATIC INDICES" }, "NIFTY INDIA CONSUMPTION": {"index": "NIFTY INDIA CONSUMPTION", "indexSymbol": "NIFTY CONSUMPTION", "key": "THEMATIC INDICES" }, "NIFTY CPSE": {"index": "NIFTY CPSE", "indexSymbol": "NIFTY CPSE", "key": "THEMATIC INDICES" }, "NIFTY ENERGY": {"index": "NIFTY ENERGY", "indexSymbol": "NIFTY ENERGY", "key": "THEMATIC INDICES" }, "NIFTY INFRASTRUCTURE": {"index": "NIFTY INFRASTRUCTURE", "indexSymbol": "NIFTY INFRA", "key": "THEMATIC INDICES" }, "NIFTY100 LIQUID 15": {"index": "NIFTY100 LIQUID 15", "indexSymbol": "NIFTY100 LIQ 15", "key": "THEMATIC INDICES" }, "NIFTY MIDCAP LIQUID 15": {"index": "NIFTY MIDCAP LIQUID 15", "indexSymbol": "NIFTY MID LIQ 15", "key": "THEMATIC INDICES" }, "NIFTY MNC": {"index": "NIFTY MNC", "indexSymbol": "NIFTY MNC", "key": "THEMATIC INDICES" }, "NIFTY PSE": {"index": "NIFTY PSE", "indexSymbol": "NIFTY PSE", "key": "THEMATIC INDICES" }, "NIFTY SERVICES SECTOR": {"index": "NIFTY SERVICES SECTOR", "indexSymbol": "NIFTY SERV SECTOR", "key": "THEMATIC INDICES" }, "NIFTY100 ESG SECTOR LEADERS": {"index": "NIFTY100 ESG SECTOR LEADERS", "indexSymbol": "NIFTY100ESGSECLDR", "key": "THEMATIC INDICES" }, "NIFTY INDIA DIGITAL": {"index": "NIFTY INDIA DIGITAL", "indexSymbol": "NIFTY IND DIGITAL", "key": "THEMATIC INDICES" }, "NIFTY100 ESG": {"index": "NIFTY100 ESG", "indexSymbol": "NIFTY100 ESG", "key": "THEMATIC INDICES" }, "NIFTY INDIA MANUFACTURING": {"index": "NIFTY INDIA MANUFACTURING", "indexSymbol": "NIFTY INDIA MFG", "key": "THEMATIC INDICES" }, "NIFTY 8-13 YR G-SEC": {"index": "NIFTY 8-13 YR G-SEC", "indexSymbol": "NIFTY GS 8 13YR", "key": "FIXED INCOME INDICES" }, "NIFTY 10 YR BENCHMARK G-SEC": {"index": "NIFTY 10 YR BENCHMARK G-SEC", "indexSymbol": "NIFTY GS 10YR", "key": "FIXED INCOME INDICES" }, "NIFTY 10 YR BENCHMARK G-SEC (CLEAN PRICE)": {"index": "NIFTY 10 YR BENCHMARK G-SEC (CLEAN PRICE)", "indexSymbol": "NIFTY GS 10YR CLN", "key": "FIXED INCOME INDICES" }, "NIFTY 4-8 YR G-SEC INDEX": {"index": "NIFTY 4-8 YR G-SEC INDEX", "indexSymbol": "NIFTY GS 4 8YR", "key": "FIXED INCOME INDICES" }, "NIFTY 11-15 YR G-SEC INDEX": {"index": "NIFTY 11-15 YR G-SEC INDEX", "indexSymbol": "NIFTY GS 11 15YR", "key": "FIXED INCOME INDICES" }, "NIFTY 15 YR AND ABOVE G-SEC INDEX": {"index": "NIFTY 15 YR AND ABOVE G-SEC INDEX", "indexSymbol": "NIFTY GS 15YRPLUS", "key": "FIXED INCOME INDICES" }, "NIFTY COMPOSITE G-SEC INDEX": {"index": "NIFTY COMPOSITE G-SEC INDEX", "indexSymbol": "NIFTY GS COMPSITE", "key": "FIXED INCOME INDICES" }}
261
262 var sectorIdentifier = sectorName
263 if( sectors_identifiers[sectorName] !=undefined )
264 {
265 sectorIdentifier = sectors_identifiers[sectorName]["indexSymbol"]
266 }
267 sectorIdentifier = sectorIdentifier.replace(/ /g,'_')
268 var indexDivisor = fetchedData[0]['indexDivisor']
269
270 //"06-Sep-2023 09:07:31"
271 //var formattedDate = new Date( fetchedData[0]['time'] ).toLocaleString('en-IN', dateOptions)
272 var formattedDate = formatDate( fetchedData[0]['time'] )
273 //var formattedTime = new Date( fetchedData[0]['time'] ).toLocaleString('en-IN', options).split(' ')[1]
274 var nseDate = formattedDate
275
276
277
278
279 var sector_stocks_obj = {
280 "sectorName":sectorName,
281 "indexDivisor":indexDivisor,
282 "data":{},
283 }
284
285 for(var i=0;i<fetchedData.length;i++ )
286 {
287 var dataObj = fetchedData[i]
288 var symbol = dataObj['symbol']
289
290 sector_stocks_obj["data"][symbol] ={
291 Indexmcap_yst : dataObj['Indexmcap_yst'],
292 cappingFactor : dataObj['cappingFactor'],
293 investableWeightFactor : dataObj['investableWeightFactor'],
294 sharesOutstanding : dataObj['sharesOutstanding'],
295 sector : dataObj['sector'],
296 }
297 }
298
299
300
301
302
303 //console.log( sectorName,"=",sectorIdentifier, nseDate, indexDivisor )
304
305 var id = 'sector_stockNames' + '/' + sectorIdentifier;
306 var returnableObj = {
307 "data":sector_stocks_obj,
308 "nseDate":nseDate,
309 "id":id }
310 return returnableObj
311}
312
313async function addExtractedSectorStockNamesData( db, createdAtTime, currentSystemDate )
314{
315 try { // code that might throw an exception
316
317 var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
318
319 var ApiMainObject = {};
320 var returnedDataArray ;
321 var pathToStoreData;
322 var pathOfDocument;
323 var dataObj;
324
325 var options = {
326 timeZone: 'Asia/Kolkata',
327 month: 'long',
328 year: 'numeric',
329 };
330
331 var latestDate;
332
333 //var dates = ['2022-01-01', '2023-01-01', '2023-04-30'];
334 var dates = onlyForSectorsStocksObj['dates'];
335 var newestDate = new Date(dates[0]);
336
337 if (dates.length > 1) {
338 for (var i = 1; i < dates.length; i++) {
339 var currentDate = new Date(dates[i]);
340 if (currentDate > newestDate) {
341 newestDate = currentDate;
342 }
343 }
344 }
345
346
347 function convertIntoSeconds(time) {
348 var ts = time.split(':');
349 return Date.UTC(1970, 0, 1, ts[0], ts[1], ts[2]) / 1000;
350 }
351
352
353 var timeOptions = {
354 timeZone: 'Asia/Kolkata',
355 year: 'numeric',
356 month: '2-digit',
357 day: '2-digit',
358 hour: '2-digit',
359 minute: '2-digit',
360 second: '2-digit',
361 hour12: false
362 };
363
364 var currentTime = new Date().toLocaleString('en-IN', timeOptions);
365 var currentTimeHMS = new Date().toLocaleString('en-IN', timeOptions).split(' ')[1];
366
367 if ( convertIntoSeconds(currentTimeHMS) > convertIntoSeconds('09:00:00') && convertIntoSeconds(currentTimeHMS) <= convertIntoSeconds('09:15:00') )
368 { // if condition is true then assign current data as newestDate
369 newestDate = new Date();
370 }
371 //var options2 = { timeZone: 'Asia/Kolkata', year: 'numeric', month: 'short', day: 'numeric' };
372 //var formattedDate = newestDate.toLocaleDateString('en-IN', options2).replace(/ /g, '-');
373 //latestDate = formattedDate;
374 var dayOfMonth = String(newestDate.getDate());
375 if (dayOfMonth.length < 2) {
376 dayOfMonth = '0' + dayOfMonth;
377 }
378
379 latestDate = dayOfMonth +'-'+ months[newestDate.getMonth()] +'-'+ newestDate.getFullYear();
380 //console.log('Newest date:', latestDate);
381 //console.log("onlyForSectorsStocksObj",onlyForSectorsStocksObj)
382 //console.log("latestDate",latestDate, "currentSystemDate",currentSystemDate)
383 if( Date.parse(latestDate) == Date.parse(currentSystemDate) || convertIntoSeconds(currentTimeHMS) > convertIntoSeconds('09:00:00') && convertIntoSeconds(currentTimeHMS) <= convertIntoSeconds('09:15:00') )
384 {
385 //April 2023/26-Apr-2023/sector_stockNames/NIFTY_BANK/22:21:16 has
386 var monthYear = new Date( latestDate ).toLocaleString('en-IN', options);
387 for( var SectorsStocks in onlyForSectorsStocksObj['data'] )
388 {
389 try { // code that might throw an exception
390
391 //var pathOfDocument = returnedDataArray[2] + '/FnO/' +createdAtTime+ '/' +createdAtTime;
392
393 pathOfDocument = onlyForSectorsStocksObj['data'][SectorsStocks]['id']
394 //console.log("pathOfDocument",pathOfDocument)
395 pathToStoreData = monthYear +'/'+ latestDate+ '/' + pathOfDocument ;
396
397 dataObj={
398 "data": JSON.stringify( onlyForSectorsStocksObj['data'][SectorsStocks]['data'] )
399 }
400 console.log( JSON.stringify( onlyForSectorsStocksObj['data'][SectorsStocks]['data'] ).length )
401 console.log( pathToStoreData )
402 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
403 await db.doc( pathToStoreData ).set( dataObj );
404
405 } catch(error) {
406 // handle the error
407 console.log(error);
408 }
409 }
410 }
411
412 } catch(error) {
413 // handle the error
414 console.log(error);
415 }
416}
417
418
419
420 var options = {
421 timeZone: 'Asia/Kolkata',
422 year: 'numeric',
423 month: '2-digit',
424 day: '2-digit',
425 hour: '2-digit',
426 minute: '2-digit',
427 second: '2-digit' ,
428 hour12: false
429 };
430
431 var allSybmolsObj = {}
432 var urlCount = 0
433 var fetchedDataArray;
434var onlyForSectorsStocksObj= {
435 dates:[],
436 data:{}
437};
438
439
440Apify.main(async () => {
441 try {
442
443const input = await Apify.getInput();
444
445 const { data } = input;
446 let islegacyPhantomJSTask = false;
447 if (data && typeof data === 'string') {
448 // It runs from legacy phantomjs crawler task finished webhook
449 const legacyInput = JSON.parse(data);
450 Object.assign(input, legacyInput);
451 islegacyPhantomJSTask = true;
452 }
453
454 const { datasetId, apiKey, authDomain, projectId, collectionName } = input;
455 //console.log("input", input)
456
457 if (!datasetId) {
458 throw new Error('DatasetId is required on input.');
459 }
460
461 firebase.initializeApp({
462 apiKey,
463 authDomain,
464 projectId,
465 });
466
467 //console.log("after calling startRunningActorProcess function");
468 //console.log( Object.keys(allSybmolsObj).length )
469 //console.log( allSybmolsObj )
470 //console.log("done");
471
472
473
474 // Initialize Cloud Firestore through Firebase
475 var db = firebase.firestore();
476
477
478
479 var dateOptions = {
480 timeZone: 'Asia/Kolkata',
481 year: 'numeric',
482 month: 'long', // Specify 'long' for full month name
483 day: '2-digit',
484};
485
486 //var createdAtTime = new Date( datasetInfo["createdAt"] ).toTimeString('en-IN', options).split(' ')[0]
487 var currentSystemDate = new Date().toLocaleString('en-IN', dateOptions)
488 var createdAtTime = new Date().toLocaleString('en-IN', options).split(' ')[1]
489
490 createdAtTime = createdAtTime.split(':')[0] + ':' + createdAtTime.split(':')[1]+ ':00'
491
492 console.log("currentSystemDate", currentSystemDate, "createdAtTime",createdAtTime)
493
494 // console.log("calling startRunningActorProcess function");
495 await startFetchingNseIndicesData();
496 await addExtractedSectorStockNamesData( db, createdAtTime, currentSystemDate )
497
498 } catch (err) {
499 console.error(`Error while running main: ${err.message}`);
500 }
501});
502main_ORIGINAL.js
1const Apify = require('apify');
2const firebase = require('firebase');
3
4function TestFunction(myurl,createdAtTime)
5{
6 var myPath= "AprilTest2023/25Apr2023/"+myurl+"/"+createdAtTime;
7 return myPath;
8}
9
10Apify.main(async () => {
11 const input = await Apify.getInput();
12
13 const { data } = input;
14 let islegacyPhantomJSTask = false;
15 if (data && typeof data === 'string') {
16 // It runs from legacy phantomjs crawler task finished webhook
17 const legacyInput = JSON.parse(data);
18 Object.assign(input, legacyInput);
19 islegacyPhantomJSTask = true;
20 }
21
22 const { datasetId, apiKey, authDomain, projectId, collectionName } = input;
23
24 if (!datasetId) {
25 throw new Error('DatasetId is required on input.');
26 }
27
28 firebase.initializeApp({
29 apiKey,
30 authDomain,
31 projectId,
32 });
33
34 // Initialize Cloud Firestore through Firebase
35 const db = firebase.firestore();
36 console.log(`Start importing dataset ${datasetId} to firestore.`);
37 const dataset = await Apify.openDataset(datasetId, { forceCloud: true });
38 const datasetInfo = await dataset.getInfo();
39
40 //console.log('datasetInfo:', datasetInfo);
41 const createdAt = new Date( datasetInfo["createdAt"] )
42
43 var options = {
44 timeZone: 'Asia/Kolkata',
45 year: 'numeric',
46 month: '2-digit',
47 day: '2-digit',
48 hour: '2-digit',
49 minute: '2-digit',
50 second: '2-digit' ,
51 hour12: false
52 };
53
54 //var createdAtTime = new Date( datasetInfo["createdAt"] ).toTimeString('en-IN', options).split(' ')[0]
55 var createdAtTime = new Date( datasetInfo["createdAt"] ).toLocaleString('en-IN', options).split(' ')[1]
56
57 // Import dataset from actor/task
58 const limit = 1000;
59 let counter = 0;
60 for (let offset = 0; offset < datasetInfo.itemCount; offset += limit) {
61 const pagination = await dataset.getData({
62 simplified: islegacyPhantomJSTask,
63 clean: !islegacyPhantomJSTask,
64 limit,
65 offset,
66 });
67 console.log(`Get dataset items offset: ${pagination.offset}`);
68 for (const item of pagination.items) {
69 var myurl = (item["url"].replace(/[^\w\s]/gi, ""));
70 console.log('item:', myurl );
71 console.log('createdAt:', createdAt );
72
73 try {
74 if( item["pageText"]!="" )
75 {
76 //var myPath= "March2022/10Mar2023/"+myurl+"/"+createdAtTime;
77 var myPath = TestFunction(myurl,createdAtTime);
78 //await db.collection(collectionName).doc( myurl ).set( JSON.parse(item["pageText"]) ); // Use doc() and set() instead of add()
79 //await db.doc( myPath ).set( JSON.parse(item["pageText"]) ); // Use doc() and set() instead of add()
80 var dataObj={
81 "objectValue":(item["pageText"])
82 }
83 await db.doc( myPath ).set( dataObj ); // Use doc() and set() instead of add()
84 counter++;
85 //console.log('Added item:', item.replace(/[^\w\s]/gi, ''));
86
87 }
88
89 } catch (err) {
90 //console.log(`Cannot import item ${JSON.stringify(item)}: ${err.message}`);
91 console.log(`Cannot import item ${myurl}: ${err.message}`);
92 }
93 }
94 }
95
96 console.log(`Imported ${counter} from dataset ${datasetId}.`);
97
98 console.log('Done!');
99});
100package.json
1{
2 "name": "firestore-import",
3 "version": "0.0.1",
4 "description": "This is a boilerplate of an Apify actor.",
5 "dependencies": {
6 "apify": "^0.16.0",
7 "firebase": "^7.2.0"
8 },
9 "devDependencies": {
10 "@apify/eslint-config": "0.0.3",
11 "eslint": "^6.5.1"
12 },
13 "scripts": {
14 "start": "node main.js",
15 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
16 },
17 "author": "It's not you it's me",
18 "license": "ISC"
19}
20
Org 1
Maintained by Community
Build new tools
Enterprise solutions