This Actor is under maintenance.
This actor is under maintenance and it may unreliable.
.eslintrc
1{
2 "extends": "@apify"
3}
4.gitignore
1apify_storage
2node_modules.npmignore
1# This file tells Git which files shouldn't be added to source control
2
3.idea
4node_modules
5Dockerfile
1# Dockerfile contains instructions how to build a Docker image that will contain
2# all the code and configuration needed to run your actor. For a full
3# Dockerfile reference, see https://docs.docker.com/engine/reference/builder/
4
5# First, specify the base Docker image. Apify provides the following base images
6# for your convenience:
7# apify/actor-node-basic (Node.js 10 on Alpine Linux, small and fast image)
8# apify/actor-node-chrome (Node.js 10 + Chrome on Debian)
9# apify/actor-node-chrome-xvfb (Node.js 10 + Chrome + Xvfb on Debian)
10# For more information, see https://apify.com/docs/actor#base-images
11# Note that you can use any other image from Docker Hub.
12FROM apify/actor-node-basic
13
14# Second, copy just package.json and package-lock.json since they are the only files
15# that affect NPM install in the next step
16COPY package*.json ./
17
18# Install NPM packages, skip optional and development dependencies to keep the
19# image small. Avoid logging too much and print the dependency tree for debugging
20RUN npm --quiet set progress=false \
21 && npm install --only=prod --no-optional \
22 && echo "Installed NPM packages:" \
23 && npm list \
24 && echo "Node.js version:" \
25 && node --version \
26 && echo "NPM version:" \
27 && npm --version
28
29# Next, copy the remaining files and directories with the source code.
30# Since we do this after NPM install, quick build will be really fast
31# for simple source file changes.
32COPY . ./
33
34# Specify how to run the source code
35CMD npm start
36INPUT_SCHEMA.json
1{
2 "title": "Firestore Import input",
3 "description": "Imports dataset to Firestore DB",
4 "type": "object",
5 "schemaVersion": 1,
6 "properties": {
7 "datasetId": {
8 "title": "Dataset",
9 "type": "string",
10 "description": "Dataset ID of dataset you want to import to Firestore",
11 "editor": "textfield"
12 },
13 "apiKey": {
14 "title": "Api key",
15 "type": "string",
16 "description": "Firestore API key",
17 "editor": "textfield"
18 },
19 "authDomain": {
20 "title": "Auth domain",
21 "type": "string",
22 "description": "Firestore authentication domain",
23 "editor": "textfield"
24 },
25 "projectId": {
26 "title": "Project ID",
27 "type": "string",
28 "description": "Firestore project ID",
29 "editor": "textfield"
30 },
31 "collectionName": {
32 "title": "Collection name",
33 "type": "string",
34 "description": "Firestore collection name",
35 "editor": "textfield"
36 }
37 }
38}
39INPUT_SCHEMA_Original.json
1{
2 "title": "Firestore Import input",
3 "description": "Imports dataset to Firestore DB",
4 "type": "object",
5 "schemaVersion": 1,
6 "properties": {
7 "datasetId": {
8 "title": "Dataset",
9 "type": "string",
10 "description": "Dataset ID of dataset you want to import to Firestore",
11 "editor": "textfield"
12 },
13 "apiKey": {
14 "title": "Api key",
15 "type": "string",
16 "description": "Firestore API key",
17 "editor": "textfield"
18 },
19 "authDomain": {
20 "title": "Auth domain",
21 "type": "string",
22 "description": "Firestore authentication domain",
23 "editor": "textfield"
24 },
25 "projectId": {
26 "title": "Project ID",
27 "type": "string",
28 "description": "Firestore project ID",
29 "editor": "textfield"
30 },
31 "collectionName": {
32 "title": "Collection name",
33 "type": "string",
34 "description": "Firestore collection name",
35 "editor": "textfield"
36 }
37 },
38 "required": ["datasetId", "apiKey", "authDomain", "projectId", "collectionName"]
39}
40apify.json
1{
2 "name": "firestore-import",
3 "version": "0.0",
4 "buildTag": "latest",
5 "env": null,
6 "template": "hello_world"
7}
8main saving 1-By-1.js
1const Apify = require('apify');
2const firebase = require('firebase');
3
4
5 function extractTradingDates(apiObject, iframeSrc){ // this function extracts previous trading date and then save it into IndexedDB
6
7 var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
8 var date = new Date( apiObject[1]['marketCurrentTradingDate'] )
9 var dayOfMonth = String(date.getDate());
10 if (dayOfMonth.length < 2) {
11 dayOfMonth = '0' + dayOfMonth;
12 }
13 var currentTradingDate = dayOfMonth +'-'+ months[date.getMonth()] +'-'+ date.getFullYear()
14 date =new Date( apiObject[1]['marketPreviousTradingDate'] )
15 var dayOfMonth = String(date.getDate());
16 if (dayOfMonth.length < 2) {
17 dayOfMonth = '0' + dayOfMonth;
18 }
19 var previousTradingDate = dayOfMonth +'-'+ months[date.getMonth()] +'-'+ date.getFullYear()
20
21 var nextTradingDate = apiObject[1]['marketNextTradingDate']
22
23 //var tradingDates={ "marketCurrentTradingDate":currentTradingDate, "marketPreviousTradingDate":previousTradingDate, "marketNextTradingDate":nextTradingDate }
24 var tradingDates = { "currentTradingDate":currentTradingDate, "previousTradingDate":previousTradingDate, "nextTradingDate":nextTradingDate }
25
26 var timeStamp = currentTradingDate;
27
28 var id = "trading_dates";
29
30 return [ tradingDates, timeStamp, id ]
31
32
33 } // extractTradingDates(apiObject) function ENDS HERE
34
35
36
37var allStocksFuturesOptionsDataObj ={}
38var addedQuoteDerivatives = 0
39function extractQuoteDerivativesData(apiObject, iframeSrc)
40 {
41
42 var ApiMainObject = {};
43 var timeStamp = apiObject['opt_timestamp'].split(' ')[0]
44
45var templateObj = {
46"symbol":{
47//"identifier":[ 'openPrice', 'highPrice','lowPrice', 'lastPrice', 'change', 'pChange', 'prevClose', 'numberOfContractsTraded', 'totalTurnover',['tradeInfo -> openInterest', 'changeinOpenInterest', 'pchangeinOpenInterest', 'premiumTurnover', 'tradedVolume', 'value', 'vmap', 'impliedVolatility'], ['marketDeptOrderBook -> totalBuyQuantity', 'totalSellQuantity', [ 'ask -> price : quantity'], [ 'bid -> price : quantity'], ['carryOfCost -> price of bestBuy', 'price of bestSell','price of lastPrice', 'carry of bestBuy', 'carry of bestSell', 'carry of lastPrice'] ] ],
48"identifier":[ 'openPrice', 'highPrice','lowPrice', 'lastPrice', 'change', 'pChange', 'prevClose', 'numberOfContractsTraded/tradedVolume', 'totalTurnover/value',['tradeInfo -> openInterest', 'changeinOpenInterest', 'pchangeinOpenInterest', 'premiumTurnover', 'vmap', 'impliedVolatility'], ['marketDeptOrderBook -> totalBuyQuantity', 'totalSellQuantity', [ 'ask -> price : quantity'], [ 'bid -> price : quantity'], ['carryOfCost -> price of bestBuy', 'price of bestSell','price of lastPrice', 'carry of bestBuy', 'carry of bestSell', 'carry of lastPrice'] ] ],
49'underlyingValue' : 'underlyingValue',
50'marketLot':'marketLot',
51'fut_timestamp': 'fut_timestamp',
52'opt_timestamp': 'opt_timestamp',
53
54}
55
56}
57
58var allSymbolArray = apiObject['allSymbol'].sort()
59var symbol = apiObject['info']['symbol']
60var stockFuturesOptionsDataObj = {
61 [symbol] : {}
62}
63
64for( var i=0;i<apiObject['stocks'].length;i++)
65{
66 var dataObj = apiObject['stocks'][i]
67
68 var tradeInfo = [
69 dataObj ['marketDeptOrderBook']['tradeInfo']['openInterest'],
70 dataObj ['marketDeptOrderBook']['tradeInfo']['changeinOpenInterest'],
71 dataObj ['marketDeptOrderBook']['tradeInfo']['pchangeinOpenInterest'],
72 dataObj ['marketDeptOrderBook']['tradeInfo']['premiumTurnover'],
73 //dataObj ['marketDeptOrderBook']['tradeInfo']['tradedVolume'],
74 //dataObj ['marketDeptOrderBook']['tradeInfo']['value'],
75 dataObj ['marketDeptOrderBook']['tradeInfo']['vmap'],
76 dataObj ['marketDeptOrderBook']['otherInfo']['impliedVolatility'],
77 ]
78
79 var askArray = [], bidArray = []
80 for(var a=0;a< dataObj ['marketDeptOrderBook']['ask'].length;a++ )
81 {
82 var data = dataObj ['marketDeptOrderBook']['ask'][a]
83 askArray.push( data['price'] +':'+data['quantity'] )
84 }
85
86 for(var b=0;b< dataObj ['marketDeptOrderBook']['bid'].length;b++ )
87 {
88 var data = dataObj ['marketDeptOrderBook']['bid'][b]
89 bidArray.push( data['price'] +':'+data['quantity'] )
90 }
91
92 var carryOfCostArray = [
93 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['bestBuy'],
94 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['bestSell'],
95 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['lastPrice'],
96 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['bestBuy'],
97 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['bestSell'],
98 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['lastPrice'],
99 ]
100
101 var marketDeptOrderBook = [
102 dataObj ['marketDeptOrderBook']['totalBuyQuantity'],
103 dataObj['marketDeptOrderBook']['totalSellQuantity'],
104 askArray,
105 bidArray,
106 carryOfCostArray,
107 ]
108 /*
109 var marketDeptOrderBook = {
110 'ask' : dataObj ['marketDeptOrderBook']['ask'],
111 'bid' : dataObj ['marketDeptOrderBook']['bid'],
112
113 }
114 */
115
116 var dataArray = [
117
118
119 //dataObj ['metadata']['closePrice'] ,
120 dataObj ['metadata']['openPrice'] ,
121 dataObj ['metadata']['highPrice'] ,
122 dataObj ['metadata']['lowPrice'] ,
123 dataObj ['metadata']['lastPrice'] ,
124 dataObj ['metadata']['change'] ,
125 dataObj ['metadata']['pChange'] ,
126 dataObj ['metadata']['prevClose'] ,
127 dataObj ['metadata']['numberOfContractsTraded'] ,
128 dataObj ['metadata']['totalTurnover'] ,
129 tradeInfo,
130 marketDeptOrderBook
131
132 ]
133
134 var commonDataArray = [ ]
135 if( dataObj['metadata']['numberOfContractsTraded']>0 )
136 {
137 var identifier = dataObj['metadata']['identifier'].replace(symbol,'')
138
139
140 stockFuturesOptionsDataObj[symbol][identifier] = dataArray
141
142 stockFuturesOptionsDataObj[symbol]['underlyingValue'] = dataObj ['underlyingValue']
143 stockFuturesOptionsDataObj[symbol]['marketLot'] = dataObj ['marketDeptOrderBook']['tradeInfo']['marketLot']
144 stockFuturesOptionsDataObj[symbol]['fut_timestamp'] = apiObject['fut_timestamp']
145 stockFuturesOptionsDataObj[symbol]['opt_timestamp'] = apiObject['opt_timestamp']
146
147 }
148
149}
150
151console.log(apiObject['stocks'].length)
152console.log(Object.keys(stockFuturesOptionsDataObj[symbol]).length)
153console.log( stockFuturesOptionsDataObj )
154
155var key = Object.keys(stockFuturesOptionsDataObj)[0]
156var valueOfKey = stockFuturesOptionsDataObj[key]
157
158//allStocksFuturesOptionsDataObj[ key ] = valueOfKey
159
160
161
162
163 var id = "all_derivatives"
164
165 //return [ stockFuturesOptionsDataObj, timeStamp, id, key ]
166
167 return [ valueOfKey, timeStamp, id, key, templateObj, allSymbolArray ]
168 }
169
170function extractQuoteDerivativesData_previous(apiObject, iframeSrc)
171 {
172
173 var ApiMainObject = {};
174 var timeStamp = apiObject['opt_timestamp'].split(' ')[0]
175
176 var symbol = apiObject['info']['symbol']
177 var stockFuturesOptionsDataObj = {
178 [symbol] : {}
179 }
180
181 for( var i=0;i<apiObject['stocks'].length;i++)
182 {
183 var dataObj = apiObject['stocks'][i]
184
185 var tradeInfo = [
186 dataObj ['marketDeptOrderBook']['tradeInfo']['changeinOpenInterest'],
187 dataObj ['marketDeptOrderBook']['tradeInfo']['openInterest'],
188 dataObj ['marketDeptOrderBook']['tradeInfo']['pchangeinOpenInterest'],
189 dataObj ['marketDeptOrderBook']['tradeInfo']['premiumTurnover'],
190 dataObj ['marketDeptOrderBook']['tradeInfo']['tradedVolume'],
191 dataObj ['marketDeptOrderBook']['tradeInfo']['value'],
192 dataObj ['marketDeptOrderBook']['tradeInfo']['vmap'],
193
194 ]
195
196 var askArray = [], bidArray = []
197 for(var a=0;a< dataObj ['marketDeptOrderBook']['ask'].length;a++ )
198 {
199 var data = dataObj ['marketDeptOrderBook']['ask'][a]
200 askArray.push( data['price'] +':'+data['quantity'] )
201 }
202
203 for(var b=0;b< dataObj ['marketDeptOrderBook']['bid'].length;b++ )
204 {
205 var data = dataObj ['marketDeptOrderBook']['bid'][b]
206 bidArray.push( data['price'] +':'+data['quantity'] )
207 }
208
209 var marketDeptOrderBook = [ dataObj ['marketDeptOrderBook']['otherInfo']['impliedVolatility'], dataObj ['marketDeptOrderBook']['totalBuyQuantity'], dataObj['marketDeptOrderBook']['totalSellQuantity'], askArray, bidArray ]
210 /*
211 var marketDeptOrderBook = {
212 'ask' : dataObj ['marketDeptOrderBook']['ask'],
213 'bid' : dataObj ['marketDeptOrderBook']['bid'],
214
215 }
216 */
217
218 var dataArray = [
219
220 dataObj ['metadata']['change'] ,
221 //dataObj ['metadata']['closePrice'] ,
222 dataObj ['metadata']['highPrice'] ,
223 dataObj ['metadata']['lastPrice'] ,
224 dataObj ['metadata']['lowPrice'] ,
225 dataObj ['metadata']['numberOfContractsTraded'] ,
226 dataObj ['metadata']['openPrice'] ,
227 dataObj ['metadata']['pChange'] ,
228 dataObj ['metadata']['prevClose'] ,
229 dataObj ['metadata']['totalTurnover'] ,
230 tradeInfo,
231 marketDeptOrderBook
232
233 ]
234
235 var commonDataArray = [ ]
236 if( dataObj['metadata']['numberOfContractsTraded']>1 )
237 {
238 var identifier = dataObj['metadata']['identifier'].replace(symbol,'')
239
240
241 stockFuturesOptionsDataObj[symbol][identifier] = dataArray
242
243 stockFuturesOptionsDataObj[symbol]['underlyingValue'] = dataObj ['underlyingValue']
244 stockFuturesOptionsDataObj[symbol]['marketLot'] = dataObj ['marketDeptOrderBook']['tradeInfo']['marketLot']
245
246 }
247
248 }
249
250 //console.log(apiObject['stocks'].length)
251 //console.log(Object.keys(stockFuturesOptionsDataObj[symbol]).length)
252 //console.log( stockFuturesOptionsDataObj )
253
254 var key = Object.keys(stockFuturesOptionsDataObj)[0]
255 var valueOfKey = stockFuturesOptionsDataObj[key]
256 //allStocksFuturesOptionsDataObj[ key ] = valueOfKey
257
258
259 var id = "all_derivatives"
260
261 //return [ stockFuturesOptionsDataObj, timeStamp, id, key ]
262
263 return [ stockFuturesOptionsDataObj, timeStamp, id, key ]
264 }
265
266 async function handleData(apiObject, iframeSrc, db, createdAtTime, currentSystemDate) { // this function first handles localStorages for current NSE_APIs_Data, PreviousDay_NSE_APIs_Data. and then redirect iframe's JSON object to specified function according to iframe src
267
268 var ApiMainObject = {};
269 var returnedDataArray ;
270 var pathToStoreData;
271 var pathOfDocument;
272 var dataObj;
273
274 var options = {
275 timeZone: 'Asia/Kolkata',
276 month: 'long',
277 year: 'numeric',
278 };
279
280 if ( iframeSrc.includes("api/quote-derivative?symbol") == true ) {
281 returnedDataArray = extractQuoteDerivativesData(apiObject)
282 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime+ '/' + createdAtTime+ '/' +returnedDataArray[3];
283 }
284
285
286 //April 2023/26-Apr-2023/sector_stockNames/NIFTY_BANK/22:21:16 has
287 var monthYear = new Date( returnedDataArray[1] ).toLocaleString('en-IN', options);
288 //pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + returnedDataArray[2] + '/' +createdAtTime;
289
290 if ( iframeSrc.includes("api/allMarketStatus") == true ) {
291 pathToStoreData = monthYear +'/'+ returnedDataArray[1]
292
293 dataObj={
294 //[ returnedDataArray[2] ]: JSON.stringify(returnedDataArray[0])
295 [ returnedDataArray[2] ]: (returnedDataArray[0])
296 }
297 }
298 else
299 {
300 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
301
302 dataObj={
303 "data": JSON.stringify(returnedDataArray[0])
304 }
305 }
306
307
308 console.log( JSON.stringify(returnedDataArray[0]).length )
309
310 var nseDate = pathToStoreData.split(/\//)[1]
311
312 if( Date.parse(nseDate) == Date.parse(currentSystemDate) || iframeSrc.includes("#OnlyForSectorsStocks") == true )
313 {
314 if ( iframeSrc.includes("api/allMarketStatus") == true )
315 {
316 //await db.doc( pathToStoreData ).set( dataObj );
317 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
318 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
319 }
320 else if ( iframeSrc.includes("api/quote-derivative?symbol") == true )
321 {
322 //await db.doc( pathToStoreData ).set( dataObj );
323 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
324 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
325
326 if(addedQuoteDerivatives==0)
327 {
328 /*
329 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime+ '/' + createdAtTime+ '/' +'Template';
330 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
331
332 dataObj={
333 "data": ['open','high','low','close']
334 }
335
336 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
337 */
338
339
340 //updating index
341
342
343 var id = returnedDataArray[2]
344 var symbol = returnedDataArray[3]
345 var templateObj = returnedDataArray[4]
346 var allSymbolArray = returnedDataArray[5]
347
348 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime;
349 //pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
350 pathToStoreData = monthYear +'/'+ returnedDataArray[1]
351
352
353
354
355 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
356 var firstNestedFieldName = id+ '_index'
357 var secondNestedFieldName = 'time_values'
358 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
359 var fieldPath = firstNestedFieldName+'.'+ secondNestedFieldName
360 var templateFieldPath = firstNestedFieldName+'.'+ 'template'
361 var allSymbolsFieldPath = firstNestedFieldName+'.'+ 'all_symbols'
362 //var fieldPath = 'totalCEPEOIData.time.test'
363
364
365 //var totalCEPEData = {};
366 //totalCEPEData = returnedDataArray[0] // option chain all data
367 //totalCEPEData = JSON.stringify(returnedDataArray[0])// option chain all data
368 var totalCEPEData = createdAtTime// option chain all data
369
370
371 //totalCEPEData = symbol // option chain all data
372
373 //console.log("pathToStoreData", pathToStoreData)
374 //console.log("fieldPath", fieldPath)
375 //console.log("totalCEPEData", totalCEPEData)
376
377 var intraday_chart_data_Ref = db.doc( pathToStoreData );
378
379
380 // Try to update the existing document using the update() method
381 await intraday_chart_data_Ref.update({
382 [fieldPath]: firebase.firestore.FieldValue.arrayUnion(totalCEPEData),
383 [templateFieldPath]: JSON.stringify(templateObj),
384 [allSymbolsFieldPath]: allSymbolArray
385 }).then(() => {
386 console.log("Document updated successfully.");
387 }).catch((error) => {
388 // If the update() method fails with a "document does not exist" error,
389 // create a new document using the set() method instead
390 if (error.code === "not-found") {
391
392 var updateObject = {
393 [firstNestedFieldName]: {
394 [secondNestedFieldName]: firebase.firestore.FieldValue.arrayUnion(totalCEPEData),
395 'template': JSON.stringify(templateObj),
396 'all_symbols': allSymbolArray
397 }
398 }
399 intraday_chart_data_Ref.set( updateObject , { merge: true });
400
401 } else {
402 throw error;
403 }
404 }).then(() => {
405 console.log("Document created successfully.");
406 }).catch((error) => {
407 console.log("Error creating or updating document: ", error);
408 });
409
410
411
412
413 }
414
415 addedQuoteDerivatives = addedQuoteDerivatives+1
416
417
418 /*
419
420
421 var symbol = returnedDataArray[3]
422
423
424 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime;
425 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
426
427 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
428 var firstNestedFieldName = 'data'
429 var secondNestedFieldName = symbol
430 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
431 var fieldPath = firstNestedFieldName+'.'+ secondNestedFieldName
432 //var fieldPath = 'totalCEPEOIData.time.test'
433
434
435 var totalCEPEData = {};
436 //totalCEPEData = returnedDataArray[0] // option chain all data
437 totalCEPEData = JSON.stringify(returnedDataArray[0])// option chain all data
438 //totalCEPEData = symbol // option chain all data
439
440 //console.log("pathToStoreData", pathToStoreData)
441 //console.log("fieldPath", fieldPath)
442 //console.log("totalCEPEData", totalCEPEData)
443
444 var intraday_chart_data_Ref = db.doc( pathToStoreData );
445
446
447 // Try to update the existing document using the update() method
448 await intraday_chart_data_Ref.update({
449 [fieldPath]: totalCEPEData
450 }).then(() => {
451 console.log("Document updated successfully.");
452 }).catch((error) => {
453 // If the update() method fails with a "document does not exist" error,
454 // create a new document using the set() method instead
455 if (error.code === "not-found") {
456
457 var updateObject = {
458 [firstNestedFieldName]: {
459 [secondNestedFieldName]: totalCEPEData
460 }
461 }
462 intraday_chart_data_Ref.set( updateObject , { merge: true });
463
464 } else {
465 throw error;
466 }
467 }).then(() => {
468 console.log("Document created successfully.");
469 }).catch((error) => {
470 console.log("Error creating or updating document: ", error);
471 });
472
473 */
474
475
476
477
478
479 }
480
481
482 }
483 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
484 } //localStorageChecking_and_redirecting(apiObject,iframeSrc) function ENDS HERE
485
486
487
488async function updateIndex( db, createdAtTime, currentSystemDate )
489{
490
491 console.log('docPathForIndexObj', docPathForIndexObj);
492 try { // code that might throw an exception
493
494 var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
495
496
497 var ApiMainObject = {};
498 var returnedDataArray ;
499 var pathToStoreData;
500 var pathOfDocument;
501 var dataObj;
502
503 var options = {
504 timeZone: 'Asia/Kolkata',
505 month: 'long',
506 year: 'numeric',
507 };
508
509
510 var allDates = []
511 for( var paths in docPathForIndexObj )
512 {
513 var pathDate = paths.split('/')[1]
514 allDates.push( pathDate )
515 }
516
517 var newestDate = new Date(allDates[0]);
518 var latestDate = allDates[0]
519 if (allDates.length > 1) {
520 for (var i = 1; i < allDates.length; i++) {
521 var currentDate = new Date(allDates[i]);
522 if (currentDate > newestDate) {
523 newestDate = currentDate;
524 latestDate = allDates[i]
525 }
526 }
527 }
528
529 var Index = {};
530
531 //var monthYear = months[newestDate.getMonth()] +' '+ newestDate.getFullYear();
532 var monthYear = new Date( latestDate ).toLocaleString('en-IN', options);
533
534 //var indexRef = db.doc('/April 2023/26-Apr-2023/');
535 var indexRef = db.doc('/'+monthYear+'/'+latestDate+'/');
536 await indexRef.get().then(function(querySnapshot) {
537 //console.log(doc.id, " => ", doc.data());
538 console.log( querySnapshot.data() )
539 if( querySnapshot.data() != undefined )
540 {
541 if( querySnapshot.data()['Index'] !=undefined )
542 Index = querySnapshot.data()['Index']
543 }
544
545
546 })
547 .catch(function(error) {
548 console.log("Error getting documents: ", error);
549 });
550
551 // (added on 21-May-2023) adding externally common_timeValues in docPathForIndexObj for index creation (STARTS HERE)
552 var nseDate = latestDate
553 if( Date.parse(nseDate) == Date.parse(currentSystemDate) )
554 {
555 var commonTimeValuesPathToStoreData = monthYear+'/'+latestDate+'/common_time_values/'+createdAtTime ;
556 docPathForIndexObj[ commonTimeValuesPathToStoreData ] = 'common_time_values/'+createdAtTime ;
557 }
558 // (added on 21-May-2023) adding externally common_timeValues in docPathForIndexObj for index creation (ENDS HERE)
559
560 if ( Index['delivery_data'] ==undefined )
561 {
562 var startDate = new Date( latestDate );
563 var startDateForDelivery = new Date(startDate.getFullYear(), startDate.getMonth(), startDate.getDate() - 1);
564 var calculatedDeliveryDataObj = await getCalculatedDeliveryData( startDateForDelivery )
565
566 if (calculatedDeliveryDataObj !== null) {
567 //console.log(fetchedData); // Process the retrieved data here
568 await addCalculatedDeliveryData( db, latestDate, calculatedDeliveryDataObj )
569
570 } else {
571 // Handle the error or fallback behavior here
572 //console.error('An error occurred while fetching the data');
573 }
574
575 }
576
577 console.log('docPathForIndexObj after deliveryData', docPathForIndexObj);
578 console.log("Index",Index)
579
580 for( var pathToStoreData in docPathForIndexObj )
581 {
582 try { // code that might throw an exception
583
584 var firstNestedFieldName, secondNestedFieldName, thirdNestedFieldName;
585 var timeValue;
586 var pathOfDocument;
587
588 //var nameOfSegment = pathOfDocument.split('/')[1]
589 var pathDate = paths.split('/')[1]
590 if( pathDate == latestDate )
591 {
592 console.log("pathToStoreData",pathToStoreData)
593
594 if( pathToStoreData.includes('option_chain_data')==true )
595 {
596 pathOfDocument = docPathForIndexObj[ pathToStoreData ]['pathOfDocument']
597 var expiriesArray = docPathForIndexObj[ pathToStoreData ]['expiryDates']
598
599 firstNestedFieldName = pathOfDocument.split('/')[0]
600 secondNestedFieldName = pathOfDocument.split('/')[1]
601 timeValue = pathOfDocument.split('/')[2]
602
603
604
605 console.log("pathOfDocument",pathOfDocument)
606 console.log("firstNestedFieldName",firstNestedFieldName)
607 console.log("secondNestedFieldName",secondNestedFieldName)
608 console.log("timeValue",timeValue)
609
610 console.log("Index[firstNestedFieldName]",Index[firstNestedFieldName])
611
612 if( Index[firstNestedFieldName] ==undefined )
613 {
614 Index[firstNestedFieldName] = { //e.g. option_chain_data
615 [secondNestedFieldName]:{ //eg. NIFTY50
616 "timeValues": [timeValue],
617 "expiryDates": expiriesArray
618 }
619 }
620 }
621 else
622 {
623 if( Index[firstNestedFieldName][secondNestedFieldName] ==undefined )
624 {
625 Index[firstNestedFieldName][secondNestedFieldName]={ //eg. NIFTY50
626 "timeValues": [timeValue],
627 "expiryDates": expiriesArray
628 }
629 }
630 else
631 {
632 Index[firstNestedFieldName][secondNestedFieldName]["timeValues"].push(timeValue)
633 Index[firstNestedFieldName][secondNestedFieldName]["expiryDates"] = expiriesArray
634 Index[firstNestedFieldName][secondNestedFieldName]["timeValues"] = sortTimesAsc( [...new Set( Index[firstNestedFieldName][secondNestedFieldName]["timeValues"] )] );
635 }
636 }
637
638 //Index[ firstNestedFieldName ] = [ timeValue ]
639 }
640
641 else if( ( pathToStoreData.includes('equity_market')==true && docPathForIndexObj[ pathToStoreData ].split('/').length>2 ) || pathToStoreData.includes('pre_open_market')==true )
642 {
643 pathOfDocument = docPathForIndexObj[ pathToStoreData ]
644
645 firstNestedFieldName = pathOfDocument.split('/')[0]
646 secondNestedFieldName = pathOfDocument.split('/')[1]
647 timeValue = pathOfDocument.split('/')[2]
648
649 if( Index[firstNestedFieldName] ==undefined )
650 {
651 Index[firstNestedFieldName] = { //e.g. option_chain_data
652 [secondNestedFieldName] : [timeValue]
653 }
654 }
655 else
656 {
657 if( Index[firstNestedFieldName][secondNestedFieldName] ==undefined )
658 {
659 Index[firstNestedFieldName][secondNestedFieldName] = [timeValue]
660 }
661 else
662 {
663 Index[firstNestedFieldName][secondNestedFieldName].push(timeValue)
664 Index[firstNestedFieldName][secondNestedFieldName] = sortTimesAsc( [...new Set( Index[firstNestedFieldName][secondNestedFieldName] )] )
665 }
666 }
667
668 //Index[ firstNestedFieldName ] = [ timeValue ]
669 }
670
671 //else if( pathToStoreData.includes('intraday_chart_data')==true )
672
673
674 //else if( pathToStoreData.includes('sector_stockNames')==true )
675
676 else if( pathToStoreData.includes('trading_dates')==false || (docPathForIndexObj[ pathToStoreData ].includes('trading_dates')==false ) )
677 {
678 pathOfDocument = docPathForIndexObj[ pathToStoreData ]
679 firstNestedFieldName = pathOfDocument.split('/')[0]
680 timeValue = pathOfDocument.split('/')[1]
681 //Index[ firstNestedFieldName ] = [ timeValue ]
682
683 if( Index[firstNestedFieldName] ==undefined )
684 {
685 Index[firstNestedFieldName] = [timeValue]
686 }
687 else
688 {
689 Index[firstNestedFieldName].push(timeValue)
690 Index[firstNestedFieldName] = sortTimesAsc( [...new Set( Index[firstNestedFieldName] )] )
691 }
692
693
694 }
695
696
697 }
698
699 } catch(error) {
700 // handle the error
701 console.error(error);
702 }
703
704 }
705
706 console.log("Index",Index)
707
708 pathToStoreData = monthYear +'/'+ latestDate ;
709
710 dataObj={
711 "Index": Index
712 }
713
714
715 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
716 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
717
718 } catch(error) {
719 // handle the error
720 console.log(error);
721}
722
723}
724
725
726
727function sortTimesAsc(times) {
728 times.sort((a, b) => {
729 // Convert time values to milliseconds since midnight
730 const [aHrs, aMins, aSecs] = a.split(":").map(Number);
731 const aMs = aHrs * 3600000 + aMins * 60000 + aSecs * 1000;
732
733 const [bHrs, bMins, bSecs] = b.split(":").map(Number);
734 const bMs = bHrs * 3600000 + bMins * 60000 + bSecs * 1000;
735
736 // Compare the milliseconds values
737 return aMs - bMs;
738 });
739
740 return times;
741}
742
743var equityData = {};
744var docPathForIndexObj= {};
745var onlyForSectorsStocksObj= {
746 dates:[],
747 data:{}
748};
749
750
751Apify.main(async () => {
752
753 try {
754 const input = await Apify.getInput();
755
756 const { data } = input;
757 let islegacyPhantomJSTask = false;
758 if (data && typeof data === 'string') {
759 // It runs from legacy phantomjs crawler task finished webhook
760 const legacyInput = JSON.parse(data);
761 Object.assign(input, legacyInput);
762 islegacyPhantomJSTask = true;
763 }
764
765 const { datasetId, apiKey, authDomain, projectId, collectionName } = input;
766
767 if (!datasetId) {
768 throw new Error('DatasetId is required on input.');
769 }
770
771 firebase.initializeApp({
772 apiKey,
773 authDomain,
774 projectId,
775 });
776
777 // Initialize Cloud Firestore through Firebase
778 const db = firebase.firestore();
779 console.log(`Start importing dataset ${datasetId} to firestore.`);
780 const dataset = await Apify.openDataset(datasetId, { forceCloud: true });
781 const datasetInfo = await dataset.getInfo();
782
783 //console.log('datasetInfo:', datasetInfo);
784
785 var createdAt = new Date( datasetInfo["createdAt"] )
786
787 var options = {
788 timeZone: 'Asia/Kolkata',
789 year: 'numeric',
790 month: '2-digit',
791 day: '2-digit',
792 hour: '2-digit',
793 minute: '2-digit',
794 second: '2-digit' ,
795 hour12: false
796 };
797
798 var dateOptions = {
799 timeZone: 'Asia/Kolkata',
800 year: 'numeric',
801 month: 'long', // Specify 'long' for full month name
802 day: '2-digit',
803};
804
805
806 //var createdAtTime = new Date( datasetInfo["createdAt"] ).toTimeString('en-IN', options).split(' ')[0]
807 var currentSystemDate = new Date( datasetInfo["createdAt"] ).toLocaleString('en-IN', dateOptions)
808 var createdAtTime = new Date( datasetInfo["createdAt"] ).toLocaleString('en-IN', options).split(' ')[1]
809
810 createdAtTime = createdAtTime.split(':')[0] + ':' + createdAtTime.split(':')[1]+ ':00'
811
812console.log("currentSystemDate", currentSystemDate, "createdAtTime",createdAtTime)
813
814 // Import dataset from actor/task
815 const limit = 1000;
816 let counter = 0;
817 for (let offset = 0; offset < datasetInfo.itemCount; offset += limit) {
818 const pagination = await dataset.getData({
819 simplified: islegacyPhantomJSTask,
820 clean: !islegacyPhantomJSTask,
821 limit,
822 offset,
823 });
824 console.log(`Get dataset items offset: ${pagination.offset}`);
825 //for (const item of pagination.items) {
826 for (const item of pagination.items[0]['pageText']) {
827 var fetchedURL = item["url"];
828 var myurl = (item["url"].replace(/[^\w\s]/gi, ""));
829 console.log('item:', myurl );
830 console.log('createdAt:', createdAt );
831
832 try {
833 if( item["pageText"]!="" )
834 {
835 //var myPath= "March2022/10Mar2023/"+myurl+"/"+createdAtTime;
836 /*var myPath = TestFunction(myurl,createdAtTime);
837 //await db.collection(collectionName).doc( myurl ).set( JSON.parse(item["pageText"]) ); // Use doc() and set() instead of add()
838 //await db.doc( myPath ).set( JSON.parse(item["pageText"]) ); // Use doc() and set() instead of add()
839 var dataObj={
840 "objectValue":(item["pageText"])
841 }
842 await db.doc( myPath ).set( dataObj ); // Use doc() and set() instead of add()
843
844 //console.log('Added item:', item.replace(/[^\w\s]/gi, ''));
845 */
846
847 //await handleData( JSON.parse(item["pageText"]), fetchedURL, db, createdAtTime, currentSystemDate);
848 await handleData( item["pageText"], fetchedURL, db, createdAtTime, currentSystemDate);
849
850 counter++;
851 }
852
853 } catch (err) {
854 //console.log(`Cannot import item ${JSON.stringify(item)}: ${err.message}`);
855 console.log(`Cannot import item ${myurl}: ${err.message} : ${err}`);
856 }
857 }
858 }
859
860 //await addMargedsStocksQuoteDerivativesData( db, createdAtTime, currentSystemDate )
861
862 //await addMargedExtractedNiftyTotalMarket( db, createdAtTime, currentSystemDate )
863 //await addExtractedSectorStockNamesData( db, createdAtTime, currentSystemDate )
864 //await updateIndex( db, createdAtTime, currentSystemDate )
865
866 console.log(`Imported ${counter} from dataset ${datasetId}.`);
867
868 console.log('Done!');
869 console.log('docPathForIndexObj', docPathForIndexObj);
870 //console.log('onlyForSectorsStocksObj', onlyForSectorsStocksObj);
871
872
873 const datasetDrop = await dataset.drop();
874 console.log('datasetDrop', datasetDrop);
875
876
877
878 console.log( "allStocksFuturesOptionsDataObj Length:", JSON.stringify( allStocksFuturesOptionsDataObj ).length )
879 /*
880var storageSize = getStorageSize(allStocksFuturesOptionsDataObj );
881
882console.log("Bytes:", storageSize.bytes);
883console.log("Kilobytes:", storageSize.kilobytes);
884console.log("Megabytes:", storageSize.megabytes);
885console.log("Gigabytes:", storageSize.gigabytes);
886 console.log('allStocksFuturesOptionsDataObj', allStocksFuturesOptionsDataObj);
887*/
888
889 } catch (err) {
890 //console.log(`Cannot import item ${JSON.stringify(item)}: ${err.message}`);
891 console.log(`Error while running main ${err.message} : ${err}`);
892 }
893
894});main.js
1const Apify = require('apify');
2const firebase = require('firebase');
3
4
5 function extractTradingDates( apiObject, iframeSrc){ // this function extracts previous trading date and then save it into IndexedDB
6
7 var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
8 var date = new Date( apiObject[1]['marketCurrentTradingDate'] )
9 var dayOfMonth = String(date.getDate());
10 if (dayOfMonth.length < 2) {
11 dayOfMonth = '0' + dayOfMonth;
12 }
13 var currentTradingDate = dayOfMonth +'-'+ months[date.getMonth()] +'-'+ date.getFullYear()
14 date =new Date( apiObject[1]['marketPreviousTradingDate'] )
15 var dayOfMonth = String(date.getDate());
16 if (dayOfMonth.length < 2) {
17 dayOfMonth = '0' + dayOfMonth;
18 }
19 var previousTradingDate = dayOfMonth +'-'+ months[date.getMonth()] +'-'+ date.getFullYear()
20
21 var nextTradingDate = apiObject[1]['marketNextTradingDate']
22
23 //var tradingDates={ "marketCurrentTradingDate":currentTradingDate, "marketPreviousTradingDate":previousTradingDate, "marketNextTradingDate":nextTradingDate }
24 var tradingDates = { "currentTradingDate":currentTradingDate, "previousTradingDate":previousTradingDate, "nextTradingDate":nextTradingDate }
25
26 var timeStamp = currentTradingDate;
27
28 var id = "trading_dates";
29
30 return [ tradingDates, timeStamp, id ]
31
32
33 } // extractTradingDates(apiObject) function ENDS HERE
34
35
36
37var allStocksFuturesOptionsDataObj ={}
38var addedQuoteDerivatives = 0
39async function extractQuoteDerivativesData(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount )
40 {
41
42 var ApiMainObject = {};
43 var timeStamp = apiObject['opt_timestamp'].split(' ')[0]
44
45var templateObj = {
46"symbol":{
47//"identifier":[ 'openPrice', 'highPrice','lowPrice', 'lastPrice', 'change', 'pChange', 'prevClose', 'numberOfContractsTraded', 'totalTurnover',['tradeInfo -> openInterest', 'changeinOpenInterest', 'pchangeinOpenInterest', 'premiumTurnover', 'tradedVolume', 'value', 'vmap', 'impliedVolatility'], ['marketDeptOrderBook -> totalBuyQuantity', 'totalSellQuantity', [ 'ask -> price : quantity'], [ 'bid -> price : quantity'], ['carryOfCost -> price of bestBuy', 'price of bestSell','price of lastPrice', 'carry of bestBuy', 'carry of bestSell', 'carry of lastPrice'] ] ],
48"identifier":[ 'openPrice', 'highPrice','lowPrice', 'lastPrice', 'change', 'pChange', 'prevClose', 'numberOfContractsTraded/tradedVolume', 'totalTurnover/value',['tradeInfo -> openInterest', 'changeinOpenInterest', 'pchangeinOpenInterest', 'premiumTurnover', 'vmap', 'impliedVolatility'], ['marketDeptOrderBook -> totalBuyQuantity', 'totalSellQuantity', [ 'ask -> price : quantity'], [ 'bid -> price : quantity'], ['carryOfCost -> price of bestBuy', 'price of bestSell','price of lastPrice', 'carry of bestBuy', 'carry of bestSell', 'carry of lastPrice'] ] ],
49'underlyingValue' : 'underlyingValue',
50'marketLot':'marketLot',
51'fut_timestamp': 'fut_timestamp',
52'opt_timestamp': 'opt_timestamp',
53
54}
55
56}
57
58var allSymbolArray = apiObject['allSymbol'].sort()
59var symbol = apiObject['info']['symbol']
60var stockFuturesOptionsDataObj = {
61 [symbol] : {}
62}
63
64for( var i=0;i<apiObject['stocks'].length;i++)
65{
66 var dataObj = apiObject['stocks'][i]
67
68 var tradeInfo = [
69 dataObj ['marketDeptOrderBook']['tradeInfo']['openInterest'],
70 dataObj ['marketDeptOrderBook']['tradeInfo']['changeinOpenInterest'],
71 dataObj ['marketDeptOrderBook']['tradeInfo']['pchangeinOpenInterest'],
72 dataObj ['marketDeptOrderBook']['tradeInfo']['premiumTurnover'],
73 //dataObj ['marketDeptOrderBook']['tradeInfo']['tradedVolume'],
74 //dataObj ['marketDeptOrderBook']['tradeInfo']['value'],
75 dataObj ['marketDeptOrderBook']['tradeInfo']['vmap'],
76 dataObj ['marketDeptOrderBook']['otherInfo']['impliedVolatility'],
77 ]
78
79 var askArray = [], bidArray = []
80 for(var a=0;a< dataObj ['marketDeptOrderBook']['ask'].length;a++ )
81 {
82 var data = dataObj ['marketDeptOrderBook']['ask'][a]
83 askArray.push( data['price'] +':'+data['quantity'] )
84 }
85
86 for(var b=0;b< dataObj ['marketDeptOrderBook']['bid'].length;b++ )
87 {
88 var data = dataObj ['marketDeptOrderBook']['bid'][b]
89 bidArray.push( data['price'] +':'+data['quantity'] )
90 }
91
92 var carryOfCostArray = [
93 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['bestBuy'],
94 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['bestSell'],
95 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['lastPrice'],
96 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['bestBuy'],
97 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['bestSell'],
98 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['lastPrice'],
99 ]
100
101 var marketDeptOrderBook = [
102 dataObj ['marketDeptOrderBook']['totalBuyQuantity'],
103 dataObj['marketDeptOrderBook']['totalSellQuantity'],
104 askArray,
105 bidArray,
106 carryOfCostArray,
107 ]
108 /*
109 var marketDeptOrderBook = {
110 'ask' : dataObj ['marketDeptOrderBook']['ask'],
111 'bid' : dataObj ['marketDeptOrderBook']['bid'],
112
113 }
114 */
115
116 var dataArray = [
117
118
119 //dataObj ['metadata']['closePrice'] ,
120 dataObj ['metadata']['openPrice'] ,
121 dataObj ['metadata']['highPrice'] ,
122 dataObj ['metadata']['lowPrice'] ,
123 dataObj ['metadata']['lastPrice'] ,
124 dataObj ['metadata']['change'] ,
125 dataObj ['metadata']['pChange'] ,
126 dataObj ['metadata']['prevClose'] ,
127 dataObj ['metadata']['numberOfContractsTraded'] ,
128 dataObj ['metadata']['totalTurnover'] ,
129 tradeInfo,
130 marketDeptOrderBook
131
132 ]
133
134 var commonDataArray = [ ]
135 if( dataObj['metadata']['numberOfContractsTraded']>0 )
136 {
137 var identifier = dataObj['metadata']['identifier'].replace(symbol,'')
138
139
140 stockFuturesOptionsDataObj[symbol][identifier] = dataArray
141
142 stockFuturesOptionsDataObj[symbol]['underlyingValue'] = dataObj ['underlyingValue']
143 stockFuturesOptionsDataObj[symbol]['marketLot'] = dataObj ['marketDeptOrderBook']['tradeInfo']['marketLot']
144 stockFuturesOptionsDataObj[symbol]['fut_timestamp'] = apiObject['fut_timestamp']
145 stockFuturesOptionsDataObj[symbol]['opt_timestamp'] = apiObject['opt_timestamp']
146
147 }
148
149}
150
151//console.log(apiObject['stocks'].length)
152//console.log(Object.keys(stockFuturesOptionsDataObj[symbol]).length)
153//console.log( stockFuturesOptionsDataObj )
154
155var key = Object.keys(stockFuturesOptionsDataObj)[0]
156//var key = 'data'
157var valueOfKey = stockFuturesOptionsDataObj[key]
158
159 var previousAllStocksFuturesOptionsDataObj = {}
160 previousAllStocksFuturesOptionsDataObj = allStocksFuturesOptionsDataObj
161 var id = "all_derivatives"
162
163//allStocksFuturesOptionsDataObj[ key ] = valueOfKey
164
165allStocksFuturesOptionsDataObj[ key ] = JSON.stringify( valueOfKey )
166var saveDataOrNot = false
167if( JSON.stringify(allStocksFuturesOptionsDataObj).length >= 1045000 )
168//if( pageCount%5==0 )
169{
170 var anyPreviousKey = Object.keys (previousAllStocksFuturesOptionsDataObj)[0]
171 //console.log("anyPreviousKey",anyPreviousKey)
172 //var dataArrayToBeReturned = [ previousAllStocksFuturesOptionsDataObj, timeStamp, id, key, templateObj, allSymbolArray ]
173 var dataArrayToBeReturned = [ previousAllStocksFuturesOptionsDataObj, timeStamp, id, anyPreviousKey, templateObj, allSymbolArray ]
174 await saveDataIntoFirebase(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount, dataArrayToBeReturned )
175
176 //1,048,576
177 //allStocksFuturesOptionsDataObj[ key ] = {}
178 allStocksFuturesOptionsDataObj = {}
179 //allStocksFuturesOptionsDataObj[ key ] = apiObject
180 allStocksFuturesOptionsDataObj[ key ] = JSON.stringify( valueOfKey )
181 var dataArrayToBeReturned2 = [ allStocksFuturesOptionsDataObj, timeStamp, id, key, templateObj, allSymbolArray ]
182 return dataArrayToBeReturned2
183}
184else
185{
186 var dataArrayToBeReturned = [ allStocksFuturesOptionsDataObj, timeStamp, id, key, templateObj, allSymbolArray ]
187 return dataArrayToBeReturned
188}
189
190
191
192
193
194
195
196 //return [ stockFuturesOptionsDataObj, timeStamp, id, key ]
197
198 //return [ valueOfKey, timeStamp, id, key, templateObj, allSymbolArray ]
199
200 }
201
202 async function handleData(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount ) { // this function first handles localStorages for current NSE_APIs_Data, PreviousDay_NSE_APIs_Data. and then redirect iframe's JSON object to specified function according to iframe src
203
204 var ApiMainObject = {};
205 var returnedDataArray ;
206 var pathToStoreData;
207 var pathOfDocument;
208 var dataObj;
209
210 var options = {
211 timeZone: 'Asia/Kolkata',
212 month: 'long',
213 year: 'numeric',
214 };
215
216 if ( iframeSrc.includes("api/quote-derivative?symbol") == true ) {
217 console.log( totalPagesLength, pageCount )
218 //returnedDataArray = await extractQuoteDerivativesData(apiObject)
219 returnedDataArray = await extractQuoteDerivativesData(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount )
220 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime+ '/' + createdAtTime+ '/' +returnedDataArray[3];
221 }
222
223 if( totalPagesLength == pageCount )
224 {
225 console.log("saving at reaching last page")
226
227 await saveDataIntoFirebase(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount, returnedDataArray )
228 }
229
230 } //localStorageChecking_and_redirecting(apiObject,iframeSrc) function ENDS HERE
231
232
233 async function saveDataIntoFirebase(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount, returnedDataArray ) { // this function first handles localStorages for current NSE_APIs_Data, PreviousDay_NSE_APIs_Data. and then redirect iframe's JSON object to specified function according to iframe src
234 console.log( "returnedDataArray", returnedDataArray )
235 var ApiMainObject = {};
236 //var returnedDataArray ;
237 var pathToStoreData;
238 var pathOfDocument;
239 var dataObj;
240
241 var options = {
242 timeZone: 'Asia/Kolkata',
243 month: 'long',
244 year: 'numeric',
245 };
246
247 if ( iframeSrc.includes("api/quote-derivative?symbol") == true ) {
248 //console.log( totalPagesLength, pageCount )
249 //returnedDataArray = extractQuoteDerivativesData(apiObject)
250 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime+ '/' + createdAtTime+ '/' +returnedDataArray[3];
251 }
252
253 //if( totalPagesLength-1 == pageCount )
254 {
255 //April 2023/26-Apr-2023/sector_stockNames/NIFTY_BANK/22:21:16 has
256 var monthYear = new Date( returnedDataArray[1] ).toLocaleString('en-IN', options);
257 //pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + returnedDataArray[2] + '/' +createdAtTime;
258
259 if ( iframeSrc.includes("api/allMarketStatus") == true ) {
260 pathToStoreData = monthYear +'/'+ returnedDataArray[1]
261
262 dataObj={
263 //[ returnedDataArray[2] ]: JSON.stringify(returnedDataArray[0])
264 [ returnedDataArray[2] ]: (returnedDataArray[0])
265 }
266 }
267 else
268 {
269 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
270 /*
271 dataObj={
272 //"data": JSON.stringify(returnedDataArray[0])
273 "data": (returnedDataArray[0])
274 }
275 */
276 dataObj=returnedDataArray[0]
277 }
278
279
280 console.log( JSON.stringify(returnedDataArray[0]).length )
281
282 var nseDate = pathToStoreData.split(/\//)[1]
283
284 if( Date.parse(nseDate) == Date.parse(currentSystemDate) )
285 {
286 if ( iframeSrc.includes("api/allMarketStatus") == true )
287 {
288 //await db.doc( pathToStoreData ).set( dataObj );
289 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
290 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
291 }
292 else if ( iframeSrc.includes("api/quote-derivative?symbol") == true )
293 {
294 //await db.doc( pathToStoreData ).set( dataObj );
295 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
296 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
297
298 if(addedQuoteDerivatives==0)
299 { //this is for adding template of derivatives in index object
300 /*
301 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime+ '/' + createdAtTime+ '/' +'Template';
302 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
303
304 dataObj={
305 "data": ['open','high','low','close']
306 }
307
308 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
309 */
310
311
312 //updating index
313
314
315 var id = returnedDataArray[2]
316 var symbol = returnedDataArray[3]
317 var templateObj = returnedDataArray[4]
318 var allSymbolArray = returnedDataArray[5]
319
320 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime;
321 //pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
322 pathToStoreData = monthYear +'/'+ returnedDataArray[1]
323
324
325
326
327 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
328 var firstNestedFieldName = id+ '_index'
329 var secondNestedFieldName = 'time_values'
330 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
331 var fieldPath = firstNestedFieldName+'.'+ secondNestedFieldName
332 var templateFieldPath = firstNestedFieldName+'.'+ 'template'
333 var allSymbolsFieldPath = firstNestedFieldName+'.'+ 'all_symbols'
334 //var fieldPath = 'totalCEPEOIData.time.test'
335
336
337 //var totalCEPEData = {};
338 //totalCEPEData = returnedDataArray[0] // option chain all data
339 //totalCEPEData = JSON.stringify(returnedDataArray[0])// option chain all data
340 var totalCEPEData = createdAtTime// option chain all data
341
342
343 //totalCEPEData = symbol // option chain all data
344
345 //console.log("pathToStoreData", pathToStoreData)
346 //console.log("fieldPath", fieldPath)
347 //console.log("totalCEPEData", totalCEPEData)
348
349 var intraday_chart_data_Ref = db.doc( pathToStoreData );
350
351
352 // Try to update the existing document using the update() method
353 await intraday_chart_data_Ref.update({
354 [fieldPath]: firebase.firestore.FieldValue.arrayUnion(totalCEPEData),
355 [templateFieldPath]: JSON.stringify(templateObj),
356 [allSymbolsFieldPath]: allSymbolArray
357 }).then(() => {
358 console.log("Document updated successfully.");
359 }).catch((error) => {
360 // If the update() method fails with a "document does not exist" error,
361 // create a new document using the set() method instead
362 if (error.code === "not-found") {
363
364 var updateObject = {
365 [firstNestedFieldName]: {
366 [secondNestedFieldName]: firebase.firestore.FieldValue.arrayUnion(totalCEPEData),
367 'template': JSON.stringify(templateObj),
368 'all_symbols': allSymbolArray
369 }
370 }
371 intraday_chart_data_Ref.set( updateObject , { merge: true });
372
373 } else {
374 throw error;
375 }
376 }).then(() => {
377 console.log("Document created successfully.");
378 }).catch((error) => {
379 console.log("Error creating or updating document: ", error);
380 });
381
382
383
384
385 }
386
387 addedQuoteDerivatives = addedQuoteDerivatives+1
388
389
390 /*
391
392
393 var symbol = returnedDataArray[3]
394
395
396 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime;
397 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
398
399 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
400 var firstNestedFieldName = 'data'
401 var secondNestedFieldName = symbol
402 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
403 var fieldPath = firstNestedFieldName+'.'+ secondNestedFieldName
404 //var fieldPath = 'totalCEPEOIData.time.test'
405
406
407 var totalCEPEData = {};
408 //totalCEPEData = returnedDataArray[0] // option chain all data
409 totalCEPEData = JSON.stringify(returnedDataArray[0])// option chain all data
410 //totalCEPEData = symbol // option chain all data
411
412 //console.log("pathToStoreData", pathToStoreData)
413 //console.log("fieldPath", fieldPath)
414 //console.log("totalCEPEData", totalCEPEData)
415
416 var intraday_chart_data_Ref = db.doc( pathToStoreData );
417
418
419 // Try to update the existing document using the update() method
420 await intraday_chart_data_Ref.update({
421 [fieldPath]: totalCEPEData
422 }).then(() => {
423 console.log("Document updated successfully.");
424 }).catch((error) => {
425 // If the update() method fails with a "document does not exist" error,
426 // create a new document using the set() method instead
427 if (error.code === "not-found") {
428
429 var updateObject = {
430 [firstNestedFieldName]: {
431 [secondNestedFieldName]: totalCEPEData
432 }
433 }
434 intraday_chart_data_Ref.set( updateObject , { merge: true });
435
436 } else {
437 throw error;
438 }
439 }).then(() => {
440 console.log("Document created successfully.");
441 }).catch((error) => {
442 console.log("Error creating or updating document: ", error);
443 });
444
445 */
446
447
448
449
450
451 }
452
453
454 }
455 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
456 }
457 } //localStorageChecking_and_redirecting(apiObject,iframeSrc) function ENDS HERE
458
459
460
461async function updateIndex( db, createdAtTime, currentSystemDate )
462{
463
464 console.log('docPathForIndexObj', docPathForIndexObj);
465 try { // code that might throw an exception
466
467 var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
468
469
470 var ApiMainObject = {};
471 var returnedDataArray ;
472 var pathToStoreData;
473 var pathOfDocument;
474 var dataObj;
475
476 var options = {
477 timeZone: 'Asia/Kolkata',
478 month: 'long',
479 year: 'numeric',
480 };
481
482
483 var allDates = []
484 for( var paths in docPathForIndexObj )
485 {
486 var pathDate = paths.split('/')[1]
487 allDates.push( pathDate )
488 }
489
490 var newestDate = new Date(allDates[0]);
491 var latestDate = allDates[0]
492 if (allDates.length > 1) {
493 for (var i = 1; i < allDates.length; i++) {
494 var currentDate = new Date(allDates[i]);
495 if (currentDate > newestDate) {
496 newestDate = currentDate;
497 latestDate = allDates[i]
498 }
499 }
500 }
501
502 var Index = {};
503
504 //var monthYear = months[newestDate.getMonth()] +' '+ newestDate.getFullYear();
505 var monthYear = new Date( latestDate ).toLocaleString('en-IN', options);
506
507 //var indexRef = db.doc('/April 2023/26-Apr-2023/');
508 var indexRef = db.doc('/'+monthYear+'/'+latestDate+'/');
509 await indexRef.get().then(function(querySnapshot) {
510 //console.log(doc.id, " => ", doc.data());
511 console.log( querySnapshot.data() )
512 if( querySnapshot.data() != undefined )
513 {
514 if( querySnapshot.data()['Index'] !=undefined )
515 Index = querySnapshot.data()['Index']
516 }
517
518
519 })
520 .catch(function(error) {
521 console.log("Error getting documents: ", error);
522 });
523
524 // (added on 21-May-2023) adding externally common_timeValues in docPathForIndexObj for index creation (STARTS HERE)
525 var nseDate = latestDate
526 if( Date.parse(nseDate) == Date.parse(currentSystemDate) )
527 {
528 var commonTimeValuesPathToStoreData = monthYear+'/'+latestDate+'/common_time_values/'+createdAtTime ;
529 docPathForIndexObj[ commonTimeValuesPathToStoreData ] = 'common_time_values/'+createdAtTime ;
530 }
531 // (added on 21-May-2023) adding externally common_timeValues in docPathForIndexObj for index creation (ENDS HERE)
532
533 if ( Index['delivery_data'] ==undefined )
534 {
535 var startDate = new Date( latestDate );
536 var startDateForDelivery = new Date(startDate.getFullYear(), startDate.getMonth(), startDate.getDate() - 1);
537 var calculatedDeliveryDataObj = await getCalculatedDeliveryData( startDateForDelivery )
538
539 if (calculatedDeliveryDataObj !== null) {
540 //console.log(fetchedData); // Process the retrieved data here
541 await addCalculatedDeliveryData( db, latestDate, calculatedDeliveryDataObj )
542
543 } else {
544 // Handle the error or fallback behavior here
545 //console.error('An error occurred while fetching the data');
546 }
547
548 }
549
550 console.log('docPathForIndexObj after deliveryData', docPathForIndexObj);
551 console.log("Index",Index)
552
553 for( var pathToStoreData in docPathForIndexObj )
554 {
555 try { // code that might throw an exception
556
557 var firstNestedFieldName, secondNestedFieldName, thirdNestedFieldName;
558 var timeValue;
559 var pathOfDocument;
560
561 //var nameOfSegment = pathOfDocument.split('/')[1]
562 var pathDate = paths.split('/')[1]
563 if( pathDate == latestDate )
564 {
565 console.log("pathToStoreData",pathToStoreData)
566
567 if( pathToStoreData.includes('option_chain_data')==true )
568 {
569 pathOfDocument = docPathForIndexObj[ pathToStoreData ]['pathOfDocument']
570 var expiriesArray = docPathForIndexObj[ pathToStoreData ]['expiryDates']
571
572 firstNestedFieldName = pathOfDocument.split('/')[0]
573 secondNestedFieldName = pathOfDocument.split('/')[1]
574 timeValue = pathOfDocument.split('/')[2]
575
576
577
578 console.log("pathOfDocument",pathOfDocument)
579 console.log("firstNestedFieldName",firstNestedFieldName)
580 console.log("secondNestedFieldName",secondNestedFieldName)
581 console.log("timeValue",timeValue)
582
583 console.log("Index[firstNestedFieldName]",Index[firstNestedFieldName])
584
585 if( Index[firstNestedFieldName] ==undefined )
586 {
587 Index[firstNestedFieldName] = { //e.g. option_chain_data
588 [secondNestedFieldName]:{ //eg. NIFTY50
589 "timeValues": [timeValue],
590 "expiryDates": expiriesArray
591 }
592 }
593 }
594 else
595 {
596 if( Index[firstNestedFieldName][secondNestedFieldName] ==undefined )
597 {
598 Index[firstNestedFieldName][secondNestedFieldName]={ //eg. NIFTY50
599 "timeValues": [timeValue],
600 "expiryDates": expiriesArray
601 }
602 }
603 else
604 {
605 Index[firstNestedFieldName][secondNestedFieldName]["timeValues"].push(timeValue)
606 Index[firstNestedFieldName][secondNestedFieldName]["expiryDates"] = expiriesArray
607 Index[firstNestedFieldName][secondNestedFieldName]["timeValues"] = sortTimesAsc( [...new Set( Index[firstNestedFieldName][secondNestedFieldName]["timeValues"] )] );
608 }
609 }
610
611 //Index[ firstNestedFieldName ] = [ timeValue ]
612 }
613
614 else if( ( pathToStoreData.includes('equity_market')==true && docPathForIndexObj[ pathToStoreData ].split('/').length>2 ) || pathToStoreData.includes('pre_open_market')==true )
615 {
616 pathOfDocument = docPathForIndexObj[ pathToStoreData ]
617
618 firstNestedFieldName = pathOfDocument.split('/')[0]
619 secondNestedFieldName = pathOfDocument.split('/')[1]
620 timeValue = pathOfDocument.split('/')[2]
621
622 if( Index[firstNestedFieldName] ==undefined )
623 {
624 Index[firstNestedFieldName] = { //e.g. option_chain_data
625 [secondNestedFieldName] : [timeValue]
626 }
627 }
628 else
629 {
630 if( Index[firstNestedFieldName][secondNestedFieldName] ==undefined )
631 {
632 Index[firstNestedFieldName][secondNestedFieldName] = [timeValue]
633 }
634 else
635 {
636 Index[firstNestedFieldName][secondNestedFieldName].push(timeValue)
637 Index[firstNestedFieldName][secondNestedFieldName] = sortTimesAsc( [...new Set( Index[firstNestedFieldName][secondNestedFieldName] )] )
638 }
639 }
640
641 //Index[ firstNestedFieldName ] = [ timeValue ]
642 }
643
644 //else if( pathToStoreData.includes('intraday_chart_data')==true )
645
646
647 //else if( pathToStoreData.includes('sector_stockNames')==true )
648
649 else if( pathToStoreData.includes('trading_dates')==false || (docPathForIndexObj[ pathToStoreData ].includes('trading_dates')==false ) )
650 {
651 pathOfDocument = docPathForIndexObj[ pathToStoreData ]
652 firstNestedFieldName = pathOfDocument.split('/')[0]
653 timeValue = pathOfDocument.split('/')[1]
654 //Index[ firstNestedFieldName ] = [ timeValue ]
655
656 if( Index[firstNestedFieldName] ==undefined )
657 {
658 Index[firstNestedFieldName] = [timeValue]
659 }
660 else
661 {
662 Index[firstNestedFieldName].push(timeValue)
663 Index[firstNestedFieldName] = sortTimesAsc( [...new Set( Index[firstNestedFieldName] )] )
664 }
665
666
667 }
668
669
670 }
671
672 } catch(error) {
673 // handle the error
674 console.error(error);
675 }
676
677 }
678
679 console.log("Index",Index)
680
681 pathToStoreData = monthYear +'/'+ latestDate ;
682
683 dataObj={
684 "Index": Index
685 }
686
687
688 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
689 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
690
691 } catch(error) {
692 // handle the error
693 console.log(error);
694}
695
696}
697
698
699
700function sortTimesAsc(times) {
701 times.sort((a, b) => {
702 // Convert time values to milliseconds since midnight
703 const [aHrs, aMins, aSecs] = a.split(":").map(Number);
704 const aMs = aHrs * 3600000 + aMins * 60000 + aSecs * 1000;
705
706 const [bHrs, bMins, bSecs] = b.split(":").map(Number);
707 const bMs = bHrs * 3600000 + bMins * 60000 + bSecs * 1000;
708
709 // Compare the milliseconds values
710 return aMs - bMs;
711 });
712
713 return times;
714}
715
716var equityData = {};
717var docPathForIndexObj= {};
718var onlyForSectorsStocksObj= {
719 dates:[],
720 data:{}
721};
722
723
724Apify.main(async () => {
725
726 try {
727 const input = await Apify.getInput();
728
729 const { data } = input;
730 let islegacyPhantomJSTask = false;
731 if (data && typeof data === 'string') {
732 // It runs from legacy phantomjs crawler task finished webhook
733 const legacyInput = JSON.parse(data);
734 Object.assign(input, legacyInput);
735 islegacyPhantomJSTask = true;
736 }
737
738 const { datasetId, apiKey, authDomain, projectId, collectionName } = input;
739
740 if (!datasetId) {
741 throw new Error('DatasetId is required on input.');
742 }
743
744 firebase.initializeApp({
745 apiKey,
746 authDomain,
747 projectId,
748 });
749
750 // Initialize Cloud Firestore through Firebase
751 const db = firebase.firestore();
752 console.log(`Start importing dataset ${datasetId} to firestore.`);
753 const dataset = await Apify.openDataset(datasetId, { forceCloud: true });
754 const datasetInfo = await dataset.getInfo();
755
756 //console.log('datasetInfo:', datasetInfo);
757
758 var createdAt = new Date( datasetInfo["createdAt"] )
759
760 var options = {
761 timeZone: 'Asia/Kolkata',
762 year: 'numeric',
763 month: '2-digit',
764 day: '2-digit',
765 hour: '2-digit',
766 minute: '2-digit',
767 second: '2-digit' ,
768 hour12: false
769 };
770
771 var dateOptions = {
772 timeZone: 'Asia/Kolkata',
773 year: 'numeric',
774 month: 'long', // Specify 'long' for full month name
775 day: '2-digit',
776};
777
778
779 //var createdAtTime = new Date( datasetInfo["createdAt"] ).toTimeString('en-IN', options).split(' ')[0]
780 var currentSystemDate = new Date( datasetInfo["createdAt"] ).toLocaleString('en-IN', dateOptions)
781 var createdAtTime = new Date( datasetInfo["createdAt"] ).toLocaleString('en-IN', options).split(' ')[1]
782
783 createdAtTime = createdAtTime.split(':')[0] + ':' + createdAtTime.split(':')[1]+ ':00'
784
785console.log("currentSystemDate", currentSystemDate, "createdAtTime",createdAtTime)
786
787 // Import dataset from actor/task
788 const limit = 1000;
789 let counter = 0;
790 for (let offset = 0; offset < datasetInfo.itemCount; offset += limit) {
791 const pagination = await dataset.getData({
792 simplified: islegacyPhantomJSTask,
793 clean: !islegacyPhantomJSTask,
794 limit,
795 offset,
796 });
797 console.log(`Get dataset items offset: ${pagination.offset}`);
798 var itemCount = 0
799
800 var combinedItemsArrayofObj = [
801 {
802 "url":"combinedArrayofObj",
803 "pageText":[]
804 }
805 ]
806
807 for(var dtaset=0;dtaset<pagination.items.length;dtaset++)
808 {
809 var dataArray =[]
810 dataArray = pagination.items[dtaset]["pageText"]
811 for(var a=0;a<dataArray.length;a++)
812 {
813
814 //concatedArray.concat(dataArray);
815 combinedItemsArrayofObj[0]['pageText'].push(dataArray[a])
816 }
817
818 }
819
820 //var itemsLength = Object.keys( pagination.items[0]["pageText"] ).length
821 var itemsLength = Object.keys( combinedItemsArrayofObj[0]["pageText"] ).length
822 //for (const item of pagination.items[0]["pageText"]) {
823 for (const item of combinedItemsArrayofObj[0]["pageText"]) {
824
825 itemCount = itemCount+1
826
827 var fetchedURL = item["url"];
828 var myurl = (item["url"].replace(/[^\w\s]/gi, ""));
829 console.log('item:', myurl );
830 console.log('createdAt:', createdAt );
831
832 try {
833 if( item["pageText"]!="" )
834 {
835 //var myPath= "March2022/10Mar2023/"+myurl+"/"+createdAtTime;
836 /*var myPath = TestFunction(myurl,createdAtTime);
837 //await db.collection(collectionName).doc( myurl ).set( JSON.parse(item["pageText"]) ); // Use doc() and set() instead of add()
838 //await db.doc( myPath ).set( JSON.parse(item["pageText"]) ); // Use doc() and set() instead of add()
839 var dataObj={
840 "objectValue":(item["pageText"])
841 }
842 await db.doc( myPath ).set( dataObj ); // Use doc() and set() instead of add()
843
844 //console.log('Added item:', item.replace(/[^\w\s]/gi, ''));
845 */
846
847 //await handleData( JSON.parse(item["pageText"]), fetchedURL, db, createdAtTime, currentSystemDate, itemsLength, itemCount );
848 await handleData( item["pageText"], fetchedURL, db, createdAtTime, currentSystemDate, itemsLength, itemCount );
849
850 counter++;
851 }
852
853 } catch (err) {
854 //console.log(`Cannot import item ${JSON.stringify(item)}: ${err.message}`);
855 console.log(`Cannot import item ${myurl}: ${err.message} : ${err}`);
856 }
857 }
858 }
859
860 //await addMargedsStocksQuoteDerivativesData( db, createdAtTime, currentSystemDate )
861
862 //await addMargedExtractedNiftyTotalMarket( db, createdAtTime, currentSystemDate )
863 //await addExtractedSectorStockNamesData( db, createdAtTime, currentSystemDate )
864 //await updateIndex( db, createdAtTime, currentSystemDate )
865
866 console.log(`Imported ${counter} from dataset ${datasetId}.`);
867
868 console.log('Done!');
869 console.log('docPathForIndexObj', docPathForIndexObj);
870 //console.log('onlyForSectorsStocksObj', onlyForSectorsStocksObj);
871
872
873 const datasetDrop = await dataset.drop();
874 console.log('datasetDrop', datasetDrop);
875
876
877
878 console.log( "allStocksFuturesOptionsDataObj Length:", JSON.stringify( allStocksFuturesOptionsDataObj ).length )
879 /*
880var storageSize = getStorageSize(allStocksFuturesOptionsDataObj );
881
882console.log("Bytes:", storageSize.bytes);
883console.log("Kilobytes:", storageSize.kilobytes);
884console.log("Megabytes:", storageSize.megabytes);
885console.log("Gigabytes:", storageSize.gigabytes);
886 console.log('allStocksFuturesOptionsDataObj', allStocksFuturesOptionsDataObj);
887*/
888
889 } catch (err) {
890 //console.log(`Cannot import item ${JSON.stringify(item)}: ${err.message}`);
891 console.log(`Error while running main ${err.message} : ${err}`);
892 }
893
894});package.json
1{
2 "name": "firestore-import",
3 "version": "0.0.1",
4 "description": "This is a boilerplate of an Apify actor.",
5 "dependencies": {
6 "apify": "^0.16.0",
7 "firebase": "^7.2.0"
8 },
9 "devDependencies": {
10 "@apify/eslint-config": "0.0.3",
11 "eslint": "^6.5.1"
12 },
13 "scripts": {
14 "start": "node main.js",
15 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
16 },
17 "author": "It's not you it's me",
18 "license": "ISC"
19}
20
Org 1
Maintained by Community
Build new tools
Enterprise solutions