Web Test Sav Deri 2301
DeprecatedView all Actors
This Actor is deprecated
This Actor is unavailable because the developer has decided to deprecate it. Would you like to try a similar Actor instead?
See alternative ActorsWeb Test Sav Deri 2301
xrhibiyftd/web-test-sav-deri-2301
.eslintrc
1{
2 "extends": "@apify"
3}.gitignore
1apify_storage
2node_modules.npmignore
1# This file tells Git which files shouldn't be added to source control
2
3.idea
4node_modulesDockerfile
1# Dockerfile contains instructions how to build a Docker image that will contain
2# all the code and configuration needed to run your actor. For a full
3# Dockerfile reference, see https://docs.docker.com/engine/reference/builder/
4
5# First, specify the base Docker image. Apify provides the following base images
6# for your convenience:
7# apify/actor-node-basic (Node.js 10 on Alpine Linux, small and fast image)
8# apify/actor-node-chrome (Node.js 10 + Chrome on Debian)
9# apify/actor-node-chrome-xvfb (Node.js 10 + Chrome + Xvfb on Debian)
10# For more information, see https://apify.com/docs/actor#base-images
11# Note that you can use any other image from Docker Hub.
12FROM apify/actor-node-basic
13
14# Second, copy just package.json and package-lock.json since they are the only files
15# that affect NPM install in the next step
16COPY package*.json ./
17
18# Install NPM packages, skip optional and development dependencies to keep the
19# image small. Avoid logging too much and print the dependency tree for debugging
20RUN npm --quiet set progress=false \
21 && npm install --only=prod --no-optional \
22 && echo "Installed NPM packages:" \
23 && npm list \
24 && echo "Node.js version:" \
25 && node --version \
26 && echo "NPM version:" \
27 && npm --version
28
29# Next, copy the remaining files and directories with the source code.
30# Since we do this after NPM install, quick build will be really fast
31# for simple source file changes.
32COPY . ./
33
34# Specify how to run the source code
35CMD npm startINPUT_SCHEMA.json
1{
2 "title": "Firestore Import input",
3 "description": "Imports dataset to Firestore DB",
4 "type": "object",
5 "schemaVersion": 1,
6 "properties": {
7 "datasetId": {
8 "title": "Dataset",
9 "type": "string",
10 "description": "Dataset ID of dataset you want to import to Firestore",
11 "editor": "textfield"
12 },
13 "apiKey": {
14 "title": "Api key",
15 "type": "string",
16 "description": "Firestore API key",
17 "editor": "textfield"
18 },
19 "authDomain": {
20 "title": "Auth domain",
21 "type": "string",
22 "description": "Firestore authentication domain",
23 "editor": "textfield"
24 },
25 "projectId": {
26 "title": "Project ID",
27 "type": "string",
28 "description": "Firestore project ID",
29 "editor": "textfield"
30 },
31 "collectionName": {
32 "title": "Collection name",
33 "type": "string",
34 "description": "Firestore collection name",
35 "editor": "textfield"
36 }
37 }
38}apify.json
1{
2 "name": "firestore-import",
3 "version": "0.0",
4 "buildTag": "latest",
5 "env": null,
6 "template": "hello_world"
7}main.js
1const Apify = require('apify');
2const firebase = require('firebase');
3
4
5 function extractTradingDates( apiObject, iframeSrc){ // this function extracts previous trading date and then save it into IndexedDB
6
7 var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
8 var date = new Date( apiObject[1]['marketCurrentTradingDate'] )
9 var dayOfMonth = String(date.getDate());
10 if (dayOfMonth.length < 2) {
11 dayOfMonth = '0' + dayOfMonth;
12 }
13 var currentTradingDate = dayOfMonth +'-'+ months[date.getMonth()] +'-'+ date.getFullYear()
14 date =new Date( apiObject[1]['marketPreviousTradingDate'] )
15 var dayOfMonth = String(date.getDate());
16 if (dayOfMonth.length < 2) {
17 dayOfMonth = '0' + dayOfMonth;
18 }
19 var previousTradingDate = dayOfMonth +'-'+ months[date.getMonth()] +'-'+ date.getFullYear()
20
21 var nextTradingDate = apiObject[1]['marketNextTradingDate']
22
23 //var tradingDates={ "marketCurrentTradingDate":currentTradingDate, "marketPreviousTradingDate":previousTradingDate, "marketNextTradingDate":nextTradingDate }
24 var tradingDates = { "currentTradingDate":currentTradingDate, "previousTradingDate":previousTradingDate, "nextTradingDate":nextTradingDate }
25
26 var timeStamp = currentTradingDate;
27
28 var id = "trading_dates";
29
30 return [ tradingDates, timeStamp, id ]
31
32
33 } // extractTradingDates(apiObject) function ENDS HERE
34
35
36
37var allStocksFuturesOptionsDataObj ={}
38var addedQuoteDerivatives = 0
39async function extractQuoteDerivativesData(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount )
40 {
41
42 var ApiMainObject = {};
43 var timeStamp = apiObject['opt_timestamp'].split(' ')[0]
44
45var templateObj = {
46"symbol":{
47//"identifier":[ 'openPrice', 'highPrice','lowPrice', 'lastPrice', 'change', 'pChange', 'prevClose', 'numberOfContractsTraded', 'totalTurnover',['tradeInfo -> openInterest', 'changeinOpenInterest', 'pchangeinOpenInterest', 'premiumTurnover', 'tradedVolume', 'value', 'vmap', 'impliedVolatility'], ['marketDeptOrderBook -> totalBuyQuantity', 'totalSellQuantity', [ 'ask -> price : quantity'], [ 'bid -> price : quantity'], ['carryOfCost -> price of bestBuy', 'price of bestSell','price of lastPrice', 'carry of bestBuy', 'carry of bestSell', 'carry of lastPrice'] ] ],
48"identifier":[ 'openPrice', 'highPrice','lowPrice', 'lastPrice', 'change', 'pChange', 'prevClose', 'numberOfContractsTraded/tradedVolume', 'totalTurnover/value',['tradeInfo -> openInterest', 'changeinOpenInterest', 'pchangeinOpenInterest', 'premiumTurnover', 'vmap', 'impliedVolatility'], ['marketDeptOrderBook -> totalBuyQuantity', 'totalSellQuantity', [ 'ask -> price : quantity'], [ 'bid -> price : quantity'], ['carryOfCost -> price of bestBuy', 'price of bestSell','price of lastPrice', 'carry of bestBuy', 'carry of bestSell', 'carry of lastPrice'] ] ],
49'underlyingValue' : 'underlyingValue',
50'marketLot':'marketLot',
51'fut_timestamp': 'fut_timestamp',
52'opt_timestamp': 'opt_timestamp',
53
54}
55
56}
57
58var allSymbolArray = apiObject['allSymbol'].sort()
59var symbol = apiObject['info']['symbol']
60var stockFuturesOptionsDataObj = {
61 [symbol] : {}
62}
63
64for( var i=0;i<apiObject['stocks'].length;i++)
65{
66 var dataObj = apiObject['stocks'][i]
67
68 var tradeInfo = [
69 dataObj ['marketDeptOrderBook']['tradeInfo']['openInterest'],
70 dataObj ['marketDeptOrderBook']['tradeInfo']['changeinOpenInterest'],
71 dataObj ['marketDeptOrderBook']['tradeInfo']['pchangeinOpenInterest'],
72 dataObj ['marketDeptOrderBook']['tradeInfo']['premiumTurnover'],
73 //dataObj ['marketDeptOrderBook']['tradeInfo']['tradedVolume'],
74 //dataObj ['marketDeptOrderBook']['tradeInfo']['value'],
75 dataObj ['marketDeptOrderBook']['tradeInfo']['vmap'],
76 dataObj ['marketDeptOrderBook']['otherInfo']['impliedVolatility'],
77 ]
78
79 var askArray = [], bidArray = []
80 for(var a=0;a< dataObj ['marketDeptOrderBook']['ask'].length;a++ )
81 {
82 var data = dataObj ['marketDeptOrderBook']['ask'][a]
83 askArray.push( data['price'] +':'+data['quantity'] )
84 }
85
86 for(var b=0;b< dataObj ['marketDeptOrderBook']['bid'].length;b++ )
87 {
88 var data = dataObj ['marketDeptOrderBook']['bid'][b]
89 bidArray.push( data['price'] +':'+data['quantity'] )
90 }
91
92 var carryOfCostArray = [
93 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['bestBuy'],
94 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['bestSell'],
95 dataObj ['marketDeptOrderBook']['carryOfCost']['price']['lastPrice'],
96 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['bestBuy'],
97 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['bestSell'],
98 dataObj ['marketDeptOrderBook']['carryOfCost']['carry']['lastPrice'],
99 ]
100
101 var marketDeptOrderBook = [
102 dataObj ['marketDeptOrderBook']['totalBuyQuantity'],
103 dataObj['marketDeptOrderBook']['totalSellQuantity'],
104 askArray,
105 bidArray,
106 carryOfCostArray,
107 ]
108 /*
109 var marketDeptOrderBook = {
110 'ask' : dataObj ['marketDeptOrderBook']['ask'],
111 'bid' : dataObj ['marketDeptOrderBook']['bid'],
112
113 }
114 */
115
116 var dataArray = [
117
118
119 //dataObj ['metadata']['closePrice'] ,
120 dataObj ['metadata']['openPrice'] ,
121 dataObj ['metadata']['highPrice'] ,
122 dataObj ['metadata']['lowPrice'] ,
123 dataObj ['metadata']['lastPrice'] ,
124 dataObj ['metadata']['change'] ,
125 dataObj ['metadata']['pChange'] ,
126 dataObj ['metadata']['prevClose'] ,
127 dataObj ['metadata']['numberOfContractsTraded'] ,
128 dataObj ['metadata']['totalTurnover'] ,
129 tradeInfo,
130 marketDeptOrderBook
131
132 ]
133
134 var commonDataArray = [ ]
135 if( dataObj['metadata']['numberOfContractsTraded']>0 )
136 {
137 var identifier = dataObj['metadata']['identifier'].replace(symbol,'')
138
139
140 stockFuturesOptionsDataObj[symbol][identifier] = dataArray
141
142 stockFuturesOptionsDataObj[symbol]['underlyingValue'] = dataObj ['underlyingValue']
143 stockFuturesOptionsDataObj[symbol]['marketLot'] = dataObj ['marketDeptOrderBook']['tradeInfo']['marketLot']
144 stockFuturesOptionsDataObj[symbol]['fut_timestamp'] = apiObject['fut_timestamp']
145 stockFuturesOptionsDataObj[symbol]['opt_timestamp'] = apiObject['opt_timestamp']
146
147 }
148
149}
150
151//console.log(apiObject['stocks'].length)
152//console.log(Object.keys(stockFuturesOptionsDataObj[symbol]).length)
153//console.log( stockFuturesOptionsDataObj )
154
155var key = Object.keys(stockFuturesOptionsDataObj)[0]
156//var key = 'data'
157var valueOfKey = stockFuturesOptionsDataObj[key]
158
159 var previousAllStocksFuturesOptionsDataObj = {}
160 previousAllStocksFuturesOptionsDataObj = allStocksFuturesOptionsDataObj
161 var id = "all_derivatives"
162
163//allStocksFuturesOptionsDataObj[ key ] = valueOfKey
164
165allStocksFuturesOptionsDataObj[ key ] = JSON.stringify( valueOfKey )
166var saveDataOrNot = false
167if( JSON.stringify(allStocksFuturesOptionsDataObj).length >= 1045000 )
168//if( pageCount%5==0 )
169{
170 var anyPreviousKey = Object.keys (previousAllStocksFuturesOptionsDataObj)[0]
171 //console.log("anyPreviousKey",anyPreviousKey)
172 //var dataArrayToBeReturned = [ previousAllStocksFuturesOptionsDataObj, timeStamp, id, key, templateObj, allSymbolArray ]
173 var dataArrayToBeReturned = [ previousAllStocksFuturesOptionsDataObj, timeStamp, id, anyPreviousKey, templateObj, allSymbolArray ]
174 await saveDataIntoFirebase(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount, dataArrayToBeReturned )
175
176 //1,048,576
177 //allStocksFuturesOptionsDataObj[ key ] = {}
178 allStocksFuturesOptionsDataObj = {}
179 //allStocksFuturesOptionsDataObj[ key ] = apiObject
180 allStocksFuturesOptionsDataObj[ key ] = JSON.stringify( valueOfKey )
181 var dataArrayToBeReturned2 = [ allStocksFuturesOptionsDataObj, timeStamp, id, key, templateObj, allSymbolArray ]
182 return dataArrayToBeReturned2
183}
184else
185{
186 var dataArrayToBeReturned = [ allStocksFuturesOptionsDataObj, timeStamp, id, key, templateObj, allSymbolArray ]
187 return dataArrayToBeReturned
188}
189
190
191
192
193
194
195
196 //return [ stockFuturesOptionsDataObj, timeStamp, id, key ]
197
198 //return [ valueOfKey, timeStamp, id, key, templateObj, allSymbolArray ]
199
200 }
201
202 async function handleData(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount ) { // this function first handles localStorages for current NSE_APIs_Data, PreviousDay_NSE_APIs_Data. and then redirect iframe's JSON object to specified function according to iframe src
203
204 var ApiMainObject = {};
205 var returnedDataArray ;
206 var pathToStoreData;
207 var pathOfDocument;
208 var dataObj;
209
210 var options = {
211 timeZone: 'Asia/Kolkata',
212 month: 'long',
213 year: 'numeric',
214 };
215
216 if ( iframeSrc.includes("api/quote-derivative?symbol") == true ) {
217 console.log( totalPagesLength, pageCount )
218 //returnedDataArray = await extractQuoteDerivativesData(apiObject)
219 returnedDataArray = await extractQuoteDerivativesData(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount )
220 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime+ '/' + createdAtTime+ '/' +returnedDataArray[3];
221 }
222
223 if( totalPagesLength == pageCount )
224 {
225 console.log("saving at reaching last page")
226
227 await saveDataIntoFirebase(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount, returnedDataArray )
228 }
229
230 } //localStorageChecking_and_redirecting(apiObject,iframeSrc) function ENDS HERE
231
232
233 async function saveDataIntoFirebase(apiObject, iframeSrc, db, createdAtTime, currentSystemDate, totalPagesLength, pageCount, returnedDataArray ) { // this function first handles localStorages for current NSE_APIs_Data, PreviousDay_NSE_APIs_Data. and then redirect iframe's JSON object to specified function according to iframe src
234 console.log( "returnedDataArray", returnedDataArray )
235 var ApiMainObject = {};
236 //var returnedDataArray ;
237 var pathToStoreData;
238 var pathOfDocument;
239 var dataObj;
240
241 var options = {
242 timeZone: 'Asia/Kolkata',
243 month: 'long',
244 year: 'numeric',
245 };
246
247 if ( iframeSrc.includes("api/quote-derivative?symbol") == true ) {
248 //console.log( totalPagesLength, pageCount )
249 //returnedDataArray = extractQuoteDerivativesData(apiObject)
250 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime+ '/' + createdAtTime+ '/' +returnedDataArray[3];
251 }
252
253 //if( totalPagesLength-1 == pageCount )
254 {
255 //April 2023/26-Apr-2023/sector_stockNames/NIFTY_BANK/22:21:16 has
256 var monthYear = new Date( returnedDataArray[1] ).toLocaleString('en-IN', options);
257 //pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + returnedDataArray[2] + '/' +createdAtTime;
258
259 if ( iframeSrc.includes("api/allMarketStatus") == true ) {
260 pathToStoreData = monthYear +'/'+ returnedDataArray[1]
261
262 dataObj={
263 //[ returnedDataArray[2] ]: JSON.stringify(returnedDataArray[0])
264 [ returnedDataArray[2] ]: (returnedDataArray[0])
265 }
266 }
267 else
268 {
269 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
270 /*
271 dataObj={
272 //"data": JSON.stringify(returnedDataArray[0])
273 "data": (returnedDataArray[0])
274 }
275 */
276 dataObj=returnedDataArray[0]
277 }
278
279
280 console.log( JSON.stringify(returnedDataArray[0]).length )
281
282 var nseDate = pathToStoreData.split(/\//)[1]
283
284 if( Date.parse(nseDate) == Date.parse(currentSystemDate) )
285 {
286 if ( iframeSrc.includes("api/allMarketStatus") == true )
287 {
288 //await db.doc( pathToStoreData ).set( dataObj );
289 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
290 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
291 }
292 else if ( iframeSrc.includes("api/quote-derivative?symbol") == true )
293 {
294 //await db.doc( pathToStoreData ).set( dataObj );
295 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
296 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
297
298 if(addedQuoteDerivatives==0)
299 { //this is for adding template of derivatives in index object
300 /*
301 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime+ '/' + createdAtTime+ '/' +'Template';
302 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
303
304 dataObj={
305 "data": ['open','high','low','close']
306 }
307
308 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
309 */
310
311
312 //updating index
313
314
315 var id = returnedDataArray[2]
316 var symbol = returnedDataArray[3]
317 var templateObj = returnedDataArray[4]
318 var allSymbolArray = returnedDataArray[5]
319
320 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime;
321 //pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
322 pathToStoreData = monthYear +'/'+ returnedDataArray[1]
323
324
325
326
327 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
328 var firstNestedFieldName = id+ '_index'
329 var secondNestedFieldName = 'time_values'
330 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
331 var fieldPath = firstNestedFieldName+'.'+ secondNestedFieldName
332 var templateFieldPath = firstNestedFieldName+'.'+ 'template'
333 var allSymbolsFieldPath = firstNestedFieldName+'.'+ 'all_symbols'
334 //var fieldPath = 'totalCEPEOIData.time.test'
335
336
337 //var totalCEPEData = {};
338 //totalCEPEData = returnedDataArray[0] // option chain all data
339 //totalCEPEData = JSON.stringify(returnedDataArray[0])// option chain all data
340 var totalCEPEData = createdAtTime// option chain all data
341
342
343 //totalCEPEData = symbol // option chain all data
344
345 //console.log("pathToStoreData", pathToStoreData)
346 //console.log("fieldPath", fieldPath)
347 //console.log("totalCEPEData", totalCEPEData)
348
349 var intraday_chart_data_Ref = db.doc( pathToStoreData );
350
351
352 // Try to update the existing document using the update() method
353 await intraday_chart_data_Ref.update({
354 [fieldPath]: firebase.firestore.FieldValue.arrayUnion(totalCEPEData),
355 [templateFieldPath]: JSON.stringify(templateObj),
356 [allSymbolsFieldPath]: allSymbolArray
357 }).then(() => {
358 console.log("Document updated successfully.");
359 }).catch((error) => {
360 // If the update() method fails with a "document does not exist" error,
361 // create a new document using the set() method instead
362 if (error.code === "not-found") {
363
364 var updateObject = {
365 [firstNestedFieldName]: {
366 [secondNestedFieldName]: firebase.firestore.FieldValue.arrayUnion(totalCEPEData),
367 'template': JSON.stringify(templateObj),
368 'all_symbols': allSymbolArray
369 }
370 }
371 intraday_chart_data_Ref.set( updateObject , { merge: true });
372
373 } else {
374 throw error;
375 }
376 }).then(() => {
377 console.log("Document created successfully.");
378 }).catch((error) => {
379 console.log("Error creating or updating document: ", error);
380 });
381
382
383
384
385 }
386
387 addedQuoteDerivatives = addedQuoteDerivatives+1
388
389
390 /*
391
392
393 var symbol = returnedDataArray[3]
394
395
396 pathOfDocument = returnedDataArray[2] + '/' +createdAtTime;
397 pathToStoreData = monthYear +'/'+ returnedDataArray[1] + '/' + pathOfDocument ;
398
399 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
400 var firstNestedFieldName = 'data'
401 var secondNestedFieldName = symbol
402 //var fieldPath = 'totalCEPEOIData.'+createdAtTime+'_'+( returnedDataArray[1].split(' ')[1] )
403 var fieldPath = firstNestedFieldName+'.'+ secondNestedFieldName
404 //var fieldPath = 'totalCEPEOIData.time.test'
405
406
407 var totalCEPEData = {};
408 //totalCEPEData = returnedDataArray[0] // option chain all data
409 totalCEPEData = JSON.stringify(returnedDataArray[0])// option chain all data
410 //totalCEPEData = symbol // option chain all data
411
412 //console.log("pathToStoreData", pathToStoreData)
413 //console.log("fieldPath", fieldPath)
414 //console.log("totalCEPEData", totalCEPEData)
415
416 var intraday_chart_data_Ref = db.doc( pathToStoreData );
417
418
419 // Try to update the existing document using the update() method
420 await intraday_chart_data_Ref.update({
421 [fieldPath]: totalCEPEData
422 }).then(() => {
423 console.log("Document updated successfully.");
424 }).catch((error) => {
425 // If the update() method fails with a "document does not exist" error,
426 // create a new document using the set() method instead
427 if (error.code === "not-found") {
428
429 var updateObject = {
430 [firstNestedFieldName]: {
431 [secondNestedFieldName]: totalCEPEData
432 }
433 }
434 intraday_chart_data_Ref.set( updateObject , { merge: true });
435
436 } else {
437 throw error;
438 }
439 }).then(() => {
440 console.log("Document created successfully.");
441 }).catch((error) => {
442 console.log("Error creating or updating document: ", error);
443 });
444
445 */
446
447
448
449
450
451 }
452
453
454 }
455 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
456 }
457 } //localStorageChecking_and_redirecting(apiObject,iframeSrc) function ENDS HERE
458
459
460
461async function updateIndex( db, createdAtTime, currentSystemDate )
462{
463
464 console.log('docPathForIndexObj', docPathForIndexObj);
465 try { // code that might throw an exception
466
467 var months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'];
468
469
470 var ApiMainObject = {};
471 var returnedDataArray ;
472 var pathToStoreData;
473 var pathOfDocument;
474 var dataObj;
475
476 var options = {
477 timeZone: 'Asia/Kolkata',
478 month: 'long',
479 year: 'numeric',
480 };
481
482
483 var allDates = []
484 for( var paths in docPathForIndexObj )
485 {
486 var pathDate = paths.split('/')[1]
487 allDates.push( pathDate )
488 }
489
490 var newestDate = new Date(allDates[0]);
491 var latestDate = allDates[0]
492 if (allDates.length > 1) {
493 for (var i = 1; i < allDates.length; i++) {
494 var currentDate = new Date(allDates[i]);
495 if (currentDate > newestDate) {
496 newestDate = currentDate;
497 latestDate = allDates[i]
498 }
499 }
500 }
501
502 var Index = {};
503
504 //var monthYear = months[newestDate.getMonth()] +' '+ newestDate.getFullYear();
505 var monthYear = new Date( latestDate ).toLocaleString('en-IN', options);
506
507 //var indexRef = db.doc('/April 2023/26-Apr-2023/');
508 var indexRef = db.doc('/'+monthYear+'/'+latestDate+'/');
509 await indexRef.get().then(function(querySnapshot) {
510 //console.log(doc.id, " => ", doc.data());
511 console.log( querySnapshot.data() )
512 if( querySnapshot.data() != undefined )
513 {
514 if( querySnapshot.data()['Index'] !=undefined )
515 Index = querySnapshot.data()['Index']
516 }
517
518
519 })
520 .catch(function(error) {
521 console.log("Error getting documents: ", error);
522 });
523
524 // (added on 21-May-2023) adding externally common_timeValues in docPathForIndexObj for index creation (STARTS HERE)
525 var nseDate = latestDate
526 if( Date.parse(nseDate) == Date.parse(currentSystemDate) )
527 {
528 var commonTimeValuesPathToStoreData = monthYear+'/'+latestDate+'/common_time_values/'+createdAtTime ;
529 docPathForIndexObj[ commonTimeValuesPathToStoreData ] = 'common_time_values/'+createdAtTime ;
530 }
531 // (added on 21-May-2023) adding externally common_timeValues in docPathForIndexObj for index creation (ENDS HERE)
532
533 if ( Index['delivery_data'] ==undefined )
534 {
535 var startDate = new Date( latestDate );
536 var startDateForDelivery = new Date(startDate.getFullYear(), startDate.getMonth(), startDate.getDate() - 1);
537 var calculatedDeliveryDataObj = await getCalculatedDeliveryData( startDateForDelivery )
538
539 if (calculatedDeliveryDataObj !== null) {
540 //console.log(fetchedData); // Process the retrieved data here
541 await addCalculatedDeliveryData( db, latestDate, calculatedDeliveryDataObj )
542
543 } else {
544 // Handle the error or fallback behavior here
545 //console.error('An error occurred while fetching the data');
546 }
547
548 }
549
550 console.log('docPathForIndexObj after deliveryData', docPathForIndexObj);
551 console.log("Index",Index)
552
553 for( var pathToStoreData in docPathForIndexObj )
554 {
555 try { // code that might throw an exception
556
557 var firstNestedFieldName, secondNestedFieldName, thirdNestedFieldName;
558 var timeValue;
559 var pathOfDocument;
560
561 //var nameOfSegment = pathOfDocument.split('/')[1]
562 var pathDate = paths.split('/')[1]
563 if( pathDate == latestDate )
564 {
565 console.log("pathToStoreData",pathToStoreData)
566
567 if( pathToStoreData.includes('option_chain_data')==true )
568 {
569 pathOfDocument = docPathForIndexObj[ pathToStoreData ]['pathOfDocument']
570 var expiriesArray = docPathForIndexObj[ pathToStoreData ]['expiryDates']
571
572 firstNestedFieldName = pathOfDocument.split('/')[0]
573 secondNestedFieldName = pathOfDocument.split('/')[1]
574 timeValue = pathOfDocument.split('/')[2]
575
576
577
578 console.log("pathOfDocument",pathOfDocument)
579 console.log("firstNestedFieldName",firstNestedFieldName)
580 console.log("secondNestedFieldName",secondNestedFieldName)
581 console.log("timeValue",timeValue)
582
583 console.log("Index[firstNestedFieldName]",Index[firstNestedFieldName])
584
585 if( Index[firstNestedFieldName] ==undefined )
586 {
587 Index[firstNestedFieldName] = { //e.g. option_chain_data
588 [secondNestedFieldName]:{ //eg. NIFTY50
589 "timeValues": [timeValue],
590 "expiryDates": expiriesArray
591 }
592 }
593 }
594 else
595 {
596 if( Index[firstNestedFieldName][secondNestedFieldName] ==undefined )
597 {
598 Index[firstNestedFieldName][secondNestedFieldName]={ //eg. NIFTY50
599 "timeValues": [timeValue],
600 "expiryDates": expiriesArray
601 }
602 }
603 else
604 {
605 Index[firstNestedFieldName][secondNestedFieldName]["timeValues"].push(timeValue)
606 Index[firstNestedFieldName][secondNestedFieldName]["expiryDates"] = expiriesArray
607 Index[firstNestedFieldName][secondNestedFieldName]["timeValues"] = sortTimesAsc( [...new Set( Index[firstNestedFieldName][secondNestedFieldName]["timeValues"] )] );
608 }
609 }
610
611 //Index[ firstNestedFieldName ] = [ timeValue ]
612 }
613
614 else if( ( pathToStoreData.includes('equity_market')==true && docPathForIndexObj[ pathToStoreData ].split('/').length>2 ) || pathToStoreData.includes('pre_open_market')==true )
615 {
616 pathOfDocument = docPathForIndexObj[ pathToStoreData ]
617
618 firstNestedFieldName = pathOfDocument.split('/')[0]
619 secondNestedFieldName = pathOfDocument.split('/')[1]
620 timeValue = pathOfDocument.split('/')[2]
621
622 if( Index[firstNestedFieldName] ==undefined )
623 {
624 Index[firstNestedFieldName] = { //e.g. option_chain_data
625 [secondNestedFieldName] : [timeValue]
626 }
627 }
628 else
629 {
630 if( Index[firstNestedFieldName][secondNestedFieldName] ==undefined )
631 {
632 Index[firstNestedFieldName][secondNestedFieldName] = [timeValue]
633 }
634 else
635 {
636 Index[firstNestedFieldName][secondNestedFieldName].push(timeValue)
637 Index[firstNestedFieldName][secondNestedFieldName] = sortTimesAsc( [...new Set( Index[firstNestedFieldName][secondNestedFieldName] )] )
638 }
639 }
640
641 //Index[ firstNestedFieldName ] = [ timeValue ]
642 }
643
644 //else if( pathToStoreData.includes('intraday_chart_data')==true )
645
646
647 //else if( pathToStoreData.includes('sector_stockNames')==true )
648
649 else if( pathToStoreData.includes('trading_dates')==false || (docPathForIndexObj[ pathToStoreData ].includes('trading_dates')==false ) )
650 {
651 pathOfDocument = docPathForIndexObj[ pathToStoreData ]
652 firstNestedFieldName = pathOfDocument.split('/')[0]
653 timeValue = pathOfDocument.split('/')[1]
654 //Index[ firstNestedFieldName ] = [ timeValue ]
655
656 if( Index[firstNestedFieldName] ==undefined )
657 {
658 Index[firstNestedFieldName] = [timeValue]
659 }
660 else
661 {
662 Index[firstNestedFieldName].push(timeValue)
663 Index[firstNestedFieldName] = sortTimesAsc( [...new Set( Index[firstNestedFieldName] )] )
664 }
665
666
667 }
668
669
670 }
671
672 } catch(error) {
673 // handle the error
674 console.error(error);
675 }
676
677 }
678
679 console.log("Index",Index)
680
681 pathToStoreData = monthYear +'/'+ latestDate ;
682
683 dataObj={
684 "Index": Index
685 }
686
687
688 //docPathForIndexObj [ pathToStoreData ] = pathOfDocument
689 await db.doc( pathToStoreData ).set( dataObj, { merge: true } );
690
691 } catch(error) {
692 // handle the error
693 console.log(error);
694}
695
696}
697
698
699
700function sortTimesAsc(times) {
701 times.sort((a, b) => {
702 // Convert time values to milliseconds since midnight
703 const [aHrs, aMins, aSecs] = a.split(":").map(Number);
704 const aMs = aHrs * 3600000 + aMins * 60000 + aSecs * 1000;
705
706 const [bHrs, bMins, bSecs] = b.split(":").map(Number);
707 const bMs = bHrs * 3600000 + bMins * 60000 + bSecs * 1000;
708
709 // Compare the milliseconds values
710 return aMs - bMs;
711 });
712
713 return times;
714}
715
716var equityData = {};
717var docPathForIndexObj= {};
718var onlyForSectorsStocksObj= {
719 dates:[],
720 data:{}
721};
722
723
724Apify.main(async () => {
725
726 try {
727 const input = await Apify.getInput();
728
729 const { data } = input;
730 let islegacyPhantomJSTask = false;
731 if (data && typeof data === 'string') {
732 // It runs from legacy phantomjs crawler task finished webhook
733 const legacyInput = JSON.parse(data);
734 Object.assign(input, legacyInput);
735 islegacyPhantomJSTask = true;
736 }
737
738 const { datasetId, apiKey, authDomain, projectId, collectionName } = input;
739
740 if (!datasetId) {
741 throw new Error('DatasetId is required on input.');
742 }
743
744 firebase.initializeApp({
745 apiKey,
746 authDomain,
747 projectId,
748 });
749
750 // Initialize Cloud Firestore through Firebase
751 const db = firebase.firestore();
752 console.log(`Start importing dataset ${datasetId} to firestore.`);
753 const dataset = await Apify.openDataset(datasetId, { forceCloud: true });
754 const datasetInfo = await dataset.getInfo();
755
756 //console.log('datasetInfo:', datasetInfo);
757
758 var createdAt = new Date( datasetInfo["createdAt"] )
759
760 var options = {
761 timeZone: 'Asia/Kolkata',
762 year: 'numeric',
763 month: '2-digit',
764 day: '2-digit',
765 hour: '2-digit',
766 minute: '2-digit',
767 second: '2-digit' ,
768 hour12: false
769 };
770
771 var dateOptions = {
772 timeZone: 'Asia/Kolkata',
773 year: 'numeric',
774 month: 'long', // Specify 'long' for full month name
775 day: '2-digit',
776};
777
778
779 //var createdAtTime = new Date( datasetInfo["createdAt"] ).toTimeString('en-IN', options).split(' ')[0]
780 var currentSystemDate = new Date( datasetInfo["createdAt"] ).toLocaleString('en-IN', dateOptions)
781 var createdAtTime = new Date( datasetInfo["createdAt"] ).toLocaleString('en-IN', options).split(' ')[1]
782
783 createdAtTime = createdAtTime.split(':')[0] + ':' + createdAtTime.split(':')[1]+ ':00'
784
785console.log("currentSystemDate", currentSystemDate, "createdAtTime",createdAtTime)
786
787 // Import dataset from actor/task
788 const limit = 1000;
789 let counter = 0;
790 for (let offset = 0; offset < datasetInfo.itemCount; offset += limit) {
791 const pagination = await dataset.getData({
792 simplified: islegacyPhantomJSTask,
793 clean: !islegacyPhantomJSTask,
794 limit,
795 offset,
796 });
797 console.log(`Get dataset items offset: ${pagination.offset}`);
798 var itemCount = 0
799
800 var combinedItemsArrayofObj = [
801 {
802 "url":"combinedArrayofObj",
803 "pageText":[]
804 }
805 ]
806
807 for(var dtaset=0;dtaset<pagination.items.length;dtaset++)
808 {
809 var dataArray =[]
810 dataArray = pagination.items[dtaset]["pageText"]
811 for(var a=0;a<dataArray.length;a++)
812 {
813
814 //concatedArray.concat(dataArray);
815 combinedItemsArrayofObj[0]['pageText'].push(dataArray[a])
816 }
817
818 }
819
820 //var itemsLength = Object.keys( pagination.items[0]["pageText"] ).length
821 var itemsLength = Object.keys( combinedItemsArrayofObj[0]["pageText"] ).length
822 //for (const item of pagination.items[0]["pageText"]) {
823 for (const item of combinedItemsArrayofObj[0]["pageText"]) {
824
825 itemCount = itemCount+1
826
827 var fetchedURL = item["url"];
828 var myurl = (item["url"].replace(/[^\w\s]/gi, ""));
829 console.log('item:', myurl );
830 console.log('createdAt:', createdAt );
831
832 try {
833 if( item["pageText"]!="" )
834 {
835 //var myPath= "March2022/10Mar2023/"+myurl+"/"+createdAtTime;
836 /*var myPath = TestFunction(myurl,createdAtTime);
837 //await db.collection(collectionName).doc( myurl ).set( JSON.parse(item["pageText"]) ); // Use doc() and set() instead of add()
838 //await db.doc( myPath ).set( JSON.parse(item["pageText"]) ); // Use doc() and set() instead of add()
839 var dataObj={
840 "objectValue":(item["pageText"])
841 }
842 await db.doc( myPath ).set( dataObj ); // Use doc() and set() instead of add()
843
844 //console.log('Added item:', item.replace(/[^\w\s]/gi, ''));
845 */
846
847 //await handleData( JSON.parse(item["pageText"]), fetchedURL, db, createdAtTime, currentSystemDate, itemsLength, itemCount );
848 await handleData( item["pageText"], fetchedURL, db, createdAtTime, currentSystemDate, itemsLength, itemCount );
849
850 counter++;
851 }
852
853 } catch (err) {
854 //console.log(`Cannot import item ${JSON.stringify(item)}: ${err.message}`);
855 console.log(`Cannot import item ${myurl}: ${err.message} : ${err}`);
856 }
857 }
858 }
859
860 //await addMargedsStocksQuoteDerivativesData( db, createdAtTime, currentSystemDate )
861
862 //await addMargedExtractedNiftyTotalMarket( db, createdAtTime, currentSystemDate )
863 //await addExtractedSectorStockNamesData( db, createdAtTime, currentSystemDate )
864 //await updateIndex( db, createdAtTime, currentSystemDate )
865
866 console.log(`Imported ${counter} from dataset ${datasetId}.`);
867
868 console.log('Done!');
869 console.log('docPathForIndexObj', docPathForIndexObj);
870 //console.log('onlyForSectorsStocksObj', onlyForSectorsStocksObj);
871
872
873 const datasetDrop = await dataset.drop();
874 console.log('datasetDrop', datasetDrop);
875
876
877
878 console.log( "allStocksFuturesOptionsDataObj Length:", JSON.stringify( allStocksFuturesOptionsDataObj ).length )
879 /*
880var storageSize = getStorageSize(allStocksFuturesOptionsDataObj );
881
882console.log("Bytes:", storageSize.bytes);
883console.log("Kilobytes:", storageSize.kilobytes);
884console.log("Megabytes:", storageSize.megabytes);
885console.log("Gigabytes:", storageSize.gigabytes);
886 console.log('allStocksFuturesOptionsDataObj', allStocksFuturesOptionsDataObj);
887*/
888
889 } catch (err) {
890 //console.log(`Cannot import item ${JSON.stringify(item)}: ${err.message}`);
891 console.log(`Error while running main ${err.message} : ${err}`);
892 }
893
894});package.json
1{
2 "name": "firestore-import",
3 "version": "0.0.1",
4 "description": "This is a boilerplate of an Apify actor.",
5 "dependencies": {
6 "apify": "^0.16.0",
7 "firebase": "^7.2.0"
8 },
9 "devDependencies": {
10 "@apify/eslint-config": "0.0.3",
11 "eslint": "^6.5.1"
12 },
13 "scripts": {
14 "start": "node main.js",
15 "test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
16 },
17 "author": "It's not you it's me",
18 "license": "ISC"
19}Developer
Maintained by Community
Categories
