Prepare for v4.9

virresh · Aug 4, 2024 · 35d79a4 · 35d79a4
1 parent ad11cd2
commit 35d79a4
Show file tree

Hide file tree

Showing 4 changed files with 46 additions and 15 deletions.
diff --git a/README.md b/README.md
@@ -18,12 +18,15 @@ If you find this software useful, consider [contributing](https://github.com/vir
 - Can use custom profiles for links and settings
 - Convert the EOD data into a common format which can be used by several professional softwares for charting
 - Can download delivery data for equity
+- CLI support (v4.8+ required).
 
 ## F.A.Q
 - EXE is detected as a Virus!  
 Given that you've downloaded the exe from official sources (the github release website or the official website, link above), the exe file should be virus free. Unfortunately many anti-viruses use inaccurate heuristics to detect viruses and will occasionally give you a false positive, especially when the exe is not signed. Since I'm an individual maintaining this software, I don't have enough funds to get the exe signed by a trusted provided, so if you run into this problem, I recommend you to make exceptions in your antivirus or compile the application from source (in case you don't trust the sources). I'll also add md5 hashes of the files so you can verify them post downloading to eliminate any possibility of virus injection.   
 - Application doesn't start up!  
 The application is slow to load when starting for the first time. Thus, please wait a while (~5 minutes) on the first load. If you get an error, or it doesn't start up, ensure you have latest [Microsoft Dot Net](https://dotnet.microsoft.com/download/dotnet-framework) version installed. If the application takes long time to start even after your first load, try making an exception fro StockD in your antivirus. Antiviruses might be interfering with any exe performing disk access and StockD is no exception. Using an SSD should also help speeding up program loading. Ideally it should start in ~5-10 seconds after the first load. Also, do not delete the `generate_config.json` file after the first load, otherwise the program will take the same time to start next time.  
+- Which version contains a CLI?  
+CLI is available in all binaries, but in order to take full advantage of cli, please use the with_cli_console version. The only difference between these two versions is that the with_cli_console version contains a console that shows the result of commands executed via cli. It is expected to have this console show up when operating this version in GUI mode as well. Since many people don't prefer that, I additionally provide a version with hidden console.
 - Who provides this data?  
 The data provided can be set using given profiles. All the current profiles fetch data from Official NSE website. You may be able to use other community-supplied profiles as well.
 - What is the Data Format?  

diff --git a/python_client/app/routes.py b/python_client/app/routes.py
@@ -74,8 +74,11 @@ def get_date(dString):
     # so this might just work
     type1 = r"%d-%b-%Y"
     type2 = r"%d-%m-%Y"
+    type3 = r"%Y-%m-%d"
     if re.search('[a-zA-Z]', dString):
         return datetime.datetime.strptime(dString, type1)
+    elif re.search('[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]', dString):
+        return datetime.datetime.strptime(dString, type3)
     else:
         return datetime.datetime.strptime(dString, type2)
 
@@ -130,13 +133,19 @@ def get_csv(weblink):
 def process_eq(weblink, saveloc, d, get_delivery=None):
     df = get_csv(weblink)
     df = df.replace('-', '0')
-    df = df[df['SERIES'].isin(['EQ', 'BE'])]
+    if 'SERIES' in df:
+        df = df[df['SERIES'].isin(['EQ', 'BE'])]
+    elif 'SctySrs' in df:
+        df = df[df['SctySrs'].isin(['EQ', 'BE'])]
+        df = df.sort_values('TckrSymb', kind='stable')
     dataDate = None
 
     if 'DATE1' in df.columns:
         dataDate = get_date(df['DATE1'].iloc[0])
     elif 'TIMESTAMP' in df.columns:
         dataDate = get_date(df['TIMESTAMP'].iloc[0])
+    elif 'TradDt' in df.columns:
+        dataDate = get_date(df['TradDt'].iloc[0])
 
     if not (parse(dataDate, '{0:%Y}{0:%m}{0:%d}') == parse(d, '{0:%Y}{0:%m}{0:%d}')):
         getLogger().error("Date Integrity check failed. Found date {} but expected {}. Skipping.".format(dataDate, d))
@@ -150,7 +159,13 @@ def process_eq(weblink, saveloc, d, get_delivery=None):
         'HIGH_PRICE': 'HIGH',
         'LOW_PRICE': 'LOW',
         'CLOSE_PRICE': 'CLOSE',
-        'DELIV_QTY': 'DELIVERY'
+        'DELIV_QTY': 'DELIVERY',
+        'TckrSymb': 'SYMBOL',
+        'TtlTradgVol': 'VOLUME',
+        'OpnPric': 'OPEN',
+        'HghPric': 'HIGH',
+        'LwPric': 'LOW',
+        'ClsPric': 'CLOSE',
     }
     df = df.rename(columns=cname_map)
     df['DATE'] = [parse(d, '{0:%Y}{0:%m}{0:%d}')] * len(df)
@@ -167,7 +182,25 @@ def process_eq(weblink, saveloc, d, get_delivery=None):
 
 def process_fu(weblink, saveloc, d, asPrefix=False):
     df = get_csv(weblink)
-    df = df[df['INSTRUMENT'].isin(['FUTIDX', 'FUTSTK'])]
+    # Separate out Futures and remove entries corresponding to options
+    if 'INSTRUMENT' in df:
+        df = df[df['INSTRUMENT'].isin(['FUTIDX', 'FUTSTK'])]
+    elif 'FinInstrmTp' in df:
+        df = df[df['FinInstrmTp'].isin(['IDF', 'STF'])]
+        df = df.sort_values(['TckrSymb', 'XpryDt'], kind='stable')
+
+    cname_map = {
+        'CONTRACTS': 'VOLUME',
+        'OPEN_INT': 'OI',
+        'TckrSymb': 'SYMBOL',
+        'TtlTradgVol': 'VOLUME',
+        'OpnPric': 'OPEN',
+        'HghPric': 'HIGH',
+        'LwPric': 'LOW',
+        'ClsPric': 'CLOSE',
+        'OpnIntrst': 'OI',
+    }
+    df = df.rename(columns=cname_map)
 
     last_symbol = None
     prefix = 'I'
@@ -184,11 +217,6 @@ def process_fu(weblink, saveloc, d, asPrefix=False):
         else:
             row['SYMBOL'] = row['SYMBOL'] + '-' + prefix
 
-    cname_map = {
-        'CONTRACTS': 'VOLUME',
-        'OPEN_INT': 'OI'
-    }
-    df = df.rename(columns=cname_map)
     df['DATE'] = [parse(d, '{0:%Y}{0:%m}{0:%d}')] * len(df)
     df = df[['SYMBOL', 'DATE', 'OPEN', 'HIGH', 'LOW', 'CLOSE', 'VOLUME', 'OI']]
     df.to_csv(saveloc, header=None, index=None)
@@ -429,7 +457,7 @@ def index():
 
 @app.route('/version')
 def version():
-    return "4.8"
+    return "4.9"
 
 @app.route('/test', methods=['POST'])
 def test():

diff --git a/python_client/app/static/default_config.json b/python_client/app/static/default_config.json
@@ -67,13 +67,13 @@
     },
     "LINKS": {
         "eqBhav":{
-            "link":"https://archives.nseindia.com/content/historical/EQUITIES/{0:%Y}/{0:%^b}/cm{0:%d}{0:%^b}{0:%Y}bhav.csv.zip"
+            "link":"https://www.nseindia.com/api/reports?archives=%5B%7B%22name%22%3A%22CM-UDiFF%20Common%20Bhavcopy%20Final%20(zip)%22%2C%22type%22%3A%22daily-reports%22%2C%22category%22%3A%22capital-market%22%2C%22section%22%3A%22equities%22%7D%5D&date={0:%d}-{0:%b}-{0:%Y}&type=equities&mode=single"
         },
         "fuBhav":{
-            "link":"https://archives.nseindia.com/content/historical/DERIVATIVES/{0:%Y}/{0:%^b}/fo{0:%d}{0:%^b}{0:%Y}bhav.csv.zip"
+            "link":"https://www.nseindia.com/api/reports?archives=%5B%7B%22name%22%3A%22F%26O%20-%20UDiFF%20Common%20Bhavcopy%20Final%20(zip)%22%2C%22type%22%3A%22archives%22%2C%22category%22%3A%22derivatives%22%2C%22section%22%3A%22equity%22%7D%5D&date={0:%d}-{0:%b}-{0:%Y}&type=equity&mode=single"
         },
         "indall":{
-            "link":"https://archives.nseindia.com/content/indices/ind_close_all_{0:%d}{0:%m}{0:%Y}.csv"
+            "link":"https://www.nseindia.com/api/reports?archives=%5B%7B%22name%22%3A%22Daily%20Snapshot%22%2C%22type%22%3A%22archives%22%2C%22category%22%3A%22capital-market%22%2C%22section%22%3A%22indices%22%7D%5D&date={0:%d}-{0:%b}-{0:%Y}&type=indices&mode=single"
         },
         "version": {
             "link": "https://docs.google.com/document/export?format=txt&id=1B_QqPYNQvTPN817DkDxESOv9AgjxUnhTCPP7rXFZxr4&token=AC4w5Vg1I7eEt3MFysH9EMgsVKxd1bCb4w%3A1593185289693"

diff --git a/python_client/app/templates/index.html b/python_client/app/templates/index.html
@@ -219,9 +219,9 @@ <h5 class="modal-title" id="exampleModalLabel">Modal title</h5>
                         <div class="col-sm-4">
                             <input type="hidden" id="linkProfile"/>
                             <select class="form-control" id="linkProfileEditable">
-                                <option data-cc="https://virresh.github.io/StockD/py/nse_live.json">NSE Daily (default)</option>
-                                <option data-cc="https://virresh.github.io/StockD/py/nse_with_delivery.json">NSE Daily (with Delivery)</option>
-                                <option data-cc="https://virresh.github.io/StockD/py/nse_archives.json">NSE Archives</option>
+                                <option data-cc="https://virresh.github.io/StockD/py/nse_live_8july2024.json">NSE Daily (default)</option>
+                                <option data-cc="https://virresh.github.io/StockD/py/nse_with_delivery_8july2024.json">NSE Daily (with Delivery)</option>
+                                <option data-cc="https://virresh.github.io/StockD/py/nse_archives_8july2024.json">NSE Archives</option>
                                 <option data-cc="https://virresh.github.io/StockD/py/nse_old.json">NSE Old</option>
                             </select>
                         </div>