Add cli support, prepare for v4.8 release.

python_client/.gitignore

@@ -7,4 +7,6 @@ Indices
 build/
 dist/
 generate_config.json
+stockd_debuglog.txt
+stockd_clilog.txt
 *.spec

python_client/StockD_Windows.spec

@@ -9,7 +9,7 @@ a = Analysis(['runner.py'],
              binaries=[],
              datas=[('app\\static', 'static'), ('app\\templates', 'templates')],
              hiddenimports=[],
-             hookspath=['C:\\Users\\virre\\miniconda3\\envs\\stockd_32bit\\lib\\site-packages\\cefpython3\\examples\\pyinstaller\\'],
+             hookspath=['C:\\Users\\virre\\anaconda3\\envs\\stockd_32\\lib\\site-packages\\cefpython3\\examples\\pyinstaller\\'],
              hooksconfig={},
              runtime_hooks=[],
              excludes=[],
@@ -38,3 +38,22 @@ exe = EXE(pyz,
           target_arch=None,
           codesign_identity=None,
           entitlements_file=None , icon='app\\static\\img\\favicon.ico')
+
+exe2 = EXE(pyz,
+          a.scripts,
+          a.binaries,
+          a.zipfiles,
+          a.datas,  
+          [],
+          name='StockD_Windows_with_cli_console',
+          debug=False,
+          bootloader_ignore_signals=False,
+          strip=False,
+          upx=True,
+          upx_exclude=[],
+          runtime_tmpdir=None,
+          console=True,
+          disable_windowed_traceback=False,
+          target_arch=None,
+          codesign_identity=None,
+          entitlements_file=None , icon='app\\static\\img\\favicon.ico')

python_client/app/routes.py

@@ -161,7 +161,6 @@ def process_eq(weblink, saveloc, d, get_delivery=None):
         except Exception as ex:
             getQ().put({'event': 'log', 'data': 'Delivery data unavailable on selected server.'})
             getLogger().info(str(ex))
-    getLogger().info('Could not reach here!.')
     df = df[['SYMBOL', 'DATE', 'OPEN', 'HIGH', 'LOW', 'CLOSE', 'VOLUME', 'OI']]
     df.to_csv(saveloc, header=None, index=None)
     return df
@@ -220,6 +219,7 @@ def _rename(item):
         elif keepall != 'false':
             return item.replace('NIFTY', 'NSE').replace(' ', '')
         else:
+            logger.error("Cannot find a symbol for {}. Enable keep others to keep the symbol.".format(item))
             return None
     # df['SYMBOL'] = df['SYMBOL'].apply(lambda x: x.replace(' ', '_'))
     df['SYMBOL'] = df['SYMBOL'].apply(_rename)
@@ -352,6 +352,14 @@ def saveConfigToDisk(main_config):
         json.dump(main_config, f)
         getLogger().info('Configuration save success')
 
+def process_aux_config(form_dict, main_config):
+    if 'auxConfig' in form_dict:
+        aux_config = json.loads(request.form['auxConfig'])
+        getLogger().info('Overriding saved config with ' + request.form['auxConfig'])
+        getQ().put({'event': 'log', 'data': 'Downloading with temporarily overriden config.'})
+        main_config = update(main_config, aux_config)
+    return main_config
+
 @app.route('/choose', methods=['POST'])
 def choose_path():
     dirs = app.winreference.create_file_dialog(webview.FOLDER_DIALOG)
@@ -387,6 +395,7 @@ def process_range():
             return
 
         main_config = loadConfigFromDisk()
+        main_config = process_aux_config(request.form, main_config)
 
         getQ().put({'event': 'log', 'data': '##### Using link Profile {} #####'.format(main_config['BASELINK']['stock_TYPE'])})
         getQ().put({'event': 'log', 'data': '======= Starting Downlad ======='})
@@ -420,7 +429,7 @@ def index():
 
 @app.route('/version')
 def version():
-    return "4.7"
+    return "4.8"
 
 @app.route('/test', methods=['POST'])
 def test():
@@ -433,12 +442,13 @@ def qadder(datapackage):
     getQ().put({'event': 'message', 'data': datapackage})
     return "Currently " + str(getQ().qsize()) + " events."
 
-@app.route('/getConfig', methods=['GET'])
+@app.route('/getConfig', methods=['GET', 'POST'])
 def getConfig():
     if not os.path.exists(os.path.join(app.static_folder, 'default_config.json')):
         abort(404)
 
     main_config = loadConfigFromDisk()
+    main_config = process_aux_config(request.form, main_config)
 
     return jsonify(main_config)
 
@@ -490,7 +500,7 @@ def saveConfig():
 def getstream():
     global SECURE_FLAG
     global TIMEOUT_DURATION
-    m = "";
+    m = ""
     main_config = None
     if not os.path.exists(os.path.join(app.static_folder, 'default_config.json')):
         m = "Configuration files missing!"

python_client/cliclient.py

@@ -0,0 +1,134 @@
+import json
+import collections
+import requests
+from sseclient import SSEClient
+from threading import Thread
+from bs4 import BeautifulSoup
+from blessed import Terminal
+import urllib.parse
+
+def threaded_sselistener(domain, terminal):
+    sse = SSEClient(domain + "/stream")
+    for event in sse:
+        if event.event == 'message' and event.data and len(event.data) > 0:
+            if event.data == 'stop':
+                return
+            print(terminal.magenta + event.data + terminal.normal)
+        elif event.event == 'progress':
+            if event.data == '-1':
+                print(terminal.magenta + "Couldn't download data!" + terminal.normal)
+            else:
+                print("{}Progress: {}{}%{}".format(terminal.orange, terminal.green, event.data, terminal.normal))
+        elif event.event == 'log':
+            print(event.data)
+
+def fetch_hierarchy(tree: dict, searchkey, final_val: dict, outkey):
+    if not isinstance(tree, collections.abc.Mapping):
+        return False
+
+    if searchkey in tree.keys():
+        final_val[outkey] = {searchkey: tree[searchkey]}
+        return True
+
+    for key in tree.keys():
+        if fetch_hierarchy(tree[key], searchkey, final_val, outkey):
+            final_val[outkey] = {key: final_val[outkey]}
+            return True
+
+    return False
+
+def merge_overrides(overrides: list):
+    output = {}
+    for dictionary in overrides:
+        for entry in dictionary:
+            output[entry] = dictionary[entry]
+    return output
+
+class CliClient:
+    def __init__(self, arguments, port):
+        self.args = arguments
+        self.port = port
+        self.domain = 'http://localhost:' + str(port)
+        self.rsession = requests.Session()
+        if self.args.quiet:
+            self.terminal = Terminal(force_styling=None)
+        else:
+            self.terminal = Terminal()
+        self.overrides = None
+
+    def print_news(self):
+        resp = self.rsession.get(self.domain + "/news")
+        soup = BeautifulSoup(resp.content, 'html.parser')
+        all_links = soup.find_all('a')
+        for link in all_links:
+            link.extract()
+        print(self.terminal.green2 + soup.get_text().strip() + self.terminal.normal)
+        for link in all_links:
+            text = link.get_text().strip()
+            if len(text) == 0 and link.find('img', alt=True) is not None:
+                text = link.find('img', alt=True)['alt']
+            print(self.terminal.link(link.get('href'), text, text))
+        print()
+
+    def print_version(self):
+        resp = self.rsession.get(self.domain + "/version")
+        print(self.terminal.cyan + "Your StockD Version --> " + self.terminal.magenta + resp.text + self.terminal.normal)
+
+    def print_config(self, config, overrides):
+        if not overrides:
+            resp = self.rsession.get(self.domain + "/getConfig")
+        else:
+            resp = self.rsession.post(self.domain + "/getConfig", data={"auxConfig": json.dumps(overrides)})
+        if config:
+            fval = {}
+            outkey = "output"
+            if fetch_hierarchy(resp.json(), config, fval, outkey):
+                if self.args.print_config_oneline:
+                    print(json.dumps(fval[outkey]).replace("\"", "\\\""))
+                else:
+                    print(json.dumps(fval[outkey], indent=3))
+            else:
+                print(self.terminal.red + "No entry for '{}' found in main config. Note that the keys are case sensitive.".format(config) + self.terminal.normal)
+        else:
+            print(json.dumps(resp.json(), indent=3))
+
+    def download(self):
+        download_payload = {
+            "fromDate": self.args.from_date.strftime("%Y-%m-%d"),
+            "toDate": self.args.to_date.strftime("%Y-%m-%d")
+        }
+        if self.overrides:
+            download_payload["auxConfig"] = json.dumps(self.overrides)
+        resp = self.rsession.post(self.domain + "/download", data=download_payload)
+        return self.terminal.cyan + resp.text + self.terminal.normal
+
+    def add_to_q(self, message):
+        self.rsession.get(self.domain + "/addToQueue/{}".format(urllib.parse.quote(message)))
+
+    def run(self):
+        final_output = "Processing Complete."
+        self.SSEListen()
+        if not self.args.skip_version_info:
+            self.print_version()
+        if not self.args.skip_news:
+            self.print_news()
+        if self.args.override_setting:
+            self.overrides = merge_overrides(self.args.override_setting)
+        if self.args.print_config is not None:
+            self.print_config(self.args.print_config, self.overrides)
+        if self.args.from_date or self.args.to_date:
+            if not self.args.from_date:
+                print(self.terminal.red + "From date is required for downloading." + self.terminal.normal)
+                return
+            if not self.args.to_date:
+                print(self.terminal.red + "To date is required for downloading." + self.terminal.normal)
+                return
+            final_output = self.download()
+        self.add_to_q("stop")
+        self.thread.join()
+        print(final_output)
+
+    def SSEListen(self):
+        self.thread = Thread(target = threaded_sselistener, args=(self.domain, self.terminal,))
+        self.thread.daemon = True
+        self.thread.start()

python_client/requirements.txt

@@ -1,5 +1,8 @@
 Flask==2.0.1
 pandas==1.1.5
 pyinstaller==4.5.1
-pywebview==3.5
+pywebview[cef]==3.5
 requests==2.26.0
+bs4==0.0.1
+sseclient==0.0.27
+blessed==1.20.0

python_client/runner.py

@@ -1,17 +1,62 @@
+import argparse
 import logging
 import webview
-import multiprocessing
 import threading
 import sys
 import random
+import json
 import socket
 import platform
-from contextlib import redirect_stdout
-from io import StringIO
+from datetime import datetime
 from app import app as server
 from werkzeug.serving import make_server
+from cliclient import CliClient
 
 logger = logging.getLogger(__name__)
+parser = argparse.ArgumentParser()
+
+if sys.platform.lower().startswith("win"):
+    import ctypes
+
+    def hideConsole():
+        """
+        Hides the console window in GUI mode. Necessary for frozen application, because
+        this application support both, command line processing AND GUI mode and theirfor
+        cannot be run via pythonw.exe.
+        """
+        whnd = ctypes.windll.kernel32.GetConsoleWindow()
+        if whnd != 0:
+            ctypes.windll.user32.ShowWindow(whnd, 0)
+
+    def showConsole():
+        """Unhides console window"""
+        whnd = ctypes.windll.kernel32.GetConsoleWindow()
+        if whnd != 0:
+            ctypes.windll.user32.ShowWindow(whnd, 1)
+
+class UnbufferedWriter:
+    def __init__(self, stream):
+        self.stream = stream
+        self.quiet = False
+        self.file_stream = open("stockd_clilog.txt", "w")
+
+    def write(self, data):
+        if not self.quiet:
+            self.stream.write(data)
+            self.stream.flush()
+        self.file_stream.write(data)
+        self.file_stream.flush()
+
+    def flush(self):
+        self.stream.flush()
+        self.file_stream.flush()
+
+def valid_date(date_str):
+    try:
+        return datetime.strptime(date_str, "%Y-%m-%d")
+    except ValueError:
+        msg = "not a valid date: {0!r}. Please enter date in YYYY-MM-DD format.".format(date_str)
+        raise argparse.ArgumentTypeError(msg)
 
 def _get_random_port():
     while True:
@@ -26,25 +71,49 @@ def _get_random_port():
             else:
                 return port
 
+# CLI arguments
+parser.add_argument("-s", "--from_date", help="Start date in YYYY-MM-DD format", type=valid_date)
+parser.add_argument("-e", "--to_date", help="End date in YYYY-MM-DD format", type=valid_date)
+parser.add_argument("--quiet", help="Don't print anything on console. Only prints to file and removes all color formatting.", action='store_true')
+parser.add_argument("--print_config", help="Print current configuration for given section. Will print all config, if section not specified", action="store", const='', nargs='?')
+parser.add_argument("--print_config_oneline", help="Use together with --print_config. Prints section without formatting", action="store_true")
+parser.add_argument("--skip_news", help="Don't print latest news", action="store_true")
+parser.add_argument("--override_setting", help="Override some settings temporarily. Edit and Save settings from GUI to for saving permanently. Any key that is shown via the --print_config option can be overridden by specifying the respective json hierarchy.",  nargs='*', type=json.loads)
+parser.add_argument("--skip_version_info", help="Don't print current version info", action="store_true")
 
 if __name__ == '__main__':
+    sys.stdout = UnbufferedWriter(sys.stdout)
+    sys.stderr = sys.stdout
+
+    p = _get_random_port()
+    srv = make_server('localhost', p, server, threaded=True)
+
+    x = threading.Thread(target=srv.serve_forever)
+    x.daemon = True
+    logger.warning("Running thread on {}".format(p))
+
+    x.start()
+
+    if (len(sys.argv) == 1):
+        print("Initializing engine. This console will be minimized once loading is complete.")
+
+        if sys.platform.lower().startswith('win'):
+            if getattr(sys, 'frozen', False):
+                hideConsole()
 
-    stream = StringIO()
-    with redirect_stdout(stream):
-        p = _get_random_port()
-        srv = make_server('localhost', p, server, threaded=True)
-        # x = multiprocessing.Process(target=srv.serve_forever)
-        x = threading.Thread(target=srv.serve_forever)
-        x.daemon = True
-        logger.warning("Running thread on {}".format(p))
-        # logger.warning("Static Path {}".format(server.static_folder))
-        x.start()
         window = webview.create_window(
             'StockD', 'http://localhost:{}'.format(p))
         server.winreference = window
         if platform.system() == "Windows":
             webview.start(gui='cef', debug=False)
         else:
             webview.start(debug=False)
-        # x.terminate()
-        sys.exit(0)
+
+    else:
+
+        args = parser.parse_args()
+        if args.quiet:
+            sys.stdout.quiet = True
+        CliClient(args, p).run()
+
+    sys.exit(0)