diff --git a/python_client/.gitignore b/python_client/.gitignore index 14157f1..be495ad 100644 --- a/python_client/.gitignore +++ b/python_client/.gitignore @@ -7,4 +7,6 @@ Indices build/ dist/ generate_config.json +stockd_debuglog.txt +stockd_clilog.txt *.spec diff --git a/python_client/StockD_Windows.spec b/python_client/StockD_Windows.spec index e0e75c0..1e12fc0 100644 --- a/python_client/StockD_Windows.spec +++ b/python_client/StockD_Windows.spec @@ -9,7 +9,7 @@ a = Analysis(['runner.py'], binaries=[], datas=[('app\\static', 'static'), ('app\\templates', 'templates')], hiddenimports=[], - hookspath=['C:\\Users\\virre\\miniconda3\\envs\\stockd_32bit\\lib\\site-packages\\cefpython3\\examples\\pyinstaller\\'], + hookspath=['C:\\Users\\virre\\anaconda3\\envs\\stockd_32\\lib\\site-packages\\cefpython3\\examples\\pyinstaller\\'], hooksconfig={}, runtime_hooks=[], excludes=[], @@ -38,3 +38,22 @@ exe = EXE(pyz, target_arch=None, codesign_identity=None, entitlements_file=None , icon='app\\static\\img\\favicon.ico') + +exe2 = EXE(pyz, + a.scripts, + a.binaries, + a.zipfiles, + a.datas, + [], + name='StockD_Windows_with_cli_console', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + upx_exclude=[], + runtime_tmpdir=None, + console=True, + disable_windowed_traceback=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None , icon='app\\static\\img\\favicon.ico') diff --git a/python_client/app/routes.py b/python_client/app/routes.py index de3bdef..2177656 100644 --- a/python_client/app/routes.py +++ b/python_client/app/routes.py @@ -161,7 +161,6 @@ def process_eq(weblink, saveloc, d, get_delivery=None): except Exception as ex: getQ().put({'event': 'log', 'data': 'Delivery data unavailable on selected server.'}) getLogger().info(str(ex)) - getLogger().info('Could not reach here!.') df = df[['SYMBOL', 'DATE', 'OPEN', 'HIGH', 'LOW', 'CLOSE', 'VOLUME', 'OI']] df.to_csv(saveloc, header=None, index=None) return df @@ -220,6 +219,7 @@ def _rename(item): elif keepall != 'false': return item.replace('NIFTY', 'NSE').replace(' ', '') else: + logger.error("Cannot find a symbol for {}. Enable keep others to keep the symbol.".format(item)) return None # df['SYMBOL'] = df['SYMBOL'].apply(lambda x: x.replace(' ', '_')) df['SYMBOL'] = df['SYMBOL'].apply(_rename) @@ -352,6 +352,14 @@ def saveConfigToDisk(main_config): json.dump(main_config, f) getLogger().info('Configuration save success') +def process_aux_config(form_dict, main_config): + if 'auxConfig' in form_dict: + aux_config = json.loads(request.form['auxConfig']) + getLogger().info('Overriding saved config with ' + request.form['auxConfig']) + getQ().put({'event': 'log', 'data': 'Downloading with temporarily overriden config.'}) + main_config = update(main_config, aux_config) + return main_config + @app.route('/choose', methods=['POST']) def choose_path(): dirs = app.winreference.create_file_dialog(webview.FOLDER_DIALOG) @@ -387,6 +395,7 @@ def process_range(): return main_config = loadConfigFromDisk() + main_config = process_aux_config(request.form, main_config) getQ().put({'event': 'log', 'data': '##### Using link Profile {} #####'.format(main_config['BASELINK']['stock_TYPE'])}) getQ().put({'event': 'log', 'data': '======= Starting Downlad ======='}) @@ -420,7 +429,7 @@ def index(): @app.route('/version') def version(): - return "4.7" + return "4.8" @app.route('/test', methods=['POST']) def test(): @@ -433,12 +442,13 @@ def qadder(datapackage): getQ().put({'event': 'message', 'data': datapackage}) return "Currently " + str(getQ().qsize()) + " events." -@app.route('/getConfig', methods=['GET']) +@app.route('/getConfig', methods=['GET', 'POST']) def getConfig(): if not os.path.exists(os.path.join(app.static_folder, 'default_config.json')): abort(404) main_config = loadConfigFromDisk() + main_config = process_aux_config(request.form, main_config) return jsonify(main_config) @@ -490,7 +500,7 @@ def saveConfig(): def getstream(): global SECURE_FLAG global TIMEOUT_DURATION - m = ""; + m = "" main_config = None if not os.path.exists(os.path.join(app.static_folder, 'default_config.json')): m = "Configuration files missing!" diff --git a/python_client/cliclient.py b/python_client/cliclient.py new file mode 100644 index 0000000..01116dd --- /dev/null +++ b/python_client/cliclient.py @@ -0,0 +1,134 @@ +import json +import collections +import requests +from sseclient import SSEClient +from threading import Thread +from bs4 import BeautifulSoup +from blessed import Terminal +import urllib.parse + +def threaded_sselistener(domain, terminal): + sse = SSEClient(domain + "/stream") + for event in sse: + if event.event == 'message' and event.data and len(event.data) > 0: + if event.data == 'stop': + return + print(terminal.magenta + event.data + terminal.normal) + elif event.event == 'progress': + if event.data == '-1': + print(terminal.magenta + "Couldn't download data!" + terminal.normal) + else: + print("{}Progress: {}{}%{}".format(terminal.orange, terminal.green, event.data, terminal.normal)) + elif event.event == 'log': + print(event.data) + +def fetch_hierarchy(tree: dict, searchkey, final_val: dict, outkey): + if not isinstance(tree, collections.abc.Mapping): + return False + + if searchkey in tree.keys(): + final_val[outkey] = {searchkey: tree[searchkey]} + return True + + for key in tree.keys(): + if fetch_hierarchy(tree[key], searchkey, final_val, outkey): + final_val[outkey] = {key: final_val[outkey]} + return True + + return False + +def merge_overrides(overrides: list): + output = {} + for dictionary in overrides: + for entry in dictionary: + output[entry] = dictionary[entry] + return output + +class CliClient: + def __init__(self, arguments, port): + self.args = arguments + self.port = port + self.domain = 'http://localhost:' + str(port) + self.rsession = requests.Session() + if self.args.quiet: + self.terminal = Terminal(force_styling=None) + else: + self.terminal = Terminal() + self.overrides = None + + def print_news(self): + resp = self.rsession.get(self.domain + "/news") + soup = BeautifulSoup(resp.content, 'html.parser') + all_links = soup.find_all('a') + for link in all_links: + link.extract() + print(self.terminal.green2 + soup.get_text().strip() + self.terminal.normal) + for link in all_links: + text = link.get_text().strip() + if len(text) == 0 and link.find('img', alt=True) is not None: + text = link.find('img', alt=True)['alt'] + print(self.terminal.link(link.get('href'), text, text)) + print() + + def print_version(self): + resp = self.rsession.get(self.domain + "/version") + print(self.terminal.cyan + "Your StockD Version --> " + self.terminal.magenta + resp.text + self.terminal.normal) + + def print_config(self, config, overrides): + if not overrides: + resp = self.rsession.get(self.domain + "/getConfig") + else: + resp = self.rsession.post(self.domain + "/getConfig", data={"auxConfig": json.dumps(overrides)}) + if config: + fval = {} + outkey = "output" + if fetch_hierarchy(resp.json(), config, fval, outkey): + if self.args.print_config_oneline: + print(json.dumps(fval[outkey]).replace("\"", "\\\"")) + else: + print(json.dumps(fval[outkey], indent=3)) + else: + print(self.terminal.red + "No entry for '{}' found in main config. Note that the keys are case sensitive.".format(config) + self.terminal.normal) + else: + print(json.dumps(resp.json(), indent=3)) + + def download(self): + download_payload = { + "fromDate": self.args.from_date.strftime("%Y-%m-%d"), + "toDate": self.args.to_date.strftime("%Y-%m-%d") + } + if self.overrides: + download_payload["auxConfig"] = json.dumps(self.overrides) + resp = self.rsession.post(self.domain + "/download", data=download_payload) + return self.terminal.cyan + resp.text + self.terminal.normal + + def add_to_q(self, message): + self.rsession.get(self.domain + "/addToQueue/{}".format(urllib.parse.quote(message))) + + def run(self): + final_output = "Processing Complete." + self.SSEListen() + if not self.args.skip_version_info: + self.print_version() + if not self.args.skip_news: + self.print_news() + if self.args.override_setting: + self.overrides = merge_overrides(self.args.override_setting) + if self.args.print_config is not None: + self.print_config(self.args.print_config, self.overrides) + if self.args.from_date or self.args.to_date: + if not self.args.from_date: + print(self.terminal.red + "From date is required for downloading." + self.terminal.normal) + return + if not self.args.to_date: + print(self.terminal.red + "To date is required for downloading." + self.terminal.normal) + return + final_output = self.download() + self.add_to_q("stop") + self.thread.join() + print(final_output) + + def SSEListen(self): + self.thread = Thread(target = threaded_sselistener, args=(self.domain, self.terminal,)) + self.thread.daemon = True + self.thread.start() diff --git a/python_client/requirements.txt b/python_client/requirements.txt index c985cf0..1220b60 100644 --- a/python_client/requirements.txt +++ b/python_client/requirements.txt @@ -1,5 +1,8 @@ Flask==2.0.1 pandas==1.1.5 pyinstaller==4.5.1 -pywebview==3.5 +pywebview[cef]==3.5 requests==2.26.0 +bs4==0.0.1 +sseclient==0.0.27 +blessed==1.20.0 diff --git a/python_client/runner.py b/python_client/runner.py index 09a3efa..75cfe22 100644 --- a/python_client/runner.py +++ b/python_client/runner.py @@ -1,17 +1,62 @@ +import argparse import logging import webview -import multiprocessing import threading import sys import random +import json import socket import platform -from contextlib import redirect_stdout -from io import StringIO +from datetime import datetime from app import app as server from werkzeug.serving import make_server +from cliclient import CliClient logger = logging.getLogger(__name__) +parser = argparse.ArgumentParser() + +if sys.platform.lower().startswith("win"): + import ctypes + + def hideConsole(): + """ + Hides the console window in GUI mode. Necessary for frozen application, because + this application support both, command line processing AND GUI mode and theirfor + cannot be run via pythonw.exe. + """ + whnd = ctypes.windll.kernel32.GetConsoleWindow() + if whnd != 0: + ctypes.windll.user32.ShowWindow(whnd, 0) + + def showConsole(): + """Unhides console window""" + whnd = ctypes.windll.kernel32.GetConsoleWindow() + if whnd != 0: + ctypes.windll.user32.ShowWindow(whnd, 1) + +class UnbufferedWriter: + def __init__(self, stream): + self.stream = stream + self.quiet = False + self.file_stream = open("stockd_clilog.txt", "w") + + def write(self, data): + if not self.quiet: + self.stream.write(data) + self.stream.flush() + self.file_stream.write(data) + self.file_stream.flush() + + def flush(self): + self.stream.flush() + self.file_stream.flush() + +def valid_date(date_str): + try: + return datetime.strptime(date_str, "%Y-%m-%d") + except ValueError: + msg = "not a valid date: {0!r}. Please enter date in YYYY-MM-DD format.".format(date_str) + raise argparse.ArgumentTypeError(msg) def _get_random_port(): while True: @@ -26,19 +71,36 @@ def _get_random_port(): else: return port +# CLI arguments +parser.add_argument("-s", "--from_date", help="Start date in YYYY-MM-DD format", type=valid_date) +parser.add_argument("-e", "--to_date", help="End date in YYYY-MM-DD format", type=valid_date) +parser.add_argument("--quiet", help="Don't print anything on console. Only prints to file and removes all color formatting.", action='store_true') +parser.add_argument("--print_config", help="Print current configuration for given section. Will print all config, if section not specified", action="store", const='', nargs='?') +parser.add_argument("--print_config_oneline", help="Use together with --print_config. Prints section without formatting", action="store_true") +parser.add_argument("--skip_news", help="Don't print latest news", action="store_true") +parser.add_argument("--override_setting", help="Override some settings temporarily. Edit and Save settings from GUI to for saving permanently. Any key that is shown via the --print_config option can be overridden by specifying the respective json hierarchy.", nargs='*', type=json.loads) +parser.add_argument("--skip_version_info", help="Don't print current version info", action="store_true") if __name__ == '__main__': + sys.stdout = UnbufferedWriter(sys.stdout) + sys.stderr = sys.stdout + + p = _get_random_port() + srv = make_server('localhost', p, server, threaded=True) + + x = threading.Thread(target=srv.serve_forever) + x.daemon = True + logger.warning("Running thread on {}".format(p)) + + x.start() + + if (len(sys.argv) == 1): + print("Initializing engine. This console will be minimized once loading is complete.") + + if sys.platform.lower().startswith('win'): + if getattr(sys, 'frozen', False): + hideConsole() - stream = StringIO() - with redirect_stdout(stream): - p = _get_random_port() - srv = make_server('localhost', p, server, threaded=True) - # x = multiprocessing.Process(target=srv.serve_forever) - x = threading.Thread(target=srv.serve_forever) - x.daemon = True - logger.warning("Running thread on {}".format(p)) - # logger.warning("Static Path {}".format(server.static_folder)) - x.start() window = webview.create_window( 'StockD', 'http://localhost:{}'.format(p)) server.winreference = window @@ -46,5 +108,12 @@ def _get_random_port(): webview.start(gui='cef', debug=False) else: webview.start(debug=False) - # x.terminate() - sys.exit(0) + + else: + + args = parser.parse_args() + if args.quiet: + sys.stdout.quiet = True + CliClient(args, p).run() + + sys.exit(0)