From c24965d74b53c6bb7524588e3949952899222ae5 Mon Sep 17 00:00:00 2001 From: spadakan Date: Fri, 12 Jan 2024 08:04:11 -0500 Subject: [PATCH] updated after JHutar comments --- opl/http.py | 7 ++++++- opl/investigator/config.py | 6 ++++-- opl/investigator/elasticsearch_loader.py | 21 ++++++++++----------- opl/pass_or_fail.py | 18 ++++++++---------- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/opl/http.py b/opl/http.py index 4ce0d16..2069c28 100644 --- a/opl/http.py +++ b/opl/http.py @@ -8,11 +8,16 @@ session = requests.Session() +def insecure(): + session.verify = False + logging.debug("Disabling insecure request warnings") + disable_insecure_request_warnings(True) + + def disable_insecure_request_warnings(disable_it): if disable_it: logging.debug("Disabling insecure request warnings") urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - session.verify = False def req(method, url, **kwargs): diff --git a/opl/investigator/config.py b/opl/investigator/config.py index 9731a85..08a9d0f 100644 --- a/opl/investigator/config.py +++ b/opl/investigator/config.py @@ -77,7 +77,7 @@ def load_config(conf, fp): conf.history_es_query = data["history"]["es_query"] if "es_server_user" in data["history"]: conf.es_server_user = data["history"]["es_server_user"] - conf.es_server_pass = data["history"]["es_server_pass"] + conf.es_server_pass_env_var = data["history"]["es_server_pass_env_var"] conf.es_server_verify = data["history"]["es_server_verify"] if conf.history_type == "sd_dir": @@ -94,7 +94,9 @@ def load_config(conf, fp): conf.decisions_es_index = data["decisions"]["es_index"] if "es_server_user" in data["decisions"]: conf.decisions_es_server_user = data["decisions"]["es_server_user"] - conf.decisions_es_server_pass = data["decisions"]["es_server_pass"] + conf.decisions_es_server_pass_env_var = data["decisions"][ + "decisions_es_server_pass_env_var" + ] conf.decisions_es_server_verify = data["decisions"]["es_server_verify"] if conf.decisions_type == "csv": conf.decisions_filename = data["decisions"]["filename"] diff --git a/opl/investigator/elasticsearch_loader.py b/opl/investigator/elasticsearch_loader.py index 7ca600d..8b67797 100644 --- a/opl/investigator/elasticsearch_loader.py +++ b/opl/investigator/elasticsearch_loader.py @@ -38,17 +38,16 @@ def load(server, index, query, paths, **kwargs): else: response = opl.http.get(url, headers=headers, json=data) - if response: - for item in response["hits"]["hits"]: - logging.debug( - f"Loading data from document ID {item['_id']} with field id={item['_source']['id'] if 'id' in item['_source'] else None} or parameters.run={item['_source']['parameters']['run'] if 'run' in item['_source']['parameters'] else None}" - ) - tmpfile = tempfile.NamedTemporaryFile(prefix=item["_id"], delete=False).name - sd = opl.status_data.StatusData(tmpfile, data=item["_source"]) - for path in paths: - tmp = sd.get(path) - if tmp is not None: - out[path].append(tmp) + for item in response["hits"]["hits"]: + logging.debug( + f"Loading data from document ID {item['_id']} with field id={item['_source']['id'] if 'id' in item['_source'] else None} or parameters.run={item['_source']['parameters']['run'] if 'run' in item['_source']['parameters'] else None}" + ) + tmpfile = tempfile.NamedTemporaryFile(prefix=item["_id"], delete=False).name + sd = opl.status_data.StatusData(tmpfile, data=item["_source"]) + for path in paths: + tmp = sd.get(path) + if tmp is not None: + out[path].append(tmp) logging.debug(f"Loaded {out}") return out diff --git a/opl/pass_or_fail.py b/opl/pass_or_fail.py index 4bb3377..c23ed02 100755 --- a/opl/pass_or_fail.py +++ b/opl/pass_or_fail.py @@ -114,16 +114,16 @@ def main(): if args.history_type == "csv": history = opl.investigator.csv_loader.load(args.history_file, args.sets) elif args.history_type == "elasticsearch": - if hasattr(args, "es_server_user"): - # if new elasticsearch credentials are provided, use them - opl.http.disable_insecure_request_warnings(args.es_server_verify) + if hasattr(args, "es_server_verify"): + # SSL verification is disabled by default + opl.http.insecure() history = opl.investigator.elasticsearch_loader.load( args.history_es_server, args.history_es_index, args.history_es_query, args.sets, es_server_user=getattr(args, "es_server_user", None), - es_server_pass=getattr(args, "es_server_pass", None), + es_server_pass=getattr(args, "es_server_pass_env_var", None), ) elif args.history_type == "sd_dir": @@ -206,17 +206,15 @@ def main(): if not args.dry_run: for d_type in args.decisions_type: if d_type == "elasticsearch": - if hasattr(args, "es_server_user"): - # if new elasticsearch credentials are provided, use them - opl.http.disable_insecure_request_warnings( - args.decisions_es_server_verify - ) + if hasattr(args, "es_server_verify"): + # disable SSL verification + opl.http.insecure() opl.investigator.elasticsearch_decisions.store( args.decisions_es_server, args.decisions_es_index, info_all, es_server_user=getattr(args, "decisions_es_server_user", None), - es_server_pass=getattr(args, "decisions_es_server_pass", None), + es_server_pass=getattr(args, "decisions_es_server_pass_env_var", None), ) if d_type == "csv": opl.investigator.csv_decisions.store(args.decisions_filename, info_all)