updated after JHutar comments

opl/http.py

@@ -8,11 +8,16 @@
 session = requests.Session()
 
 
+def insecure():
+    session.verify = False
+    logging.debug("Disabling insecure request warnings")
+    disable_insecure_request_warnings(True)
+
+
 def disable_insecure_request_warnings(disable_it):
     if disable_it:
         logging.debug("Disabling insecure request warnings")
         urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
-        session.verify = False
 
 
 def req(method, url, **kwargs):

opl/investigator/config.py

@@ -77,7 +77,7 @@ def load_config(conf, fp):
         conf.history_es_query = data["history"]["es_query"]
         if "es_server_user" in data["history"]:
             conf.es_server_user = data["history"]["es_server_user"]
-            conf.es_server_pass = data["history"]["es_server_pass"]
+            conf.es_server_pass_env_var = data["history"]["es_server_pass_env_var"]
             conf.es_server_verify = data["history"]["es_server_verify"]
 
     if conf.history_type == "sd_dir":
@@ -94,7 +94,9 @@ def load_config(conf, fp):
         conf.decisions_es_index = data["decisions"]["es_index"]
         if "es_server_user" in data["decisions"]:
             conf.decisions_es_server_user = data["decisions"]["es_server_user"]
-            conf.decisions_es_server_pass = data["decisions"]["es_server_pass"]
+            conf.decisions_es_server_pass_env_var = data["decisions"][
+                "decisions_es_server_pass_env_var"
+            ]
             conf.decisions_es_server_verify = data["decisions"]["es_server_verify"]
     if conf.decisions_type == "csv":
         conf.decisions_filename = data["decisions"]["filename"]

opl/investigator/elasticsearch_loader.py

@@ -38,17 +38,16 @@ def load(server, index, query, paths, **kwargs):
     else:
         response = opl.http.get(url, headers=headers, json=data)
 
-    if response:
-        for item in response["hits"]["hits"]:
-            logging.debug(
-                f"Loading data from document ID {item['_id']} with field id={item['_source']['id'] if 'id' in item['_source'] else None} or parameters.run={item['_source']['parameters']['run'] if 'run' in item['_source']['parameters'] else None}"
-            )
-            tmpfile = tempfile.NamedTemporaryFile(prefix=item["_id"], delete=False).name
-            sd = opl.status_data.StatusData(tmpfile, data=item["_source"])
-            for path in paths:
-                tmp = sd.get(path)
-                if tmp is not None:
-                    out[path].append(tmp)
+    for item in response["hits"]["hits"]:
+        logging.debug(
+            f"Loading data from document ID {item['_id']} with field id={item['_source']['id'] if 'id' in item['_source'] else None} or parameters.run={item['_source']['parameters']['run'] if 'run' in item['_source']['parameters'] else None}"
+        )
+        tmpfile = tempfile.NamedTemporaryFile(prefix=item["_id"], delete=False).name
+        sd = opl.status_data.StatusData(tmpfile, data=item["_source"])
+        for path in paths:
+            tmp = sd.get(path)
+            if tmp is not None:
+                out[path].append(tmp)
 
     logging.debug(f"Loaded {out}")
     return out

opl/pass_or_fail.py

@@ -114,16 +114,16 @@ def main():
     if args.history_type == "csv":
         history = opl.investigator.csv_loader.load(args.history_file, args.sets)
     elif args.history_type == "elasticsearch":
-        if hasattr(args, "es_server_user"):
-            # if new elasticsearch credentials are provided, use them
-            opl.http.disable_insecure_request_warnings(args.es_server_verify)
+        if hasattr(args, "es_server_verify"):
+            # SSL verification is disabled by default
+            opl.http.insecure()
         history = opl.investigator.elasticsearch_loader.load(
             args.history_es_server,
             args.history_es_index,
             args.history_es_query,
             args.sets,
             es_server_user=getattr(args, "es_server_user", None),
-            es_server_pass=getattr(args, "es_server_pass", None),
+            es_server_pass=getattr(args, "es_server_pass_env_var", None),
         )
 
     elif args.history_type == "sd_dir":
@@ -206,17 +206,15 @@ def main():
     if not args.dry_run:
         for d_type in args.decisions_type:
             if d_type == "elasticsearch":
-                if hasattr(args, "es_server_user"):
-                    # if new elasticsearch credentials are provided, use them
-                    opl.http.disable_insecure_request_warnings(
-                        args.decisions_es_server_verify
-                    )
+                if hasattr(args, "es_server_verify"):
+                    # disable SSL verification
+                    opl.http.insecure()
                 opl.investigator.elasticsearch_decisions.store(
                     args.decisions_es_server,
                     args.decisions_es_index,
                     info_all,
                     es_server_user=getattr(args, "decisions_es_server_user", None),
-                    es_server_pass=getattr(args, "decisions_es_server_pass", None),
+                    es_server_pass=getattr(args, "decisions_es_server_pass_env_var", None),
                 )
             if d_type == "csv":
                 opl.investigator.csv_decisions.store(args.decisions_filename, info_all)