Merge pull request #25 from ncats/development

Development
ncats · Dec 16, 2020 · a130186 · a130186
2 parents 4810ad0 + 2701cc8
commit a130186
Show file tree

Hide file tree

Showing 7 changed files with 102 additions and 258 deletions.
diff --git a/server/app.py b/server/app.py
@@ -20,6 +20,7 @@
 from predictors.rlm.rlm_predictor import RLMPredictior
 from predictors.cyp450.cyp450_predictor import CYP450Predictor
 
+
 app = flask.Flask(__name__, static_folder ='./client')
 CORS(app)
 app.config["DEBUG"] = False

diff --git a/server/consensus_model.py b/server/consensus_model.py
diff --git a/server/predictors/cyp450/__init__.py b/server/predictors/cyp450/__init__.py
@@ -10,17 +10,8 @@
 from tqdm import tqdm
 import requests
 from io import BytesIO
-from concurrent.futures import ThreadPoolExecutor, as_completed
 from os import path
-
-cyp450_models_dict = {
-    'CYP2C9_inhib': {},
-    'CYP2C9_subs': {},
-    'CYP2D6_inhib': {},
-    'CYP2D6_subs': {},
-    'CYP3A4_inhib': {},
-    'CYP3A4_subs': {}
-}
+import multiprocessing
 
 # for model_name in cyp450_models_dict.keys():
 #     for model_number in range(0, 64):
@@ -40,29 +31,57 @@ def download_file(base_url, model_name, model_number, models_dict):
     ) as fout:
         for chunk in cyp450_rf_pkl_file_request.iter_content(chunk_size=4096):
             fout.write(chunk)
-    with open(cyp450_model_path, 'wb') as cyp450_rf_pkl_file:
-        cyp450_rf_pkl_file.write(cyp450_rf_pkl_file_request.content)
-    with open(cyp450_model_path, 'rb') as cyp450_rf_pkl_file:
-        cyp450_models_dict[model_name][f'model_{model_number}'] = pickle.load(cyp450_rf_pkl_file)
+    with open(cyp450_model_path, 'wb') as cyp450_rf_pkl_file_writer:
+        cyp450_rf_pkl_file_writer.write(cyp450_rf_pkl_file_request.content)
+
+    cyp450_rf_model = pickle.load(BytesIO(cyp450_rf_pkl_file_request.content))
+    return cyp450_rf_model
+    # with open(cyp450_model_path, 'r') as cyp450_rf_pkl_file_reader:
+    #     print(model_name)
+    #     print(model_number)
+    #     cyp450_models_dict[model_name][f'model_{model_number}'] = pickle.load(cyp450_rf_pkl_file_reader)
 
-def load_models(cyp450_models_dict):
+def load_models():
     # processes = []
-    with ThreadPoolExecutor() as executor:
-        base_url = 'https://tripod.nih.gov/pub/adme/models/CYPP450/'
-        print(f'Loading CYP450 random forest models', file=sys.stdout)
-        for model_name in tqdm(cyp450_models_dict.keys()):
-        # for model_name in cyp450_models_dict.keys():
-            for model_number in tqdm(range(0, 64)):
-            # for model_number in range(0, 64):
-                cyp450_model_path = f'./models/CYP450/{model_name}/model_{model_number}'
-                if path.exists(cyp450_model_path):
-                    with open(cyp450_model_path, 'rb') as pkl_file:
-                        cyp450_models_dict[model_name][f'model_{model_number}'] = pickle.load(pkl_file)
-                else:
-                    os.makedirs(f'./models/CYP450/{model_name}', exist_ok=True)
-                    # processes.append(executor.submit(download_file, base_url, model_name, model_number, cyp450_models_dict))
-                    download_file(base_url, model_name, model_number, cyp450_models_dict)
+    #with ThreadPoolExecutor() as executor:
+    base_url = 'https://tripod.nih.gov/pub/adme/models/CYPP450/'
+    print(f'Loading CYP450 random forest models', file=sys.stdout)
+
+    # manager = multiprocessing.Manager()
+
+    # cyp450_models_dict = manager.dict({
+    #     'CYP2C9_inhib': manager.dict(),
+    #     'CYP2C9_subs': manager.dict(),
+    #     'CYP2D6_inhib': manager.dict(),
+    #     'CYP2D6_subs': manager.dict(),
+    #     'CYP3A4_inhib': manager.dict(),
+    #     'CYP3A4_subs': manager.dict()
+    # })
+
+    cyp450_models_dict = {
+        'CYP2C9_inhib': {},
+        'CYP2C9_subs': {},
+        'CYP2D6_inhib': {},
+        'CYP2D6_subs': {},
+        'CYP3A4_inhib': {},
+        'CYP3A4_subs': {}
+    }
+
+    for model_name in tqdm(cyp450_models_dict.keys()):
+    # for model_name in cyp450_models_dict.keys():
+        for model_number in tqdm(range(0, 64)):
+        # for model_number in range(0, 64):
+            cyp450_model_path = f'./models/CYP450/{model_name}/model_{model_number}'
+            if path.exists(cyp450_model_path) and os.path.getsize(cyp450_model_path) > 0:
+                with open(cyp450_model_path, 'rb') as pkl_file:
+                    cyp450_models_dict[model_name][f'model_{model_number}'] = pickle.load(pkl_file)
+            else:
+                os.makedirs(f'./models/CYP450/{model_name}', exist_ok=True)
+                # processes.append(executor.submit(download_file, base_url, model_name, model_number, cyp450_models_dict))
+                cyp450_models_dict[model_name][f'model_{model_number}'] = download_file(base_url, model_name, model_number, cyp450_models_dict)
 
     print(f'Finished loading CYP450 model files', file=sys.stdout)
+    return cyp450_models_dict
+
 
-load_models(cyp450_models_dict)
+cyp450_models_dict = load_models()