Path improvements and new label additions. Updates ref #7 and #17

geru-scotland · Nov 12, 2024 · 5695f59 · 5695f59
1 parent 82e1de9
commit 5695f59
Show file tree

Hide file tree

Showing 24 changed files with 112,248 additions and 479,654 deletions.
diff --git a/app.py b/app.py
@@ -11,6 +11,7 @@
  * Description:
  *****************************************************
 """
+import os
 import re
 import sys
 from enum import Enum
@@ -20,6 +21,7 @@
 sys.path.append(str(Path(__file__).resolve().parents[2]))
 from models.flair.model import LoreNexusFlairModel
 from models.pytorch.model import LoreNexusPytorchModel
+from paths import BASE_DIR, get_checkpoints_dir
 
 class UserOptions(Enum):
     UNVEIL_LORE = 1
@@ -37,15 +39,16 @@ def __init__(self):
         """
         """
         config = self.load_config()
-        model_path = config['model']['pytorch']['path']
-        self.lore_nexus = LoreNexusPytorchModel(mode="cli_app", model_path=model_path)
+        model_name = config['models']['flair']['checkpoint']
+        model_path = f'{get_checkpoints_dir("flair")}/{model_name}'
+        self.lore_nexus = LoreNexusFlairModel(mode="cli_app", model_path=model_path)
         self.display_title()
         print("📜 The Lore Nexus is ready to unveil the mysteries of any name you provide 📜 ")
 
     def load_config(self):
         """
         """
-        config_path = "config.yaml"
+        config_path = os.path.join(BASE_DIR, "config.yaml")
         with open(config_path, 'r') as file:
             config = yaml.safe_load(file)
         return config

diff --git a/config.json b/config.json
@@ -2,33 +2,33 @@
     "datasets": [
         {
             "name": "Wikidata",
-            "path": "dataset/preprocessing/apis/wikidata",
+            "path": "wikidata",
             "input_folder": "raw_data",
             "output_folder": "processed_data",
+            "labels_folder": "labels",
+            "labels_file": "labels.txt",
             "dataset_file": "wikidata-universes.csv",
-            "processed_file": "wikidata_dataset_FastText.txt"
+            "output_file": "wikidata_dataset_FastText.txt"
         },
         {
             "name": "Mythdata",
-            "path": "dataset/preprocessing/apis/mythology",
+            "path": "mythology",
             "input_folder": "raw_data",
             "output_folder": "processed_data",
             "dataset_file": "myth_dataset.csv",
-            "output_file": "myth_dataset.txt",
-            "processed_file": "myth_dataset.txt"
+            "output_file": "myth_dataset.txt"
         },
         {
             "name": "NERdata",
-            "path": "dataset/preprocessing/ner",
+            "path": "ner",
             "input_folder": "raw_data",
             "output_folder": "processed_data",
-            "processed_file": "ner_dataset.txt"
+            "output_file": "ner_dataset.txt"
         }
     ],
 
     "data_processor": {
         "path": "dataset/preprocessing",
-        "output_folder": "output",
         "output_file": "dataset.txt",
         "train_file": "train.txt",
         "dev_file": "dev.txt",
@@ -46,6 +46,9 @@
             "ForgottenRealms",
             "FinalFantasy",
             "GameofThrones",
+            "TheWitcher",
+            "DoctorWho",
+            "Discworld",
             "Mythology"
         ]
     }

diff --git a/config.yaml b/config.yaml
@@ -1,5 +1,5 @@
-model:
+models:
   flair:
-    path: "models/flair/checkpoints/30_epochs_AWESOME/best-model.pt"
+    checkpoint: "../resources/taggers/universe_classifier/best-model.pt"
   pytorch:
-    path: "models/pytorch/checkpoints/best_model.pth"
+    checkpoint: "best_model.pth"