Fix RANO Annotation procedure (#577)

* Implement fix to annotation. Provide helper scripts for fixing issue

* Ignore hidden files when listing subjects/timepoints

* Increment medperf version

* Set version to expected value by users

* Start implementing tarball dialog

* Remove watchdog for loading annotations. Use button instead

* Fix scroll for Summary view

* WIP fix annotated local RANO dataset script

* finish annotated rano dataset fix script

* [WIP] Implement fix for bugged annotated rano tarballs

* Implement fix for annotated rano tarball

* Print number of corrected cases

* Remove fix dataset scripts to avoid misuse

* fix linter issues

* add destination path checks

cli/medperf/_version.py

@@ -1 +1 @@
-__version__ = "0.1.0"
+__version__ = "0.1.2"

scripts/get_dataset_hashes.py

@@ -0,0 +1,50 @@
+import hashlib
+import os
+import yaml
+
+from medperf import config
+from medperf.init import initialize
+from typer import Option
+
+
+def sha256sum(filename):
+    h = hashlib.sha256()
+    b = bytearray(128 * 1024)
+    mv = memoryview(b)
+    with open(filename, "rb", buffering=0) as f:
+        while n := f.readinto(mv):
+            h.update(mv[:n])
+    return h.hexdigest()
+
+
+def generate_hash_dict(path):
+    hash_dict = {}
+    contents = os.listdir(path)
+
+    for item in contents:
+        item_path = os.path.join(path, item)
+        if os.path.isdir(item_path):
+            hash_dict[item] = generate_hash_dict(item_path)
+        else:
+            hash_dict[item] = sha256sum(item_path)
+
+    return hash_dict
+
+
+def main(
+    dataset_uid: str = Option(None, "-d", "--dataset"),
+    output_file: str = Option("dataset_hashes.yaml", "-f", "--file"),
+):
+    initialize()
+    dset_path = os.path.join(config.datasets_folder, dataset_uid)
+
+    # Get hashes of tree
+    hash_dict = generate_hash_dict(dset_path)
+
+    # Write results to a file
+    with open(output_file, "w") as f:
+        yaml.dump(hash_dict, f)
+
+
+if __name__ == "__main__":
+    run(main)

scripts/get_reviewed_cases_hashes.py

@@ -0,0 +1,56 @@
+import tarfile
+import hashlib
+import shutil
+import os
+import yaml
+
+
+def sha256sum(filename):
+    h = hashlib.sha256()
+    b = bytearray(128 * 1024)
+    mv = memoryview(b)
+    with open(filename, "rb", buffering=0) as f:
+        while n := f.readinto(mv):
+            h.update(mv[:n])
+    return h.hexdigest()
+
+
+def generate_hash_dict(path):
+    hash_dict = {}
+    contents = os.listdir(path)
+
+    for item in contents:
+        item_path = os.path.join(path, item)
+        if os.path.isdir(item_path):
+            hash_dict[item] = generate_hash_dict(item_path)
+        else:
+            hash_dict[item] = sha256sum(item_path)
+
+    return hash_dict
+
+
+def main():
+    dst = ".reviewed_cases_contents"
+    hashes_file = "reviewed_cases_hashes.yaml"
+
+    # Create destination folder
+    shutil.rmtree(dst, ignore_errors=True)
+    os.makedirs(dst, exist_ok=True)
+
+    # Extract contents
+    with tarfile.open("reviewed_cases.tar.gz") as file:
+        file.extractall(dst)
+
+    # Get hashes of tree
+    hash_dict = generate_hash_dict(dst)
+
+    # Write results to a file
+    with open(hashes_file, "w") as f:
+        yaml.dump(hash_dict, f)
+
+    # Delete generated files and folders
+    shutil.rmtree(dst, ignore_errors=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/monitor/rano_monitor/__main__.py

@@ -7,13 +7,12 @@
     DEFAULT_STAGES_PATH,
     STAGES_HELP,
     DSET_LOC_HELP,
-    OUT_HELP
+    OUT_HELP,
 )
 from rano_monitor.dataset_browser import DatasetBrowser
 from rano_monitor.handlers import InvalidHandler
 from rano_monitor.handlers import PromptHandler
 from rano_monitor.handlers import ReportHandler, ReportState
-from rano_monitor.handlers import ReviewedHandler
 from rano_monitor.handlers import TarballReviewedHandler
 from rano_monitor.tarball_browser import TarballBrowser
 from typer import Option
@@ -40,13 +39,11 @@ def run_dset_app(dset_path, stages_path, output_path):
     report_state = ReportState(report_path, t_app)
     report_watchdog = ReportHandler(report_state)
     prompt_watchdog = PromptHandler(dset_data_path, t_app)
-    reviewed_watchdog = ReviewedHandler(dset_data_path, t_app)
     invalid_watchdog = InvalidHandler(invalid_path, t_app)
 
     t_app.set_vars(
         dset_data_path,
         stages_path,
-        reviewed_watchdog,
         output_path,
         invalid_path,
         invalid_watchdog,
@@ -56,7 +53,6 @@ def run_dset_app(dset_path, stages_path, output_path):
     observer = Observer()
     observer.schedule(report_watchdog, dset_path)
     observer.schedule(prompt_watchdog, os.path.join(dset_path, "data"))
-    observer.schedule(reviewed_watchdog, ".")
     observer.schedule(invalid_watchdog, os.path.dirname(invalid_path))
     observer.start()
     t_app.run()
@@ -89,13 +85,8 @@ def run_tarball_app(tarball_path):
 
 @app.command()
 def main(
-    dataset_uid: str = Option(None, "-d", "--dataset", help=DSET_HELP),
-    stages_path: str = Option(
-        DEFAULT_STAGES_PATH,
-        "-s",
-        "--stages",
-        help=STAGES_HELP
-    ),
+    dataset_uid: str = Option(..., "-d", "--dataset", help=DSET_HELP),
+    stages_path: str = Option(DEFAULT_STAGES_PATH, "-s", "--stages", help=STAGES_HELP),
     dset_path: str = Option(
         None,
         "-p",

scripts/monitor/rano_monitor/assets/subject-browser.tcss

@@ -71,11 +71,6 @@ SubjectDetails Button {
     column-span: 2;
 }
 
-SubjectDetails {
-    overflow-y: scroll;
-    height: 100%;
-}
-
 SubjectDetails CopyableItem {
     layout: grid;
     grid-size: 12 1;
@@ -121,7 +116,6 @@ Summary {
     padding: 3;
     align: center middle;
     content-align: center middle;
-    overflow-y: scroll;
 }
 
 Summary Static {
@@ -148,4 +142,9 @@ ListItem .subtitle {
 MarkdownViewer {
     height: auto;
     min-height: 2;
+}
+
+.review-btn {
+    width: 100%;
+    margin: 1;
 }

scripts/monitor/rano_monitor/constants.py

@@ -29,4 +29,5 @@
 LISTITEM_MAX_LEN = 30
 
 REVIEWED_PATTERN = r".*\/(.*)\/(.*)\/finalized\/(.*\.nii\.gz)"
+UNDER_REVIEW_PATTERN = r".*\/(.*)\/(.*)\/under_review\/(.*\.nii\.gz)"
 BRAINMASK_PATTERN = r".*\/(.*)\/(.*)\/brainMask_fused.nii.gz"

scripts/monitor/rano_monitor/dataset_browser.py

@@ -11,7 +11,7 @@
 from rano_monitor.widgets.summary import Summary
 from textual.app import App, ComposeResult
 from textual.binding import Binding
-from textual.containers import Container, Horizontal
+from textual.containers import Container, Horizontal, VerticalScroll
 from textual.reactive import reactive, var
 from textual.widgets import (
     Button,
@@ -42,15 +42,13 @@ def set_vars(
         self,
         dset_data_path,
         stages_path,
-        reviewed_watchdog,
         output_path,
         invalid_path,
         invalid_watchdog,
         prompt_watchdog,
     ):
         self.dset_data_path = dset_data_path
         self.stages_path = stages_path
-        self.reviewed_watchdog = reviewed_watchdog
         self.output_path = output_path
         self.invalid_path = invalid_path
         self.invalid_watchdog = invalid_watchdog
@@ -65,8 +63,9 @@ def compose(self) -> ComposeResult:
         with Container():
             with Container(id="list-container"):
                 yield SubjectListView(id="subjects-list")
-            yield Summary(id="summary")
-            yield SubjectDetails(id="details")
+            with VerticalScroll():
+                yield Summary(id="summary")
+                yield SubjectDetails(id="details")
         with Container(id="confirm-prompt"):
             yield Static(self.prompt, id="confirm-details")
             yield Horizontal(

scripts/monitor/rano_monitor/handlers/__init__.py

@@ -1,14 +1,12 @@
 from .invalid_handler import InvalidHandler
 from .prompt_handler import PromptHandler
 from .report_handler import ReportHandler, ReportState
-from .reviewed_handler import ReviewedHandler
 from .tarball_reviewed_watchdog import TarballReviewedHandler
 
 __all__ = [
     InvalidHandler,
     PromptHandler,
     ReportHandler,
     ReportState,
-    ReviewedHandler,
     TarballReviewedHandler,
 ]

scripts/monitor/rano_monitor/tarball_browser.py

@@ -50,10 +50,12 @@ def set_vars(self, contents_path):
 
     def __get_subjects(self):
         subjects = os.listdir(self.contents_path)
+        subjects = [subject for subject in subjects if not subject.startswith(".")]
         subject_timepoint_list = []
         for subject in subjects:
             subject_path = os.path.join(self.contents_path, subject)
             timepoints = os.listdir(subject_path)
+            timepoints = [timepoint for timepoint in timepoints if not timepoint.startswith(".")]
             subject_timepoint_list += [(subject, tp) for tp in timepoints]
 
         return subject_timepoint_list