From e640a126f6e6453e6818b6dbb763c1173c3cc816 Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Wed, 11 Oct 2023 13:54:23 -0400 Subject: [PATCH 1/7] refactor: rename module from chartreview to chart_review Also deletes a duplicate method and refreshes pyproject.toml --- .gitignore | 2 ++ README.md | 13 +++++----- {chartreview => chart_review}/__init__.py | 0 {chartreview => chart_review}/agree.py | 4 +-- {chartreview => chart_review}/cohort.py | 10 ++++---- {chartreview => chart_review}/common.py | 0 .../covid_symptom/__init__.py | 0 .../covid_symptom/config.py | 2 +- .../covid_symptom/paper.py | 10 ++++---- {chartreview => chart_review}/kappa.py | 0 {chartreview => chart_review}/labelstudio.py | 0 {chartreview => chart_review}/mentions.py | 25 ------------------- {chartreview => chart_review}/publish.py | 10 ++++---- {chartreview => chart_review}/simplify.py | 4 +-- .../suicide_icd10/__init__.py | 0 .../suicide_icd10/config.py | 0 pyproject.toml | 11 +++----- 17 files changed, 33 insertions(+), 58 deletions(-) create mode 100644 .gitignore rename {chartreview => chart_review}/__init__.py (100%) rename {chartreview => chart_review}/agree.py (99%) rename {chartreview => chart_review}/cohort.py (96%) rename {chartreview => chart_review}/common.py (100%) rename {chartreview => chart_review}/covid_symptom/__init__.py (100%) rename {chartreview => chart_review}/covid_symptom/config.py (97%) rename {chartreview => chart_review}/covid_symptom/paper.py (95%) rename {chartreview => chart_review}/kappa.py (100%) rename {chartreview => chart_review}/labelstudio.py (100%) rename {chartreview => chart_review}/mentions.py (79%) rename {chartreview => chart_review}/publish.py (81%) rename {chartreview => chart_review}/simplify.py (98%) rename {chartreview => chart_review}/suicide_icd10/__init__.py (100%) rename {chartreview => chart_review}/suicide_icd10/config.py (100%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4358842 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/.idea/ +__pycache__/ diff --git a/README.md b/README.md index 8938cc4..03f048d 100644 --- a/README.md +++ b/README.md @@ -55,17 +55,18 @@ Enum **NoteRanges** maps a selection of NoteID from the corpus **BASE COHORT METHODS** `cohort.py` -* from chartreview import _labelstudio_, _mentions_, _agree_ +* from chart_review import _labelstudio_, _mentions_, _agree_ class **Cohort** defines the base class to analyze study cohorts. * init(`config.py`) -`mentions.py` +`simplify.py` * **rollup**(...) : return _LabelStudioExport_ with 1 "rollup" annotation replacing individual mentions -* other methods are rarely used currently - * overlaps(...) : test if two mentions overlap (True/False) - * calc_term_freq(...) : term frequency of highlighted mention text - * calc_term_label_confusion : report of exact mentions with 2+ class_labels + +`mentions.py` (methods are rarely used currently) +* overlaps(...) : test if two mentions overlap (True/False) +* calc_term_freq(...) : term frequency of highlighted mention text +* calc_term_label_confusion : report of exact mentions with 2+ class_labels `agree.py` get confusion matrix comparing annotators {truth, reviewer} * **confusion_matrix** (truth, reviewer, ...) returns List[TruePos, TrueNeg, FalsePos, FalseNeg] diff --git a/chartreview/__init__.py b/chart_review/__init__.py similarity index 100% rename from chartreview/__init__.py rename to chart_review/__init__.py diff --git a/chartreview/agree.py b/chart_review/agree.py similarity index 99% rename from chartreview/agree.py rename to chart_review/agree.py index 5500d4d..990a6e5 100644 --- a/chartreview/agree.py +++ b/chart_review/agree.py @@ -1,8 +1,8 @@ from typing import Dict, List from collections.abc import Iterable from ctakesclient.typesystem import Span -from chartreview import mentions -from chartreview import simplify +from chart_review import mentions +from chart_review import simplify def confusion_matrix(simple: dict, gold_ann: str, review_ann: str, note_range: Iterable, label_pick=None) -> Dict[str, list]: """ diff --git a/chartreview/cohort.py b/chart_review/cohort.py similarity index 96% rename from chartreview/cohort.py rename to chart_review/cohort.py index fe2284e..92bf3bb 100644 --- a/chartreview/cohort.py +++ b/chart_review/cohort.py @@ -1,11 +1,11 @@ from typing import List from collections.abc import Iterable from enum import Enum, EnumMeta -from chartreview.common import guard_str, guard_iter, guard_in -from chartreview import common -from chartreview import simplify -from chartreview import mentions -from chartreview import agree +from chart_review.common import guard_str, guard_iter, guard_in +from chart_review import common +from chart_review import simplify +from chart_review import mentions +from chart_review import agree class CohortReader: diff --git a/chartreview/common.py b/chart_review/common.py similarity index 100% rename from chartreview/common.py rename to chart_review/common.py diff --git a/chartreview/covid_symptom/__init__.py b/chart_review/covid_symptom/__init__.py similarity index 100% rename from chartreview/covid_symptom/__init__.py rename to chart_review/covid_symptom/__init__.py diff --git a/chartreview/covid_symptom/config.py b/chart_review/covid_symptom/config.py similarity index 97% rename from chartreview/covid_symptom/config.py rename to chart_review/covid_symptom/config.py index 353e538..41ceb4b 100644 --- a/chartreview/covid_symptom/config.py +++ b/chart_review/covid_symptom/config.py @@ -1,6 +1,6 @@ from enum import Enum import ctakesclient -from chartreview import common +from chart_review import common ############################################################################### # COVID Symptom Class Labels diff --git a/chartreview/covid_symptom/paper.py b/chart_review/covid_symptom/paper.py similarity index 95% rename from chartreview/covid_symptom/paper.py rename to chart_review/covid_symptom/paper.py index 5ca5626..22d2250 100644 --- a/chartreview/covid_symptom/paper.py +++ b/chart_review/covid_symptom/paper.py @@ -1,8 +1,8 @@ -from chartreview.covid_symptom import config -from chartreview.covid_symptom.config import Annotator, NoteRange -from chartreview import agree -from chartreview import cohort -from chartreview import common +from chart_review.covid_symptom import config +from chart_review.covid_symptom.config import Annotator, NoteRange +from chart_review import agree +from chart_review import cohort +from chart_review import common def table2_accuracy_ctakes(self): gold_ann = Annotator.amy diff --git a/chartreview/kappa.py b/chart_review/kappa.py similarity index 100% rename from chartreview/kappa.py rename to chart_review/kappa.py diff --git a/chartreview/labelstudio.py b/chart_review/labelstudio.py similarity index 100% rename from chartreview/labelstudio.py rename to chart_review/labelstudio.py diff --git a/chartreview/mentions.py b/chart_review/mentions.py similarity index 79% rename from chartreview/mentions.py rename to chart_review/mentions.py index d4bf61d..8555d61 100644 --- a/chartreview/mentions.py +++ b/chart_review/mentions.py @@ -1,29 +1,4 @@ -from collections import Iterable - from ctakesclient.typesystem import Span -from chartreview.common import guard_str, guard_iter - -def rollup_mentions(simple: dict, annotator: str, note_range: Iterable) -> dict: - """ - @param simple: prepared map of files and annotations - @param annotator: like andy, amy, or alon - @param note_range: collection of LabelStudio document ID - @return: dict keys=note_id, values=labels - """ - rollup = dict() - - for note_id, values in simple['annotations'].items(): - if int(note_id) in guard_iter(note_range): - if values.get(annotator): - for annot in values[guard_str(annotator)]: - if not rollup.get(note_id): - rollup[note_id] = list() - - symptom = annot['labels'][0] - - if symptom not in rollup[note_id]: - rollup[note_id].append(symptom) - return rollup def calc_term_freq(simple: dict, annotator: str) -> dict: """ diff --git a/chartreview/publish.py b/chart_review/publish.py similarity index 81% rename from chartreview/publish.py rename to chart_review/publish.py index 313d0d3..c275fd9 100644 --- a/chartreview/publish.py +++ b/chart_review/publish.py @@ -1,11 +1,11 @@ from typing import List from collections.abc import Iterable from enum import Enum, EnumMeta -from chartreview.common import guard_str, guard_iter, guard_in -from chartreview import common -from chartreview import simplify -from chartreview import mentions -from chartreview import agree +from chart_review.common import guard_str, guard_iter, guard_in +from chart_review import common +from chart_review import simplify +from chart_review import mentions +from chart_review import agree def score_reviewer_table_csv(self, gold_ann, review_ann, note_range) -> str: table = list() diff --git a/chartreview/simplify.py b/chart_review/simplify.py similarity index 98% rename from chartreview/simplify.py rename to chart_review/simplify.py index 08e5571..9c93b17 100644 --- a/chartreview/simplify.py +++ b/chart_review/simplify.py @@ -1,8 +1,8 @@ from collections.abc import Iterable from enum import EnumMeta import re -from chartreview import common -from chartreview.common import guard_str, guard_iter +from chart_review import common +from chart_review.common import guard_str, guard_iter def merge_simple(source: dict, append: dict) -> dict: """ diff --git a/chartreview/suicide_icd10/__init__.py b/chart_review/suicide_icd10/__init__.py similarity index 100% rename from chartreview/suicide_icd10/__init__.py rename to chart_review/suicide_icd10/__init__.py diff --git a/chartreview/suicide_icd10/config.py b/chart_review/suicide_icd10/config.py similarity index 100% rename from chartreview/suicide_icd10/config.py rename to chart_review/suicide_icd10/config.py diff --git a/pyproject.toml b/pyproject.toml index d6d6e96..f945b78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,7 @@ [project] -name = "cumulus-chart-review" +name = "chart-review" version = "0.0.1" -# In order to support 3.12, we wil need to refactor out load_module, which is -# targeted for deprecation in that version. -requires-python = ">= 3.9, <3.12" +requires-python = ">= 3.9" dependencies = [ "ctakesclient", ] @@ -20,8 +18,7 @@ classifiers = [ [project.urls] Home = "https://smarthealthit.org/cumulus-a-universal-sidecar-for-a-smart-learning-healthcare-system/" Documentation = "https://docs.smarthealthit.org/cumulus/" -Source = "https://github.com/smart-on-fhir/cumulus-chart-review" - +Source = "https://github.com/smart-on-fhir/chart-review" [build-system] requires = ["flit_core >=3.4,<4"] @@ -34,4 +31,4 @@ dev = [ ] [tool.flit.sdist] -include = ["chartreview/"] +include = ["chart_review/"] From 9429b8b51b0a7beac794cac02201fcaa45b3610c Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Thu, 12 Oct 2023 14:17:56 -0400 Subject: [PATCH 2/7] feat: read from a yaml/json config file and add CLI CLI: - New `chart-review` script gets installed along with Python module. - One sub-command right now: `accuracy` which calculates accuracy matrixes across labels for two reviewers and a base third Config: - Switch away from Python config files and towards yaml/json files. - I've added yaml versions of the two studies in the repo, as examples. --- README.md | 37 +++++++++- chart_review/cli.py | 98 ++++++++++++++++++++++++++ chart_review/cohort.py | 32 +++------ chart_review/config.py | 67 ++++++++++++++++++ chart_review/covid_symptom/config.yaml | 31 ++++++++ chart_review/simplify.py | 6 +- chart_review/suicide_icd10/config.yaml | 19 +++++ pyproject.toml | 4 ++ 8 files changed, 266 insertions(+), 28 deletions(-) create mode 100644 chart_review/cli.py create mode 100644 chart_review/config.py create mode 100644 chart_review/covid_symptom/config.yaml create mode 100644 chart_review/suicide_icd10/config.yaml diff --git a/README.md b/README.md index 03f048d..9270fb4 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,42 @@ The most common chart-review measures agreement of the _**class_label**_ from a * 2 human reviewers _vs_ each other --- -**EACH STUDY HAS STUDY-SPECIFIC COHORT** +### How to Install +1. Clone this repo. +2. Install it locally like so: `pipx install .` + +This is not released on PyPI yet. + +--- +### How to Run + +#### Set Up Project Folder + +Chart Review operates on a project folder that holds your config & data. +1. Make a new folder. +2. Export your Label Studio annotations and put that in the folder as `labelstudio-export.json`. +3. Add a `config.yaml` file (or `config.json`) that looks something like this (read more on this format below): + +```yaml +class-labels: + - cough + - fever + +annotators: + jane: 2 + john: 6 +``` + +#### Run + +Simply call `chart-review` with your project folder: + +`chart-review /path/to/project/dir` + +Pass `--help` to see more options. + +--- +### Config File Format `config.py` defines study specific variables. diff --git a/chart_review/cli.py b/chart_review/cli.py new file mode 100644 index 0000000..d66bfe6 --- /dev/null +++ b/chart_review/cli.py @@ -0,0 +1,98 @@ +"""Run chart-review from the command-line""" + +import argparse +import os +import sys + +from chart_review import agree, cohort, common + + +############################################################################### +# +# CLI Helpers +# +############################################################################### + +def add_project_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + "--project-dir", + default=".", + help="Directory holding project files, like config.yaml and labelstudio-export.json (default: current dir)", + ) + + +def define_parser() -> argparse.ArgumentParser: + """Fills out an argument parser with all the CLI options.""" + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers(required=True) + + add_accuracy_subparser(subparsers) + + return parser + + +############################################################################### +# +# Accuracy +# +############################################################################### + +def add_accuracy_subparser(subparsers) -> None: + parser = subparsers.add_parser("accuracy") + add_project_args(parser) + parser.add_argument("one") + parser.add_argument("two") + parser.add_argument("base") + parser.set_defaults(func=run_accuracy) + + +def run_accuracy(args: argparse.Namespace) -> None: + reader = cohort.CohortReader(args.project_dir) + + first_ann = args.one + second_ann = args.two + base_ann = args.base + + # Grab ranges + first_range = reader.config.note_ranges[first_ann] + second_range = reader.config.note_ranges[second_ann] + + # All labels first + first_matrix = reader.confusion_matrix(first_ann, base_ann, first_range) + second_matrix = reader.confusion_matrix(second_ann, base_ann, second_range) + whole_matrix = agree.append_matrix(first_matrix, second_matrix) + table = agree.score_matrix(whole_matrix) + + # Now do each labels separately + for label in reader.class_labels: + first_matrix = reader.confusion_matrix(first_ann, base_ann, first_range, label) + second_matrix = reader.confusion_matrix(second_ann, base_ann, second_range, label) + whole_matrix = agree.append_matrix(first_matrix, second_matrix) + table[label] = agree.score_matrix(whole_matrix) + + # And write out the results + output_stem = os.path.join(reader.project_dir, f"accuracy-{first_ann}-{second_ann}-{base_ann}") + common.write_json(f"{output_stem}.json", table) + print(f"Wrote {output_stem}.json") + common.write_text(f"{output_stem}.csv", agree.csv_table(table, reader.class_labels)) + print(f"Wrote {output_stem}.csv") + + +############################################################################### +# +# Main CLI entrypoints +# +############################################################################### + +def main_cli() -> None: + """Main entrypoint that wraps all the core program logic""" + try: + parser = define_parser() + args = parser.parse_args() + args.func(args) + except Exception as exc: + sys.exit(str(exc)) + + +if __name__ == "__main__": + main_cli() diff --git a/chart_review/cohort.py b/chart_review/cohort.py index 92bf3bb..4620e4d 100644 --- a/chart_review/cohort.py +++ b/chart_review/cohort.py @@ -1,30 +1,26 @@ -from typing import List +import os from collections.abc import Iterable -from enum import Enum, EnumMeta from chart_review.common import guard_str, guard_iter, guard_in from chart_review import common +from chart_review import config from chart_review import simplify from chart_review import mentions from chart_review import agree class CohortReader: - def __init__(self, project_dir: str, annotator: EnumMeta, note_range: EnumMeta, class_labels: List[str]): + def __init__(self, project_dir: str): """ :param project_dir: str like /opt/labelstudio/study_name - :param annotator: Enum.name is human-readable name like "rena" and Enum.value is LabelStudio "complete_by" - :param note_range: Enum.name is human-readable name like "andy_alon" and Enum.value is LabelStudio "annotation.id" - :param class_labels: defined by "clinical annotation guidelines" """ self.project_dir = project_dir + self.config = config.ProjectConfig(project_dir) self.labelstudio_json = self.path('labelstudio-export.json') #TODO: refactor labelstudio.py - self.annotator = annotator - self.note_range = note_range - self.class_labels = class_labels + self.annotator = self.config.annotators + self.note_range = self.config.note_ranges + self.class_labels = self.config.class_labels self.annotations = None - common.print_line(f'Loading(...) \n {self.labelstudio_json}') - saved = common.read_json(self.labelstudio_json) if isinstance(saved, list): self.annotations = simplify.simplify_full(self.labelstudio_json, self.annotator) @@ -38,7 +34,7 @@ def __init__(self, project_dir: str, annotator: EnumMeta, note_range: EnumMeta, self.annotations = compat def path(self, filename): - return f'{self.project_dir}/{filename}' + return os.path.join(self.project_dir, filename) def calc_term_freq(self, annotator) -> dict: """ @@ -113,15 +109,3 @@ def score_reviewer_table_dict(self, gold_ann, review_ann, note_range) -> dict: table[label] = self.score_reviewer(gold_ann, review_ann, note_range, label) return table - - def get_config(self) -> dict: - as_dict = dict() - as_dict['class_labels'] = self.class_labels - as_dict['project_dir'] = self.project_dir - as_dict['annotation_file'] = self.labelstudio_json - as_dict['annotator'] = {i.name: i.value for i in self.annotator} - as_dict['note_range'] = {i.name: ','.join([str(j) for j in list(i.value)]) for i in self.note_range} - return as_dict - - def write_config(self): - common.write_json(self.path('config.json'), self.get_config()) \ No newline at end of file diff --git a/chart_review/config.py b/chart_review/config.py new file mode 100644 index 0000000..3043351 --- /dev/null +++ b/chart_review/config.py @@ -0,0 +1,67 @@ +import itertools +import os +import re +import sys +from typing import Iterable + +import yaml + +AnnotatorMap = dict[int, str] + + +class ProjectConfig: + + _NUMBER_REGEX = re.compile(r"\d+") + _RANGE_REGEX = re.compile(r"\d+:\d+") + + def __init__(self, project_dir: str): + """ + :param project_dir: str like /opt/labelstudio/study_name + """ + self._data = None + + for filename in ("config.yaml", "config.json"): + try: + path = os.path.join(project_dir, filename) + with open(path, "r", encoding="utf8") as f: + self._data = yaml.safe_load(f) + except FileNotFoundError: + continue + + if self._data is None: + raise FileNotFoundError(f"No config.yaml or config.json file found in {project_dir}") + + ### Annotators + # Internally, we're often dealing with numeric ID as the primary annotator identifier, since that's what + # is stored in Label Studio. So that's what we return from this method. + # But as humans writing config files, it's more natural to think of "name -> id". + # So that's what we keep in the config, and we just reverse it here for convenience. + orig_annotators = self._data.get("annotators", {}) + self.annotators = dict(map(reversed, orig_annotators.items())) + + ### Note ranges + # Handle some extra syntax like 1:3 == [1, 2, 3] + self.note_ranges = self._data.get("ranges", {}) + for key, values in self.note_ranges.items(): + if not isinstance(values, list): + values = [values] + parsed_ranges = (self._parse_note_range(value) for value in values) + self.note_ranges[key] = list(itertools.chain.from_iterable(parsed_ranges)) + + def _parse_note_range(self, value: str | int) -> Iterable[int]: + if isinstance(value, int): + return [value] + elif self._NUMBER_REGEX.fullmatch(value): + return [int(value)] + elif self._RANGE_REGEX.fullmatch(value): + edges = value.split(":") + return range(int(edges[0]), int(edges[1]) + 1) + elif value in self.note_ranges: + return self._parse_note_range(self.note_ranges[value]) # warning: no guards against infinite recursion + else: + print(f"Unknown note range '{value}'", file=sys.stderr) + return [] + + @property + def class_labels(self) -> list[str]: + return self._data.setdefault("labels", []) diff --git a/chart_review/covid_symptom/config.yaml b/chart_review/covid_symptom/config.yaml new file mode 100644 index 0000000..7d30628 --- /dev/null +++ b/chart_review/covid_symptom/config.yaml @@ -0,0 +1,31 @@ +# This is a converted version of config.py +# I'm keeping both around for now, as we transition. + +labels: + - Congestion or runny nose + - Cough + - Diarrhea + - Dyspnea + - Fatigue + - Fever or chills + - Headache + - Loss of taste or smell + - Muscle or body aches + - Nausea or vomiting + - Sore throat + +annotators: + andy = 2 + amy = 3 + alon = 6 + ctakes = 7 # mcmurry.andy + icd10 = 0 + +ranges: + corpus: 782:1006 + amy: 782:895 + andy: 895:1006 + andy_alon: 979:1006 + amy_alon: 864:891 + alon: [amy_alon, andy_alon] + icd10_missing: [782, 791, 793, 799, 811, 824, 826, 828, 833, 837, 859, 860, 870, 877, 882, 886, 921, 959, 985, 986, 994, 1004] diff --git a/chart_review/simplify.py b/chart_review/simplify.py index 9c93b17..f5c37c1 100644 --- a/chart_review/simplify.py +++ b/chart_review/simplify.py @@ -1,7 +1,7 @@ from collections.abc import Iterable from enum import EnumMeta import re -from chart_review import common +from chart_review import common, config from chart_review.common import guard_str, guard_iter def merge_simple(source: dict, append: dict) -> dict: @@ -26,7 +26,7 @@ def merge_simple(source: dict, append: dict) -> dict: merged['annotations'][int(note_id)][annotator].append(entry) return merged -def simplify_full(exported_json: str, annotator_enum: EnumMeta) -> dict: +def simplify_full(exported_json: str, annotator_enum: config.AnnotatorMap) -> dict: """ TODO: refactor JSON manipulation to labelstudio.py LabelStudio outputs contain more info than needed for IAA and term_freq. @@ -48,7 +48,7 @@ def simplify_full(exported_json: str, annotator_enum: EnumMeta) -> dict: for annot in entry.get('annotations'): completed_by = annot.get('completed_by') - annotator = annotator_enum(completed_by).name + annotator = annotator_enum[completed_by] label = None for result in annot.get('result'): if not label: diff --git a/chart_review/suicide_icd10/config.yaml b/chart_review/suicide_icd10/config.yaml new file mode 100644 index 0000000..4ce7e84 --- /dev/null +++ b/chart_review/suicide_icd10/config.yaml @@ -0,0 +1,19 @@ +# This is a converted version of config.py +# I'm keeping both around for now, as we transition. + +labels: + - ideation-present + - action-present + - ideation-past + - action-past + +annotators: + andy: 2 + amy: 3 + alon: 4 + +ranges: + corpus: 1226:1277 + andy: corpus + amy: corpus + alon: corpus diff --git a/pyproject.toml b/pyproject.toml index f945b78..2494735 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,7 @@ version = "0.0.1" requires-python = ">= 3.9" dependencies = [ "ctakesclient", + "pyyaml >= 6", ] description = "Medical Record Chart Review Calculator" readme = "README.md" @@ -20,6 +21,9 @@ Home = "https://smarthealthit.org/cumulus-a-universal-sidecar-for-a-smart-learni Documentation = "https://docs.smarthealthit.org/cumulus/" Source = "https://github.com/smart-on-fhir/chart-review" +[project.scripts] +chart-review = "chart_review.cli:main_cli" + [build-system] requires = ["flit_core >=3.4,<4"] build-backend = "flit_core.buildapi" From eb33d5a73d7f5f1e64eade0c4f522f1697cdb67a Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Thu, 12 Oct 2023 15:11:50 -0400 Subject: [PATCH 3/7] ci: add initial test case --- .github/workflows/ci.yaml | 37 +++++ README.md | 41 ++--- chart_review/cli.py | 4 +- pyproject.toml | 3 + tests/__init__.py | 0 tests/data/cold/config.yaml | 14 ++ tests/data/cold/labelstudio-export.json | 193 ++++++++++++++++++++++++ tests/test_cli.py | 83 ++++++++++ 8 files changed, 354 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 tests/__init__.py create mode 100644 tests/data/cold/config.yaml create mode 100644 tests/data/cold/labelstudio-export.json create mode 100644 tests/test_cli.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..a73134b --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,37 @@ +name: CI +on: + pull_request: + push: + branches: + - main + +# The goal here is to cancel older workflows when a PR is updated (because it's pointless work) +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + unittest: + name: unit tests + runs-on: ubuntu-22.04 + strategy: + matrix: + # don't go crazy with the Python versions as they eat up CI minutes + python-version: ["3.9"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[tests] + + - name: Test with pytest + run: | + python -m pytest diff --git a/README.md b/README.md index 9270fb4..abc7240 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ The most common chart-review measures agreement of the _**class_label**_ from a 1. Clone this repo. 2. Install it locally like so: `pipx install .` -This is not released on PyPI yet. +`chart-review` is not yet released on PyPI. --- ### How to Run @@ -47,44 +47,47 @@ Chart Review operates on a project folder that holds your config & data. 3. Add a `config.yaml` file (or `config.json`) that looks something like this (read more on this format below): ```yaml -class-labels: +labels: - cough - fever annotators: jane: 2 john: 6 + jack: 8 + +ranges: + jane: 242:250 + john: [260:271, 277] + jack: [jane, john] ``` #### Run -Simply call `chart-review` with your project folder: +Call `chart-review` with the sub-command you want and its arguments: -`chart-review /path/to/project/dir` +`chart-review accuracy --project-dir /path/to/project/dir jane john jack` Pass `--help` to see more options. --- ### Config File Format -`config.py` defines study specific variables. +`config.yaml` defines study specific variables. - * study_folder = `/opt/cumulus/chart-review/studyname` - * class_labels = `['case', 'control', 'unknown', '...']` - * Annotators - * NoteRanges + * Class labels: `labels: ['cough', 'fever']` + * Annotators: `annotators: {'jane': 3, 'john': 8}` + * Note ranges: `ranges: {'jane': 40:50, 'john': [2, 3, 4, 5]}` -Enum **Annotators** maps a SimpleName to LabelStudioUserId -* human subject matter expert _like_ "Rena" -* computer method _like_ "NLP" -* coded data sources _like_ "ICD10" +`annotators` maps a name to a Label Studio User ID +* human subject matter expert _like_ `jane` +* computer method _like_ `nlp` +* coded data sources _like_ `icd10` -Enum **NoteRanges** maps a selection of NoteID from the corpus -* corpus = range(1, end+1) -* annotator1_vs_2 = Iterable -* annotator2_vs_3 = Iterable -* annotator3_vs_1 = Iterable -* annotator3_vs_1 = Iterable +`ranges` maps a selection of Note IDs from the corpus +* `corpus: start:end` +* `annotator1_vs_2: [list, of, notes]` +* `annotator2_vs_3: corpus` --- **BASE COHORT METHODS** diff --git a/chart_review/cli.py b/chart_review/cli.py index d66bfe6..936a495 100644 --- a/chart_review/cli.py +++ b/chart_review/cli.py @@ -84,11 +84,11 @@ def run_accuracy(args: argparse.Namespace) -> None: # ############################################################################### -def main_cli() -> None: +def main_cli(argv: list[str] = None) -> None: """Main entrypoint that wraps all the core program logic""" try: parser = define_parser() - args = parser.parse_args() + args = parser.parse_args(argv) args.func(args) except Exception as exc: sys.exit(str(exc)) diff --git a/pyproject.toml b/pyproject.toml index 2494735..b2e098c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,9 @@ requires = ["flit_core >=3.4,<4"] build-backend = "flit_core.buildapi" [project.optional-dependencies] +tests = [ + "pytest", +] dev = [ "black", "pylint", diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/cold/config.yaml b/tests/data/cold/config.yaml new file mode 100644 index 0000000..4f9f416 --- /dev/null +++ b/tests/data/cold/config.yaml @@ -0,0 +1,14 @@ +labels: + - Cough + - Fatigue + - Headache + +annotators: + jane: 3 + john: 5 + jill: 6 + +ranges: + jane: [1, 3, 4] + john: [1, 2, 4] + jill: [1, 2, 3, 4] diff --git a/tests/data/cold/labelstudio-export.json b/tests/data/cold/labelstudio-export.json new file mode 100644 index 0000000..9a899b8 --- /dev/null +++ b/tests/data/cold/labelstudio-export.json @@ -0,0 +1,193 @@ +[ + { + "id": 1, + "annotations": [ + { + "id": 301, + "completed_by": 3, + "result": [ + { + "value": { + "text": "achoo", + "labels": [ + "Cough" + ] + } + }, + { + "value": { + "text": "sigh", + "labels": [ + "Fatigue", + "Headache" + ] + } + } + ] + }, + { + "id": 501, + "completed_by": 5, + "result": [ + { + "value": { + "text": "achoo", + "labels": [ + "Cough" + ] + } + }, + { + "value": { + "text": "sigh", + "labels": [ + "Fatigue" + ] + } + } + ] + }, + { + "id": 601, + "completed_by": 6, + "result": [ + { + "value": { + "text": "achoo", + "labels": [ + "Cough" + ] + } + }, + { + "value": { + "text": "sigh", + "labels": [ + "Fatigue" + ] + } + } + ] + } + ] + }, + { + "id": 2, + "annotations": [ + { + "id": 502, + "completed_by": 5, + "result": [ + { + "value": { + "text": "ouch", + "labels": [ + "Headache" + ] + } + } + ] + }, + { + "id": 602, + "completed_by": 6, + "result": [ + { + "value": { + "text": "ouch", + "labels": [ + "Fatigue" + ] + } + } + ] + } + ] + }, + { + "id": 3, + "annotations": [ + { + "id": 303, + "completed_by": 3, + "result": [] + }, + { + "id": 603, + "completed_by": 6, + "result": [] + } + ] + }, + { + "id": 4, + "annotations": [ + { + "id": 304, + "completed_by": 3, + "result": [ + { + "value": { + "text": "sleepy", + "labels": [ + "Fatigue" + ] + } + }, + { + "value": { + "text": "pain", + "labels": [ + "Headache" + ] + } + } + ] + }, + { + "id": 504, + "completed_by": 5, + "result": [ + { + "value": { + "text": "sleepy", + "labels": [ + "Fatigue" + ] + } + }, + { + "value": { + "text": "pain", + "labels": [ + "Headache" + ] + } + } + ] + }, + { + "id": 604, + "completed_by": 6, + "result": [ + { + "value": { + "text": "sleepy", + "labels": [ + "Fatigue" + ] + } + }, + { + "value": { + "text": "pain", + "labels": [ + "Cough" + ] + } + } + ] + } + ] + } +] \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..6e29d79 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,83 @@ +"""Tests for cli.py""" + +import os +import shutil +import tempfile +import unittest + +from chart_review import cli, common + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") + + +class TestCommandLine(unittest.TestCase): + """Test case for the top-level CLI code""" + + def setUp(self): + super().setUp() + self.maxDiff = None + + def test_accuracy(self): + with tempfile.TemporaryDirectory() as tmpdir: + shutil.copytree(f"{DATA_DIR}/cold", tmpdir, dirs_exist_ok=True) + cli.main_cli(["accuracy", "--project-dir", tmpdir, "jane", "john", "jill"]) + + accuracy_json = common.read_json(f"{tmpdir}/accuracy-jane-john-jill.json") + self.assertEqual( + { + "Cough": { + "F1": 0.667, + "FN": 0, + "FP": 2, + "NPV": 1.0, + "PPV": 0.5, + "Sens": 1.0, + "Spec": 0.5, + "TN": 2, + "TP": 2, + }, + "F1": 0.667, + "FN": 3, + "FP": 3, + "Fatigue": { + "F1": 0.889, + "FN": 0, + "FP": 1, + "NPV": 1.0, + "PPV": 0.8, + "Sens": 1.0, + "Spec": 0.5, + "TN": 1, + "TP": 4, + }, + "Headache": { + "F1": 0, + "FN": 3, + "FP": 0, + "NPV": 0, + "PPV": 0, + "Sens": 0, + "Spec": 0, + "TN": 3, + "TP": 0, + }, + "NPV": 0.667, + "PPV": 0.667, + "Sens": 0.667, + "Spec": 0.667, + "TN": 6, + "TP": 6, + }, + accuracy_json, + ) + + accuracy_csv = common.read_text(f"{tmpdir}/accuracy-jane-john-jill.csv") + self.assertEqual( + """F1 Sens Spec PPV NPV TP FN TN FP Label +0.667 0.667 0.667 0.667 0.667 6 3 6 3 * +0.667 1.0 0.5 0.5 1.0 2 0 2 2 Cough +0.889 1.0 0.5 0.8 1.0 4 0 1 1 Fatigue +0 0 0 0 0 0 3 3 0 Headache +""", + accuracy_csv, + ) From 7ae1ce8619abfa09f3ac5974ccab82d24ffd696e Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Thu, 12 Oct 2023 15:24:50 -0400 Subject: [PATCH 4/7] refactor: move accuracy logic to separate file This will make it easier for someone just using the python to call it, if they want to. --- chart_review/cli.py | 33 +++---------------------- chart_review/commands/__init__.py | 0 chart_review/commands/accuracy.py | 41 +++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 30 deletions(-) create mode 100644 chart_review/commands/__init__.py create mode 100644 chart_review/commands/accuracy.py diff --git a/chart_review/cli.py b/chart_review/cli.py index 936a495..0502edf 100644 --- a/chart_review/cli.py +++ b/chart_review/cli.py @@ -1,10 +1,10 @@ """Run chart-review from the command-line""" import argparse -import os import sys -from chart_review import agree, cohort, common +from chart_review import cohort +from chart_review.commands.accuracy import accuracy ############################################################################### @@ -48,34 +48,7 @@ def add_accuracy_subparser(subparsers) -> None: def run_accuracy(args: argparse.Namespace) -> None: reader = cohort.CohortReader(args.project_dir) - - first_ann = args.one - second_ann = args.two - base_ann = args.base - - # Grab ranges - first_range = reader.config.note_ranges[first_ann] - second_range = reader.config.note_ranges[second_ann] - - # All labels first - first_matrix = reader.confusion_matrix(first_ann, base_ann, first_range) - second_matrix = reader.confusion_matrix(second_ann, base_ann, second_range) - whole_matrix = agree.append_matrix(first_matrix, second_matrix) - table = agree.score_matrix(whole_matrix) - - # Now do each labels separately - for label in reader.class_labels: - first_matrix = reader.confusion_matrix(first_ann, base_ann, first_range, label) - second_matrix = reader.confusion_matrix(second_ann, base_ann, second_range, label) - whole_matrix = agree.append_matrix(first_matrix, second_matrix) - table[label] = agree.score_matrix(whole_matrix) - - # And write out the results - output_stem = os.path.join(reader.project_dir, f"accuracy-{first_ann}-{second_ann}-{base_ann}") - common.write_json(f"{output_stem}.json", table) - print(f"Wrote {output_stem}.json") - common.write_text(f"{output_stem}.csv", agree.csv_table(table, reader.class_labels)) - print(f"Wrote {output_stem}.csv") + accuracy(reader, args.one, args.two, args.base) ############################################################################### diff --git a/chart_review/commands/__init__.py b/chart_review/commands/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/chart_review/commands/accuracy.py b/chart_review/commands/accuracy.py new file mode 100644 index 0000000..9f32219 --- /dev/null +++ b/chart_review/commands/accuracy.py @@ -0,0 +1,41 @@ +"""Methods for high-level accuracy calculations.""" + +import os + +from chart_review import agree, cohort, common + + +def accuracy(reader: cohort.CohortReader, first_ann: str, second_ann: str, base_ann: str) -> None: + """ + High-level accuracy calculation between three reviewers. + + The results will be written to the project directory. + + :param reader: the cohort configuration + :param first_ann: the first annotator to compare + :param second_ann: the second annotator to compare + :param base_ann: the base annotator to compare the others against + """ + # Grab ranges + first_range = reader.config.note_ranges[first_ann] + second_range = reader.config.note_ranges[second_ann] + + # All labels first + first_matrix = reader.confusion_matrix(first_ann, base_ann, first_range) + second_matrix = reader.confusion_matrix(second_ann, base_ann, second_range) + whole_matrix = agree.append_matrix(first_matrix, second_matrix) + table = agree.score_matrix(whole_matrix) + + # Now do each labels separately + for label in reader.class_labels: + first_matrix = reader.confusion_matrix(first_ann, base_ann, first_range, label) + second_matrix = reader.confusion_matrix(second_ann, base_ann, second_range, label) + whole_matrix = agree.append_matrix(first_matrix, second_matrix) + table[label] = agree.score_matrix(whole_matrix) + + # And write out the results + output_stem = os.path.join(reader.project_dir, f"accuracy-{first_ann}-{second_ann}-{base_ann}") + common.write_json(f"{output_stem}.json", table) + print(f"Wrote {output_stem}.json") + common.write_text(f"{output_stem}.csv", agree.csv_table(table, reader.class_labels)) + print(f"Wrote {output_stem}.csv") From 82ea4d69c3cbe098fecfc5adceabf9c0b7851a03 Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Thu, 12 Oct 2023 15:35:28 -0400 Subject: [PATCH 5/7] build: bump minimum python to 3.10 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index b2e098c..39570d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "chart-review" version = "0.0.1" -requires-python = ">= 3.9" +requires-python = ">= 3.10" dependencies = [ "ctakesclient", "pyyaml >= 6", From 2c816ffd8a7ab4629a960d2610b0dd92816b279a Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Thu, 12 Oct 2023 15:37:00 -0400 Subject: [PATCH 6/7] ci: and bump the CI python too, whoops --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a73134b..57cf658 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,7 +17,7 @@ jobs: strategy: matrix: # don't go crazy with the Python versions as they eat up CI minutes - python-version: ["3.9"] + python-version: ["3.10"] steps: - uses: actions/checkout@v4 From 06c56d4e86f8409db52d23d32a2e4a2211d214dd Mon Sep 17 00:00:00 2001 From: Michael Terry Date: Fri, 13 Oct 2023 15:42:14 -0400 Subject: [PATCH 7/7] Add a config file test suite --- README.md | 6 +-- chart_review/config.py | 17 +++--- chart_review/covid_symptom/config.yaml | 10 ++-- chart_review/suicide_icd10/config.yaml | 2 +- pyproject.toml | 1 + tests/test_config.py | 72 ++++++++++++++++++++++++++ 6 files changed, 90 insertions(+), 18 deletions(-) create mode 100644 tests/test_config.py diff --git a/README.md b/README.md index abc7240..941da11 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,8 @@ annotators: jack: 8 ranges: - jane: 242:250 - john: [260:271, 277] + jane: 242-250 # inclusive + john: [260-271, 277] jack: [jane, john] ``` @@ -77,7 +77,7 @@ Pass `--help` to see more options. * Class labels: `labels: ['cough', 'fever']` * Annotators: `annotators: {'jane': 3, 'john': 8}` - * Note ranges: `ranges: {'jane': 40:50, 'john': [2, 3, 4, 5]}` + * Note ranges: `ranges: {'jane': 40-50, 'john': [2, 3, 4, 5]}` `annotators` maps a name to a Label Studio User ID * human subject matter expert _like_ `jane` diff --git a/chart_review/config.py b/chart_review/config.py index 3043351..6833329 100644 --- a/chart_review/config.py +++ b/chart_review/config.py @@ -12,7 +12,7 @@ class ProjectConfig: _NUMBER_REGEX = re.compile(r"\d+") - _RANGE_REGEX = re.compile(r"\d+:\d+") + _RANGE_REGEX = re.compile(r"\d+-\d+") def __init__(self, project_dir: str): """ @@ -40,21 +40,20 @@ def __init__(self, project_dir: str): self.annotators = dict(map(reversed, orig_annotators.items())) ### Note ranges - # Handle some extra syntax like 1:3 == [1, 2, 3] + # Handle some extra syntax like 1-3 == [1, 2, 3] self.note_ranges = self._data.get("ranges", {}) for key, values in self.note_ranges.items(): - if not isinstance(values, list): - values = [values] - parsed_ranges = (self._parse_note_range(value) for value in values) - self.note_ranges[key] = list(itertools.chain.from_iterable(parsed_ranges)) + self.note_ranges[key] = list(self._parse_note_range(values)) - def _parse_note_range(self, value: str | int) -> Iterable[int]: - if isinstance(value, int): + def _parse_note_range(self, value: str | int | list[str | int]) -> Iterable[int]: + if isinstance(value, list): + return list(itertools.chain.from_iterable(self._parse_note_range(v) for v in value)) + elif isinstance(value, int): return [value] elif self._NUMBER_REGEX.fullmatch(value): return [int(value)] elif self._RANGE_REGEX.fullmatch(value): - edges = value.split(":") + edges = value.split("-") return range(int(edges[0]), int(edges[1]) + 1) elif value in self.note_ranges: return self._parse_note_range(self.note_ranges[value]) # warning: no guards against infinite recursion diff --git a/chart_review/covid_symptom/config.yaml b/chart_review/covid_symptom/config.yaml index 7d30628..ab98a0c 100644 --- a/chart_review/covid_symptom/config.yaml +++ b/chart_review/covid_symptom/config.yaml @@ -22,10 +22,10 @@ annotators: icd10 = 0 ranges: - corpus: 782:1006 - amy: 782:895 - andy: 895:1006 - andy_alon: 979:1006 - amy_alon: 864:891 + corpus: 782-1006 + amy: 782-895 + andy: 895-1006 + andy_alon: 979-1006 + amy_alon: 864-891 alon: [amy_alon, andy_alon] icd10_missing: [782, 791, 793, 799, 811, 824, 826, 828, 833, 837, 859, 860, 870, 877, 882, 886, 921, 959, 985, 986, 994, 1004] diff --git a/chart_review/suicide_icd10/config.yaml b/chart_review/suicide_icd10/config.yaml index 4ce7e84..4212c57 100644 --- a/chart_review/suicide_icd10/config.yaml +++ b/chart_review/suicide_icd10/config.yaml @@ -13,7 +13,7 @@ annotators: alon: 4 ranges: - corpus: 1226:1277 + corpus: 1226-1277 andy: corpus amy: corpus alon: corpus diff --git a/pyproject.toml b/pyproject.toml index 39570d7..1692dcf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ build-backend = "flit_core.buildapi" [project.optional-dependencies] tests = [ + "ddt", "pytest", ] dev = [ diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..fb2ab46 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,72 @@ +"""Tests for config.py""" + +import os +import tempfile +import unittest + +import ddt + +from chart_review import common, config + + +@ddt.ddt +class TestProjectConfig(unittest.TestCase): + """Test case for basic config parsing""" + + def setUp(self): + super().setUp() + self.maxDiff = None + + def make_config(self, conf_text: str, filename: str = "config.yaml") -> config.ProjectConfig: + tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(tmpdir.cleanup) + common.write_text(os.path.join(tmpdir.name, filename), conf_text) + return config.ProjectConfig(tmpdir.name) + + @ddt.data( + ("yaml", """ + labels: + - cough + - fever + annotators: + jane: 1 + john: 2 + ranges: + jane: [3] + john: [1, 3, 5] + """), + ("json", """ + { + "labels": ["cough", "fever"], + "annotators": {"jane": 1, "john": 2}, + "ranges": {"jane": [3], "john": [1, 3, 5]} + } + """), + ) + @ddt.unpack + def test_multiple_formats(self, suffix, text): + """Verify that we can operate on multiple formats (like json & yaml).""" + proj_config = self.make_config(text, filename=f"config.{suffix}") + + self.assertEqual(["cough", "fever"], proj_config.class_labels) + self.assertEqual({1: "jane", 2: "john"}, proj_config.annotators) + self.assertEqual({"jane": [3], "john": [1, 3, 5]}, proj_config.note_ranges) + + def test_range_syntax(self): + """Verify that we support interesting note range syntax options.""" + proj_config = self.make_config(""" + ranges: + bare_num: 1 + string_num: "2" + range: 3-5 + reference: bare_num + array: [1, "2", range] + """) + + self.assertEqual({ + "bare_num": [1], + "string_num": [2], + "range": [3, 4, 5], + "reference": [1], + "array": [1, 2, 3, 4, 5], + }, proj_config.note_ranges)