deploy: 03f6c5c

VectorInstitute · Nov 28, 2023 · 51796de · 51796de
1 parent 6848184
commit 51796de
Show file tree

Hide file tree

Showing 24 changed files with 4,843 additions and 5,031 deletions.
diff --git a/api/_modules/cyclops/evaluate/evaluator.html b/api/_modules/cyclops/evaluate/evaluator.html
diff --git a/api/_modules/cyclops/evaluate/fairness/evaluator.html b/api/_modules/cyclops/evaluate/fairness/evaluator.html
@@ -196,6 +196,7 @@ <h1>Source code for cyclops.evaluate.fairness.evaluator</h1><div class="highligh
     <span class="n">_check_thresholds</span><span class="p">,</span>
     <span class="n">_get_value_if_singleton_array</span><span class="p">,</span>
 <span class="p">)</span>
+<span class="kn">from</span> <span class="nn">cyclops.evaluate.utils</span> <span class="kn">import</span> <span class="n">_format_column_names</span>
 <span class="kn">from</span> <span class="nn">cyclops.utils.log</span> <span class="kn">import</span> <span class="n">setup_logging</span>
 
 
@@ -210,7 +211,7 @@ <h1>Source code for cyclops.evaluate.fairness.evaluator</h1><div class="highligh
     <span class="n">dataset</span><span class="p">:</span> <span class="n">Dataset</span><span class="p">,</span>
     <span class="n">groups</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]],</span>
     <span class="n">target_columns</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]],</span>
-    <span class="n">prediction_columns</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]</span> <span class="o">=</span> <span class="s2">&quot;predictions&quot;</span><span class="p">,</span>
+    <span class="n">prediction_columns</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]],</span>
     <span class="n">group_values</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">group_bins</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="n">Any</span><span class="p">]]]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
     <span class="n">group_base_values</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
@@ -245,7 +246,7 @@ <h1>Source code for cyclops.evaluate.fairness.evaluator</h1><div class="highligh
 <span class="sd">        The target or targets columns used to compute metrics. If a string, it should</span>
 <span class="sd">        be the name of a column in the dataset. If a list, it should be a list of</span>
 <span class="sd">        column names in the dataset. Lists will be treated as multilabel targets.</span>
-<span class="sd">    prediction_columns : Union[str, List[str]], default=&quot;predictions&quot;</span>
+<span class="sd">    prediction_columns : Union[str, List[str]]</span>
 <span class="sd">        The names of the prediction columns used to compute metrics. If a string, it</span>
 <span class="sd">        should be the name of a column in the dataset. If a list, it should be a list</span>
 <span class="sd">        of column names in the dataset. Lists allow for evaluating multiple models</span>
@@ -583,36 +584,6 @@ <h1>Source code for cyclops.evaluate.fairness.evaluator</h1><div class="highligh
     <span class="p">)</span>
 
 
-<span class="k">def</span> <span class="nf">_format_column_names</span><span class="p">(</span><span class="n">column_names</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]])</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]:</span>
-<span class="w">    </span><span class="sd">&quot;&quot;&quot;Format the column names to list of strings if not already a list.</span>
-
-<span class="sd">    Parameters</span>
-<span class="sd">    ----------</span>
-<span class="sd">    column_names : Union[str, List[str]]</span>
-<span class="sd">        The column names to format.</span>
-
-<span class="sd">    Returns</span>
-<span class="sd">    -------</span>
-<span class="sd">    List[str]</span>
-<span class="sd">        The formatted column names.</span>
-
-<span class="sd">    Raises</span>
-<span class="sd">    ------</span>
-<span class="sd">    TypeError</span>
-<span class="sd">        If any of the column names are not strings or list of strings.</span>
-
-<span class="sd">    &quot;&quot;&quot;</span>
-    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">column_names</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
-        <span class="k">return</span> <span class="p">[</span><span class="n">column_names</span><span class="p">]</span>
-    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">column_names</span><span class="p">,</span> <span class="nb">list</span><span class="p">):</span>
-        <span class="k">return</span> <span class="n">column_names</span>
-
-    <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span>
-        <span class="sa">f</span><span class="s2">&quot;Expected column name </span><span class="si">{</span><span class="n">column_names</span><span class="si">}</span><span class="s2"> to be a string or &quot;</span>
-        <span class="sa">f</span><span class="s2">&quot;list of strings, but got </span><span class="si">{</span><span class="nb">type</span><span class="p">(</span><span class="n">column_names</span><span class="p">)</span><span class="si">}</span><span class="s2">.&quot;</span><span class="p">,</span>
-    <span class="p">)</span>
-
-
 <span class="k">def</span> <span class="nf">_get_unique_values</span><span class="p">(</span>
     <span class="n">dataset</span><span class="p">:</span> <span class="n">Dataset</span><span class="p">,</span>
     <span class="n">groups</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span>

diff --git a/api/_modules/cyclops/tasks/classification.html b/api/_modules/cyclops/tasks/classification.html
@@ -516,8 +516,10 @@ <h1>Source code for cyclops.tasks.classification</h1><div class="highlight"><pre
             <span class="n">metrics</span><span class="o">=</span><span class="n">metrics_collection</span><span class="p">,</span>
             <span class="n">target_columns</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_target</span><span class="p">,</span>
             <span class="n">slice_spec</span><span class="o">=</span><span class="n">slice_spec</span><span class="p">,</span>
-            <span class="n">prediction_column_prefix</span><span class="o">=</span><span class="n">prediction_column_prefix</span><span class="p">,</span>
-            <span class="n">remove_columns</span><span class="o">=</span><span class="n">remove_columns</span><span class="p">,</span>
+            <span class="n">prediction_columns</span><span class="o">=</span><span class="p">[</span>
+                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">prediction_column_prefix</span><span class="si">}</span><span class="s2">.</span><span class="si">{</span><span class="n">model_name</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">model_name</span> <span class="ow">in</span> <span class="n">model_names</span>
+            <span class="p">],</span>
+            <span class="n">ignore_columns</span><span class="o">=</span><span class="n">remove_columns</span><span class="p">,</span>
             <span class="n">split</span><span class="o">=</span><span class="n">splits_mapping</span><span class="p">[</span><span class="s2">&quot;test&quot;</span><span class="p">],</span>
             <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
             <span class="n">fairness_config</span><span class="o">=</span><span class="n">fairness_config</span><span class="p">,</span>
@@ -718,8 +720,10 @@ <h1>Source code for cyclops.tasks.classification</h1><div class="highlight"><pre
             <span class="n">metrics</span><span class="o">=</span><span class="n">metrics_collection</span><span class="p">,</span>
             <span class="n">slice_spec</span><span class="o">=</span><span class="n">slice_spec</span><span class="p">,</span>
             <span class="n">target_columns</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">task_target</span><span class="p">,</span>
-            <span class="n">prediction_column_prefix</span><span class="o">=</span><span class="n">prediction_column_prefix</span><span class="p">,</span>
-            <span class="n">remove_columns</span><span class="o">=</span><span class="n">remove_columns</span><span class="p">,</span>
+            <span class="n">prediction_columns</span><span class="o">=</span><span class="p">[</span>
+                <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">prediction_column_prefix</span><span class="si">}</span><span class="s2">.</span><span class="si">{</span><span class="n">model_name</span><span class="si">}</span><span class="s2">&quot;</span> <span class="k">for</span> <span class="n">model_name</span> <span class="ow">in</span> <span class="n">model_names</span>
+            <span class="p">],</span>
+            <span class="n">ignore_columns</span><span class="o">=</span><span class="n">remove_columns</span><span class="p">,</span>
             <span class="n">split</span><span class="o">=</span><span class="n">splits_mapping</span><span class="p">[</span><span class="s2">&quot;test&quot;</span><span class="p">],</span>
             <span class="n">batch_size</span><span class="o">=</span><span class="n">batch_size</span><span class="p">,</span>
             <span class="n">fairness_config</span><span class="o">=</span><span class="n">fairness_config</span><span class="p">,</span>

diff --git a/api/_sources/tutorials/kaggle/heart_failure_prediction.ipynb.txt b/api/_sources/tutorials/kaggle/heart_failure_prediction.ipynb.txt
@@ -885,12 +885,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "model_name = f\"model_for_preds.{model_name}\"\n",
     "results_flat = flatten_results_dict(\n",
     "    results=results,\n",
     "    remove_metrics=[\"BinaryROCCurve\", \"BinaryPrecisionRecallCurve\"],\n",
     "    model_name=model_name,\n",
     ")\n",
-    "\n",
     "results_female_flat = flatten_results_dict(\n",
     "    results=results_female,\n",
     "    model_name=model_name,\n",

diff --git a/api/_sources/tutorials/mimiciv/mortality_prediction.ipynb.txt b/api/_sources/tutorials/mimiciv/mortality_prediction.ipynb.txt
@@ -916,12 +916,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "model_name = f\"model_for_preds.{model_name}\"\n",
     "results_flat = flatten_results_dict(\n",
     "    results=results,\n",
     "    remove_metrics=[\"BinaryROCCurve\", \"BinaryPrecisionRecallCurve\"],\n",
     "    model_name=model_name,\n",
-    ")\n",
-    "print(results_flat)"
+    ")"
    ]
   },
   {

diff --git a/api/_sources/tutorials/nihcxr/cxr_classification.ipynb.txt b/api/_sources/tutorials/nihcxr/cxr_classification.ipynb.txt
@@ -294,10 +294,9 @@
     "nih_eval_results_gender = evaluator.evaluate(\n",
     "    dataset=nih_ds,\n",
     "    metrics=[ppv, npv, sensitivity, specificity],\n",
-    "    feature_columns=\"image\",\n",
     "    target_columns=pathologies,\n",
-    "    prediction_column_prefix=\"predictions\",\n",
-    "    remove_columns=\"image\",\n",
+    "    prediction_columns=\"predictions.densenet\",\n",
+    "    ignore_columns=\"image\",\n",
     "    slice_spec=slice_spec,\n",
     ")"
    ]
@@ -354,10 +353,9 @@
     "nih_eval_results_age = evaluator.evaluate(\n",
     "    dataset=nih_ds,\n",
     "    metrics=[ppv, npv, sensitivity, specificity],\n",
-    "    feature_columns=\"image\",\n",
     "    target_columns=pathologies,\n",
-    "    prediction_column_prefix=\"predictions\",\n",
-    "    remove_columns=\"image\",\n",
+    "    prediction_columns=\"predictions.densenet\",\n",
+    "    ignore_columns=\"image\",\n",
     "    slice_spec=slice_spec,\n",
     ")"
    ]
@@ -459,15 +457,17 @@
    "outputs": [],
    "source": [
     "results_flat = {}\n",
-    "for slice_, metrics in nih_eval_results_age[\"densenet\"].items():\n",
+    "for slice_, metrics in nih_eval_results_age[\"model_for_predictions.densenet\"].items():\n",
     "    for name, metric in metrics.items():\n",
     "        results_flat[f\"{slice_}/{name}\"] = metric.mean()\n",
     "        for itr, m in enumerate(metric):\n",
     "            if slice_ == \"overall\":\n",
     "                results_flat[f\"pathology:{pathologies[itr]}/{name}\"] = m\n",
     "            else:\n",
     "                results_flat[f\"{slice_}&pathology:{pathologies[itr]}/{name}\"] = m\n",
-    "for slice_, metrics in nih_eval_results_gender[\"densenet\"].items():\n",
+    "for slice_, metrics in nih_eval_results_gender[\n",
+    "    \"model_for_predictions.densenet\"\n",
+    "].items():\n",
     "    for name, metric in metrics.items():\n",
     "        results_flat[f\"{slice_}/{name}\"] = metric.mean()\n",
     "        for itr, m in enumerate(metric):\n",

diff --git a/api/_sources/tutorials/synthea/los_prediction.ipynb.txt b/api/_sources/tutorials/synthea/los_prediction.ipynb.txt
@@ -1092,12 +1092,12 @@
    },
    "outputs": [],
    "source": [
+    "model_name = f\"model_for_preds.{model_name}\"\n",
     "results_flat = flatten_results_dict(\n",
     "    results=results,\n",
     "    remove_metrics=[\"BinaryROCCurve\", \"BinaryPrecisionRecallCurve\"],\n",
     "    model_name=model_name,\n",
-    ")\n",
-    "print(results_flat)"
+    ")"
    ]
   },
   {