feat: final v1

siemdejong · Jul 31, 2023 · 0c1544b · 0c1544b
1 parent ef91001
commit 0c1544b
Show file tree

Hide file tree

Showing 30 changed files with 9,244 additions and 4,829 deletions.
diff --git a/.gitignore b/.gitignore
@@ -169,4 +169,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+#.idea/
+test.sbatch
+shg-optuna.sbatch
+shg-optuna-arr.sbatch
diff --git a/analyze_images.sh b/analyze_images.sh
@@ -15,7 +15,7 @@ for ((dir = 0; dir <= $maxdir ; dir++)); do
     pyimq.main \
         --mode=directory \
         --mode=analyze \
-        --working-directory=/scistor/guest/sjg203/projects/shg-strain-stress/data/grayscale/z-stacks/$dir \
+        --working-directory=/scistor/guest/sjg203/projects/shg-strain-stress/data/normalized/z-stacks/$dir \
         --rgb-channel=0 \
         > outputs/pyimq/image_analysis_stdout.txt \
         2> outputs/pyimq/image_analysis_stderr.txt

diff --git a/model_zoo/20.pt b/model_zoo/20.pt
diff --git a/notebooks/11-sjg-logistic.ipynb b/notebooks/11-sjg-logistic.ipynb
diff --git a/notebooks/14-sjg-lds-bins.ipynb b/notebooks/14-sjg-lds-bins.ipynb
diff --git a/notebooks/15-sjg-captum-intro.ipynb b/notebooks/15-sjg-captum-intro.ipynb
diff --git a/notebooks/17-sjg-normalization.ipynb b/notebooks/17-sjg-normalization.ipynb
@@ -274,6 +274,84 @@
     "_ = plt.bar(edges[:-1], hist)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Find code to Yeo-Johnson normalize all images and normalize them below."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from glob import glob\n",
+    "import numpy as np\n",
+    "from PIL import Image\n",
+    "import os\n",
+    "from scipy import stats"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "image_paths = glob(f\"../data/preprocessed/z-stacks/**/*.bmp\")\n",
+    "output_path = \"../data/normalized/z-stacks\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def yeo_johnson_and_normalize_all(image_paths: list[str], output_path: str, lmbda: float, mean: float, std: float) -> None:\n",
+    "    \"\"\"Equalizes all images selected with a list of image paths using CLAHE.\n",
+    "    \n",
+    "    The output path must not contain a trailing \"/\".\n",
+    "    \"\"\"\n",
+    "    from tqdm import tqdm\n",
+    "\n",
+    "    for path in tqdm(image_paths):\n",
+    "        stack_id = path.split('/')[-2]\n",
+    "        slice_fn = path.split('/')[-1].split('.')[0]\n",
+    "\n",
+    "        # Read image from file\n",
+    "        img_pil = Image.open(path)\n",
+    "        img = np.asarray(img_pil)\n",
+    "\n",
+    "        img_transformed = stats.yeojohnson(img, lmbda)\n",
+    "        img_normalized = (img_transformed - mean) / std\n",
+    "        \n",
+    "        if not os.path.exists(f\"{output_path}/{stack_id}\"):\n",
+    "            os.makedirs(f\"{output_path}/{stack_id}\")\n",
+    "\n",
+    "        img_out = Image.fromarray(img_normalized)\n",
+    "        img_out.save(f\"{output_path}/{stack_id}/{slice_fn}.tif\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████| 1905/1905 [03:57<00:00,  8.02it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "yeo_johnson_and_normalize_all(image_paths, output_path, 0.466319593487972, 14.716653741103862, 6.557034596034911)"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,

diff --git a/notebooks/19-sjg-kurtosis.ipynb b/notebooks/19-sjg-kurtosis.ipynb
diff --git a/notebooks/22-sjg-entropy.ipynb b/notebooks/22-sjg-entropy.ipynb
diff --git a/notebooks/26-sjg-sort-images.ipynb b/notebooks/26-sjg-sort-images.ipynb
@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -15,7 +15,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -25,7 +25,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -36,14 +36,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 61/61 [00:42<00:00,  1.45it/s]\n"
+      "100%|██████████| 61/61 [01:20<00:00,  1.32s/it]\n"
      ]
     }
    ],
@@ -59,7 +59,8 @@
     "    image_id = 0\n",
     "    for image_fn in analysis_sorted[\"Filename\"].to_numpy():\n",
     "        stack_id = image_fn.split('/')[-2]\n",
-    "        output_fn = f\"{output_folder}/{stack_id}/{image_id}.bmp\"\n",
+    "        extension = image_fn.split('.')[-1]\n",
+    "        output_fn = f\"{output_folder}/{stack_id}/{image_id}.{extension}\"\n",
     "\n",
     "        if not os.path.exists(f\"{output_folder}/{stack_id}\"):\n",
     "            os.makedirs(f\"{output_folder}/{stack_id}\")\n",
@@ -94,7 +95,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.13"
+   "version": "3.9.15"
   },
   "orig_nbformat": 4,
   "vscode": {

diff --git a/notebooks/30-sjg-inference.ipynb b/notebooks/30-sjg-inference.ipynb
diff --git a/notebooks/31-sjg-plot-from-csv.ipynb b/notebooks/31-sjg-plot-from-csv.ipynb