Pixel-level_AUROC.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cad8538f-d6b2-41b5-857b-5ca2dd230407",
   "metadata": {},
   "source": [
    "# Pixel-level AUROC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "431b2f48-a0b0-4cbc-9031-cd129bfe7004",
   "metadata": {},
   "outputs": [],
   "source": [
    "from PIL import Image\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "from glob import glob\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7705341e-27df-470b-999b-e8b1f9eff020",
   "metadata": {},
   "outputs": [],
   "source": [
    "def auroc_check(gt_path, am_path):\n",
    "    gt_map = Image.open(gt_path)\n",
    "    anomaly_map = Image.open(am_path)\n",
    "\n",
    "    gt_map_flatten = (np.array(gt_map).flatten())/255\n",
    "    anomaly_map_flatten = np.array(anomaly_map).flatten()\n",
    "    \n",
    "\n",
    "    return roc_auc_score(gt_map_flatten, anomaly_map_flatten)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43662d09-1d65-4a77-b4ef-b4353b01c344",
   "metadata": {},
   "source": [
    "# bottle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "87c91d67-07b1-443f-9953-76e6a4702e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_paths = glob(\"/workspace/EfficientAD-VQVAE/mvtec_anomaly_detection/bottle/ground_truth/*/*.png\")\n",
    "am_paths = glob(\"/workspace/EfficientAD-VQVAE/output/1/anomaly_maps/mvtec_ad/bottle/test/[!good]*/*.tiff\") # good 디렉토리 제외\n",
    "\n",
    "gt_paths.sort()\n",
    "am_paths.sort()\n",
    "\n",
    "paths = zip(gt_paths, am_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "5dea2f03-3ec4-4516-bb5b-7cd41cc71cc5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "63it [00:10,  5.82it/s]\n"
     ]
    }
   ],
   "source": [
    "result = {'name':[],\n",
    "         'auroc':[]}\n",
    "\n",
    "for gt, am in tqdm(paths):\n",
    "    text = gt.split(\"/\")\n",
    "    name = text[-2]+text[-1][:3]\n",
    "    \n",
    "    auroc = auroc_check(gt, am)\n",
    "    \n",
    "    result['name'].append(name)\n",
    "    result['auroc'].append(auroc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "90ee75a8-d77c-41e9-936d-d64c50dba6f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_dict(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "8f2b834c-bbdf-405b-80a2-0c55c76ed9f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9877775329311647"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['auroc'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7411d7ec-8783-4457-a77e-9e9e2e288217",
   "metadata": {},
   "source": [
    "# cable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f05e8a6d-3eac-4d42-8fe5-04f02e18dc4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_paths = glob(\"/workspace/EfficientAD-VQVAE/mvtec_anomaly_detection/cable/ground_truth/*/*.png\")\n",
    "am_paths = glob(\"/workspace/EfficientAD-VQVAE/output/1/anomaly_maps/mvtec_ad/cable/test/[!good]*/*.tiff\") # good 디렉토리 제외\n",
    "\n",
    "gt_paths.sort()\n",
    "am_paths.sort()\n",
    "\n",
    "paths = zip(gt_paths, am_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "83d2fed7-1639-45c8-b085-ee29739bcddd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "92it [00:24,  3.68it/s]\n"
     ]
    }
   ],
   "source": [
    "result = {'name':[],\n",
    "         'auroc':[]}\n",
    "\n",
    "for gt, am in tqdm(paths):\n",
    "    text = gt.split(\"/\")\n",
    "    name = text[-2]+text[-1][:3]\n",
    "    \n",
    "    auroc = auroc_check(gt, am)\n",
    "    \n",
    "    result['name'].append(name)\n",
    "    result['auroc'].append(auroc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "cc889963-edc0-424f-8038-138d7632c57f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_dict(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f17d9da2-2751-49bd-ae57-4405599dede4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9645904857064116"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['auroc'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8b72e68-2ec3-481f-85dc-750d1114ecdf",
   "metadata": {},
   "source": [
    "# capsule"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9f9a4b00-d244-4273-92bd-d5a4c5b2df65",
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_paths = glob(\"/workspace/EfficientAD-VQVAE/mvtec_anomaly_detection/capsule/ground_truth/*/*.png\")\n",
    "am_paths = glob(\"/workspace/EfficientAD-VQVAE/output/1/anomaly_maps/mvtec_ad/capsule/test/[!good]*/*.tiff\") # good 디렉토리 제외\n",
    "\n",
    "gt_paths.sort()\n",
    "am_paths.sort()\n",
    "\n",
    "paths = zip(gt_paths, am_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6db377ed-2e95-4e0a-a56a-c5dcffafe628",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "109it [00:27,  3.93it/s]\n"
     ]
    }
   ],
   "source": [
    "result = {'name':[],\n",
    "         'auroc':[]}\n",
    "\n",
    "for gt, am in tqdm(paths):\n",
    "    text = gt.split(\"/\")\n",
    "    name = text[-2]+text[-1][:3]\n",
    "    \n",
    "    auroc = auroc_check(gt, am)\n",
    "    \n",
    "    result['name'].append(name)\n",
    "    result['auroc'].append(auroc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6413d64b-62ad-49ba-8410-a0888ca4fbe3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_dict(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "efe88ae9-5430-4a4a-af9d-52c639d449ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9903656632138934"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['auroc'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23398a9a-b85d-46ea-9c9e-d467404931bc",
   "metadata": {},
   "source": [
    "# carpet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d29c681c-0cd1-4501-8f49-745ce13795e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_paths = glob(\"/workspace/EfficientAD-VQVAE/mvtec_anomaly_detection/carpet/ground_truth/*/*.png\")\n",
    "am_paths = glob(\"/workspace/EfficientAD-VQVAE/output/1/anomaly_maps/mvtec_ad/carpet/test/[!good]*/*.tiff\") # good 디렉토리 제외\n",
    "\n",
    "gt_paths.sort()\n",
    "am_paths.sort()\n",
    "\n",
    "paths = zip(gt_paths, am_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "88773040-b9c0-4cdd-b275-2fb1f9d06c15",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "89it [00:24,  3.66it/s]\n"
     ]
    }
   ],
   "source": [
    "result = {'name':[],\n",
    "         'auroc':[]}\n",
    "\n",
    "for gt, am in tqdm(paths):\n",
    "    text = gt.split(\"/\")\n",
    "    name = text[-2]+text[-1][:3]\n",
    "    \n",
    "    auroc = auroc_check(gt, am)\n",
    "    \n",
    "    result['name'].append(name)\n",
    "    result['auroc'].append(auroc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "eecfdd7a-d3fb-4382-9277-ab6e6eb03eba",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_dict(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "108023be-cec9-401c-8393-eccb39523fa3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9751179153650137"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['auroc'].mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28d767fa-8beb-4e43-bf43-085b6941ef4b",
   "metadata": {},
   "source": [
    "# grid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ce2db2cd-0f90-49f5-9c8e-328865e9b15c",
   "metadata": {},
   "outputs": [],
   "source": [
    "gt_paths = glob(\"/workspace/EfficientAD-VQVAE/mvtec_anomaly_detection/grid/ground_truth/*/*.png\")\n",
    "am_paths = glob(\"/workspace/EfficientAD-VQVAE/output/1/anomaly_maps/mvtec_ad/grid/test/[!good]*/*.tiff\") # good 디렉토리 제외\n",
    "\n",
    "gt_paths.sort()\n",
    "am_paths.sort()\n",
    "\n",
    "paths = zip(gt_paths, am_paths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "ec1feca2-9add-47e8-81d6-7b56a42d45be",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "46it [00:13,  3.53it/s]\n"
     ]
    }
   ],
   "source": [
    "result = {'name':[],\n",
    "         'auroc':[]}\n",
    "\n",
    "for gt, am in tqdm(paths):\n",
    "    text = gt.split(\"/\")\n",
    "    name = text[-2]+text[-1][:3]\n",
    "    \n",
    "    auroc = auroc_check(gt, am)\n",
    "    \n",
    "    result['name'].append(name)\n",
    "    result['auroc'].append(auroc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "87636136-3055-42a4-a8fb-281fc414dbf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame.from_dict(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ae3964e3-4050-4ae2-9bea-9280f7c42881",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>auroc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>bent000</td>\n",
       "      <td>0.998026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bent001</td>\n",
       "      <td>0.998042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bent002</td>\n",
       "      <td>0.971526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bent003</td>\n",
       "      <td>0.982195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bent004</td>\n",
       "      <td>0.992674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>bent005</td>\n",
       "      <td>0.996223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>bent006</td>\n",
       "      <td>0.958267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>bent007</td>\n",
       "      <td>0.995090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>bent008</td>\n",
       "      <td>0.945617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>bent009</td>\n",
       "      <td>0.988089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>bent010</td>\n",
       "      <td>0.987900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>bent011</td>\n",
       "      <td>0.993785</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>broken000</td>\n",
       "      <td>0.990591</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>broken001</td>\n",
       "      <td>0.993669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>broken002</td>\n",
       "      <td>0.781624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>broken003</td>\n",
       "      <td>0.991462</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>broken004</td>\n",
       "      <td>0.995580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>broken005</td>\n",
       "      <td>0.994627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>broken006</td>\n",
       "      <td>0.998327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>broken007</td>\n",
       "      <td>0.993155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>broken008</td>\n",
       "      <td>0.997151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>broken009</td>\n",
       "      <td>0.997321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>broken010</td>\n",
       "      <td>0.992132</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>broken011</td>\n",
       "      <td>0.994747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>glue000</td>\n",
       "      <td>0.605395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>glue001</td>\n",
       "      <td>0.452499</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>glue002</td>\n",
       "      <td>0.383253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>glue003</td>\n",
       "      <td>0.460886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>glue004</td>\n",
       "      <td>0.139509</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>glue005</td>\n",
       "      <td>0.561384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>glue006</td>\n",
       "      <td>0.067924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>glue007</td>\n",
       "      <td>0.267216</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>glue008</td>\n",
       "      <td>0.709954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>glue009</td>\n",
       "      <td>0.281313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>glue010</td>\n",
       "      <td>0.204817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>metal_contamination000</td>\n",
       "      <td>0.636052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>metal_contamination001</td>\n",
       "      <td>0.590590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>metal_contamination002</td>\n",
       "      <td>0.648248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>metal_contamination003</td>\n",
       "      <td>0.690202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>metal_contamination004</td>\n",
       "      <td>0.276609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>metal_contamination005</td>\n",
       "      <td>0.754997</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>metal_contamination006</td>\n",
       "      <td>0.277504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>metal_contamination007</td>\n",
       "      <td>0.104387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>metal_contamination008</td>\n",
       "      <td>0.788304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>metal_contamination009</td>\n",
       "      <td>0.565384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>metal_contamination010</td>\n",
       "      <td>0.399034</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      name     auroc\n",
       "0                  bent000  0.998026\n",
       "1                  bent001  0.998042\n",
       "2                  bent002  0.971526\n",
       "3                  bent003  0.982195\n",
       "4                  bent004  0.992674\n",
       "5                  bent005  0.996223\n",
       "6                  bent006  0.958267\n",
       "7                  bent007  0.995090\n",
       "8                  bent008  0.945617\n",
       "9                  bent009  0.988089\n",
       "10                 bent010  0.987900\n",
       "11                 bent011  0.993785\n",
       "12               broken000  0.990591\n",
       "13               broken001  0.993669\n",
       "14               broken002  0.781624\n",
       "15               broken003  0.991462\n",
       "16               broken004  0.995580\n",
       "17               broken005  0.994627\n",
       "18               broken006  0.998327\n",
       "19               broken007  0.993155\n",
       "20               broken008  0.997151\n",
       "21               broken009  0.997321\n",
       "22               broken010  0.992132\n",
       "23               broken011  0.994747\n",
       "24                 glue000  0.605395\n",
       "25                 glue001  0.452499\n",
       "26                 glue002  0.383253\n",
       "27                 glue003  0.460886\n",
       "28                 glue004  0.139509\n",
       "29                 glue005  0.561384\n",
       "30                 glue006  0.067924\n",
       "31                 glue007  0.267216\n",
       "32                 glue008  0.709954\n",
       "33                 glue009  0.281313\n",
       "34                 glue010  0.204817\n",
       "35  metal_contamination000  0.636052\n",
       "36  metal_contamination001  0.590590\n",
       "37  metal_contamination002  0.648248\n",
       "38  metal_contamination003  0.690202\n",
       "39  metal_contamination004  0.276609\n",
       "40  metal_contamination005  0.754997\n",
       "41  metal_contamination006  0.277504\n",
       "42  metal_contamination007  0.104387\n",
       "43  metal_contamination008  0.788304\n",
       "44  metal_contamination009  0.565384\n",
       "45  metal_contamination010  0.399034"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "8f7dfd4d-862f-4596-8600-3267f98e5372",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7259408675971255"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['auroc'].mean()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}