pytorch_ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Pytorch Models With H-Flip TTA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load libraries and define model parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "import os, gc, pickle\n",
    "import cv2\n",
    "import pdb\n",
    "import time\n",
    "\n",
    "import random\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import functional as F\n",
    "import torch.optim as optim\n",
    "import torch.backends.cudnn as cudnn\n",
    "from torch.utils.data import DataLoader, Dataset, sampler\n",
    "from matplotlib import pyplot as plt\n",
    "\n",
    "from albumentations import (HorizontalFlip, ShiftScaleRotate, Normalize, Resize, Compose, GaussNoise)\n",
    "from albumentations.torch import ToTensor\n",
    "\n",
    "import segmentation_models_pytorch as smp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_length_decode(rle, height=1024, width=1024, fill_value=1):\n",
    "    component = np.zeros((height, width), np.float32)\n",
    "    component = component.reshape(-1)\n",
    "    rle = np.array([int(s) for s in rle.strip().split(' ')])\n",
    "    rle = rle.reshape(-1, 2)\n",
    "    start = 0\n",
    "    for index, length in rle:\n",
    "        start = start+index\n",
    "        end = start+length\n",
    "        component[start: end] = fill_value\n",
    "        start = end\n",
    "    component = component.reshape(width, height).T\n",
    "    return component\n",
    "\n",
    "def run_length_encode(component):\n",
    "    component = component.T.flatten()\n",
    "    start = np.where(component[1:] > component[:-1])[0]+1\n",
    "    end = np.where(component[:-1] > component[1:])[0]+1\n",
    "    length = end-start\n",
    "    rle = []\n",
    "    for i in range(len(length)):\n",
    "        if i == 0:\n",
    "            rle.extend([start[0], length[0]])\n",
    "        else:\n",
    "            rle.extend([start[i]-end[i-1], length[i]])\n",
    "    rle = ' '.join([str(r) for r in rle])\n",
    "    return rle\n",
    "\n",
    "def post_process(probability, threshold, min_size):\n",
    "    mask = cv2.threshold(probability, threshold, 1, cv2.THRESH_BINARY)[1]\n",
    "    num_component, component = cv2.connectedComponents(mask.astype(np.uint8))\n",
    "    predictions = np.zeros((1024, 1024), np.float32)\n",
    "    num = 0\n",
    "    for c in range(1, num_component):\n",
    "        p = (component == c)\n",
    "        if p.sum() > min_size:\n",
    "            predictions[p] = 1\n",
    "            num += 1\n",
    "    return predictions, num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "WORK_DIR='../output/20190903_s2'\n",
    "INPUT_TEST_DIR_1024 = \"../input/1024_s2/test\"\n",
    "\n",
    "sample_submission_file='../output/20190829_s2/submission_swa_512_5ave_0p55th_minpix3500_s2_keras.csv' #to match keras model order\n",
    "submission_file='/home/akuritsyn/projects/pneumothorax/output/20190903_s2/submission_pytorch_5fold_ave_Wflip_0p55th.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "PYTORCH_MODELS_DIR='../output/20190903_pytorch_results_s2'\n",
    "\n",
    "path_models=[PYTORCH_MODELS_DIR+'/1024_stage2_model_fold0.pth',\\\n",
    "             PYTORCH_MODELS_DIR+'/1024_stage2_model_fold1.pth',\\\n",
    "             PYTORCH_MODELS_DIR+'/1024_stage2_model_fold2.pth',\\\n",
    "             PYTORCH_MODELS_DIR+'/1024_stage2_model_fold3.pth',\\\n",
    "             PYTORCH_MODELS_DIR+'/1024_stage2_model_fold4.pth']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Model parameters\n",
    "\n",
    "size = 1024\n",
    "mean = (0.485, 0.456, 0.406)\n",
    "std = (0.229, 0.224, 0.225)\n",
    "num_workers = 4 #8\n",
    "batch_size = 1 #4\n",
    "best_threshold = 0.55\n",
    "min_size = 3500\n",
    "device = torch.device(\"cuda:0\")\n",
    "df = pd.read_csv(sample_submission_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform inference on test images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestDataset(Dataset):\n",
    "    def __init__(self, root, df, size, mean, std, tta=4):\n",
    "        self.root = root\n",
    "        self.size = size\n",
    "        self.fnames = list(df[\"ImageId\"])\n",
    "        self.num_samples = len(self.fnames)\n",
    "        self.transform = Compose(\n",
    "            [\n",
    "                Normalize(mean=mean, std=std, p=1), #p=1\n",
    "                Resize(size, size),\n",
    "                ToTensor(),\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        fname = self.fnames[idx]\n",
    "        path = os.path.join(self.root, fname + \".png\")\n",
    "        image = cv2.imread(path)\n",
    "        images = self.transform(image=image)[\"image\"]\n",
    "        return images\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.num_samples   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Unet(\n",
       "  (encoder): ResNetEncoder(\n",
       "    (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "    (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (relu): ReLU(inplace)\n",
       "    (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "    (layer1): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (layer2): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (layer3): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (layer4): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
       "  )\n",
       "  (decoder): UnetDecoder(\n",
       "    (layer1): DecoderBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(768, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (1): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (layer2): DecoderBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(384, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (1): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (layer3): DecoderBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(192, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (1): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (layer4): DecoderBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(128, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (1): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(32, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (layer5): DecoderBlock(\n",
       "      (block): Sequential(\n",
       "        (0): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(32, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (1): Conv2dReLU(\n",
       "          (block): Sequential(\n",
       "            (0): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (1): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (2): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (final_conv): Conv2d(16, 1, kernel_size=(1, 1), stride=(1, 1))\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testset = DataLoader(\n",
    "    TestDataset(INPUT_TEST_DIR_1024, df, size, mean, std),\n",
    "    batch_size=batch_size,\n",
    "    shuffle=False,\n",
    "    num_workers=num_workers,\n",
    "    pin_memory=True,\n",
    ")\n",
    "\n",
    "model_0 = smp.Unet(\"resnet34\", encoder_weights=None, activation=None)\n",
    "model_0.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fbe0dc7a5f94e4fa1438808f6824ae3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "probs=[]\n",
    "\n",
    "for i, batch in enumerate(tqdm(testset)):\n",
    "     \n",
    "    for j in range(0,5): #average over 5 folds\n",
    "\n",
    "        checkpoint0= torch.load(path_models[j], map_location=lambda storage, loc: storage)\n",
    "        model_0.load_state_dict(checkpoint0[\"state_dict\"])\n",
    "        model_0.to(device)\n",
    "        \n",
    "        if j==0:\n",
    "            preds = torch.sigmoid(model_0(batch.to(device)))\n",
    "        else:\n",
    "            preds = preds+torch.sigmoid(model_0(batch.to(device)))\n",
    "         \n",
    "        model_0.cpu()\n",
    "    \n",
    "    preds=preds/5.\n",
    "    preds = preds.detach().cpu().numpy()[:, 0, :, :] # (batch_size, 1, size, size) -> (batch_size, size, size)\n",
    "    for probability in preds:\n",
    "        if probability.shape != (1024, 1024):\n",
    "            probability = cv2.resize(probability, dsize=(1024, 1024), interpolation=cv2.INTER_LINEAR)\n",
    "        probs.append(probability)      \n",
    "\n",
    "with open(WORK_DIR+'/probs_1024.pkl', 'wb') as f:\n",
    "     pickle.dump(probs, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# del probs\n",
    "# gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(WORK_DIR+'/probs_1024.pkl', 'rb') as f:\n",
    "#     probs=pickle.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Perform inference on h-flipped test images (may need to restart kernel to free up memory before proceeding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "367dc68bedce468b954061d52cfa7441",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3205), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "class TestDataset_Flipped(Dataset):\n",
    "    def __init__(self, root, df, size, mean, std):\n",
    "        self.root = root\n",
    "        self.size = size\n",
    "        self.fnames = list(df[\"ImageId\"])\n",
    "        self.num_samples = len(self.fnames)\n",
    "        self.transform = Compose(\n",
    "            [\n",
    "                Normalize(mean=mean, std=std, p=1),\n",
    "                Resize(size, size),\n",
    "                HorizontalFlip(True,p=1),\n",
    "                ToTensor(),\n",
    "            ]\n",
    "        )\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        fname = self.fnames[idx]\n",
    "        path = os.path.join(self.root, fname + \".png\")\n",
    "        image = cv2.imread(path)\n",
    "        images = self.transform(image=image)[\"image\"]\n",
    "        return images\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.num_samples\n",
    "\n",
    "\n",
    "#flipped_prob\n",
    "testset_flipped = DataLoader(\n",
    "    TestDataset_Flipped(INPUT_TEST_DIR_1024, df, size, mean, std),\n",
    "    batch_size=batch_size,\n",
    "    shuffle=False,\n",
    "    num_workers=num_workers,\n",
    "    pin_memory=True,\n",
    ")\n",
    "\n",
    "probs_flipped=[]\n",
    "for i, batch in enumerate(tqdm(testset_flipped)):\n",
    "    \n",
    "    for j in range(0,5): \n",
    "        checkpoint0= torch.load(path_models[j], map_location=lambda storage, loc: storage)\n",
    "        model_0.load_state_dict(checkpoint0[\"state_dict\"])\n",
    "        model_0.to(device)\n",
    "        if j==0:\n",
    "            preds = torch.sigmoid(model_0(batch.to(device)))\n",
    "        else:\n",
    "            preds = preds+torch.sigmoid(model_0(batch.to(device)))\n",
    "        model_0.cpu()\n",
    "        #del model_0\n",
    "        #gc.collect()\n",
    "        \n",
    "    preds=preds/5.\n",
    "    preds = preds.detach().cpu().numpy()[:, 0, :, :] # (batch_size, 1, size, size) -> (batch_size, size, size)\n",
    "    for probability in preds:\n",
    "        if probability.shape != (1024, 1024):\n",
    "            probability = cv2.resize(probability, dsize=(1024, 1024), interpolation=cv2.INTER_LINEAR)\n",
    "        probs_flipped.append(np.fliplr(probability))\n",
    "\n",
    "with open(WORK_DIR+'/probs_flipped_1024.pkl', 'wb') as f:\n",
    "     pickle.dump(probs_flipped, f)  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "27"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del model_0\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Average predicitions for normal and flipped images and generate submission file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(WORK_DIR+'/probs_flipped_1024.pkl', 'rb') as f:\n",
    "#     probs_flipped=pickle.load(f)\n",
    "\n",
    "with open(WORK_DIR+'/probs_1024.pkl', 'rb') as f:\n",
    "    probs=pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(probs)):\n",
    "    probs[i]=(probs[i]+probs_flipped[i])/2.\n",
    "    \n",
    "del probs_flipped\n",
    "gc.collect()\n",
    "\n",
    "# with open(WORK_DIR+'/probs_ave_1024.pkl', 'wb') as f:\n",
    "#      pickle.dump(probs, f) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "del probs\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(WORK_DIR+'/probs_ave_1024.pkl', 'rb') as f:\n",
    "#     probs=pickle.load(f) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate submission file\n",
    "\n",
    "encoded_pixels = []\n",
    "\n",
    "for probability in probs:\n",
    "    predict, num_predict = post_process(probability, best_threshold, min_size)\n",
    "    if num_predict == 0:\n",
    "        encoded_pixels.append('-1')\n",
    "    else: \n",
    "        r = run_length_encode(predict)\n",
    "        encoded_pixels.append(r)\n",
    "df['EncodedPixels'] = encoded_pixels\n",
    "df.to_csv(submission_file, columns=['ImageId', 'EncodedPixels'], index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ImageId</th>\n",
       "      <th>EncodedPixels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ID_743b284f8</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ID_ec235550f</td>\n",
       "      <td>148308 37 979 60 958 77 939 94 921 111 905 131...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ID_76747ee35</td>\n",
       "      <td>170718 8 999 32 982 43 971 54 962 61 958 65 95...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ID_71d2042e6</td>\n",
       "      <td>248138 6 1014 12 1010 15 1007 18 1004 20 1002 ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ID_6023cb909</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>ID_b439e7bb3</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>ID_93ace8dc6</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>ID_bf9328240</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ID_f9fa52a40</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ID_8a837104a</td>\n",
       "      <td>588929 5 45 6 965 14 35 12 962 20 27 16 959 34...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>ID_f9e28e10e</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>ID_c30e7e79c</td>\n",
       "      <td>322948 15 1005 20 1001 23 998 26 996 27 995 29...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>ID_120d87b8e</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ID_46bb707b7</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ID_71d106e5e</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ID_07ee731bc</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ID_647aebf99</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ID_055cf45a9</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ID_de0ef3c02</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>ID_6196e28d7</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>ID_7f554a3b1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>ID_fc9937a91</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>ID_7ba1a5058</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>ID_79fb9ddfb</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>ID_b5b1237c0</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>ID_3290afa63</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>ID_8bbbf994b</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>ID_3ca44169d</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>ID_f5ffeced4</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>ID_33aaf5ea1</td>\n",
       "      <td>595186 7 1014 13 1009 17 1005 19 1004 21 1002 ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         ImageId                                      EncodedPixels\n",
       "0   ID_743b284f8                                                 -1\n",
       "1   ID_ec235550f  148308 37 979 60 958 77 939 94 921 111 905 131...\n",
       "2   ID_76747ee35  170718 8 999 32 982 43 971 54 962 61 958 65 95...\n",
       "3   ID_71d2042e6  248138 6 1014 12 1010 15 1007 18 1004 20 1002 ...\n",
       "4   ID_6023cb909                                                 -1\n",
       "5   ID_b439e7bb3                                                 -1\n",
       "6   ID_93ace8dc6                                                 -1\n",
       "7   ID_bf9328240                                                 -1\n",
       "8   ID_f9fa52a40                                                 -1\n",
       "9   ID_8a837104a  588929 5 45 6 965 14 35 12 962 20 27 16 959 34...\n",
       "10  ID_f9e28e10e                                                 -1\n",
       "11  ID_c30e7e79c  322948 15 1005 20 1001 23 998 26 996 27 995 29...\n",
       "12  ID_120d87b8e                                                 -1\n",
       "13  ID_46bb707b7                                                 -1\n",
       "14  ID_71d106e5e                                                 -1\n",
       "15  ID_07ee731bc                                                 -1\n",
       "16  ID_647aebf99                                                 -1\n",
       "17  ID_055cf45a9                                                 -1\n",
       "18  ID_de0ef3c02                                                 -1\n",
       "19  ID_6196e28d7                                                 -1\n",
       "20  ID_7f554a3b1                                                 -1\n",
       "21  ID_fc9937a91                                                 -1\n",
       "22  ID_7ba1a5058                                                 -1\n",
       "23  ID_79fb9ddfb                                                 -1\n",
       "24  ID_b5b1237c0                                                 -1\n",
       "25  ID_3290afa63                                                 -1\n",
       "26  ID_8bbbf994b                                                 -1\n",
       "27  ID_3ca44169d                                                 -1\n",
       "28  ID_f5ffeced4                                                 -1\n",
       "29  ID_33aaf5ea1  595186 7 1014 13 1009 17 1005 19 1004 21 1002 ..."
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": true,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}