wind_power_normalizing_flow.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "wind power - normalizing flow",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "install packages"
      ],
      "metadata": {
        "id": "TWESErkypg0A"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ex5XnQKD-5j5"
      },
      "outputs": [],
      "source": [
        "!pip install UMNN\n",
        "!pip install properscoring\n",
        "!pip install git+https://github.com/bayesiains/nflows.git\n",
        "!pip install torch-summary"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "import packages"
      ],
      "metadata": {
        "id": "HYYoQ4WBpjUr"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "o344x9rX-_gX"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from torch import nn\n",
        "from torch import optim\n",
        "\n",
        "from nflows.flows.base import Flow\n",
        "from nflows.distributions.normal import ConditionalDiagonalNormal, StandardNormal\n",
        "from nflows.transforms.base import CompositeTransform\n",
        "from nflows.transforms.autoregressive import MaskedAffineAutoregressiveTransform, MaskedPiecewiseQuadraticAutoregressiveTransform, MaskedPiecewiseLinearAutoregressiveTransform\n",
        "from nflows.transforms.permutations import ReversePermutation\n",
        "from nflows.transforms.autoregressive import MaskedUMNNAutoregressiveTransform, MaskedPiecewiseRationalQuadraticAutoregressiveTransform\n",
        "#from nflows.transforms.nonlinearities import Logit\n",
        "from nflows.nn.nets import ResidualNet\n",
        "from nflows.transforms.base import (\n",
        "    CompositeTransform,\n",
        "    InputOutsideDomain,\n",
        "    InverseTransform,\n",
        "    Transform,\n",
        ")\n",
        "\n",
        "\n",
        "from nflows.transforms.autoregressive import AutoregressiveTransform\n",
        "from nflows.transforms import made as made_module\n",
        "from nflows.utils import torchutils\n",
        "from torch.nn import functional as F"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "OqxSwFo6FxnF"
      },
      "outputs": [],
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "y09wxBAiMjKC"
      },
      "outputs": [],
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import sklearn.datasets as datasets\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import copy"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "import data"
      ],
      "metadata": {
        "id": "2abgDK-tp-jF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "W0bsx1rCBq6B"
      },
      "outputs": [],
      "source": [
        "def fill_ndarray(t1):\n",
        "    for i in range(t1.shape[1]):  \n",
        "        temp_col = t1[:, i]  \n",
        "        nan_num = np.count_nonzero(temp_col != temp_col)\n",
        "        if nan_num != 0:  \n",
        "            temp_not_nan_col = temp_col[temp_col == temp_col]  \n",
        "            temp_col[np.isnan(temp_col)] = temp_not_nan_col.mean()  \n",
        "    return t1\n",
        "\n",
        "def get_data(df):\n",
        "  data = df['TARGETVAR'].values.reshape((-1,1))\n",
        "  return data\n",
        "\n",
        "def get_nwp(d):\n",
        "  cls = d.columns\n",
        "  data = []\n",
        "  for i in range(4):\n",
        "    data.append(d[cls[i+3]].values.reshape((-1,1)))\n",
        "  data = np.hstack(data)\n",
        "  return data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "moYRZggM0Trg"
      },
      "outputs": [],
      "source": [
        "data = pd.read_csv('Task15_W_Zone1.csv',delimiter=',')\n",
        "\n",
        "power = get_data(data); x = np.float32(get_nwp(data))\n",
        "y = np.float32(fill_ndarray(power))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Idz9gWX-0IHL"
      },
      "outputs": [],
      "source": [
        "y = torch.tensor(y).float()\n",
        "x = torch.tensor(x).float()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8HaM-XoiKhKU"
      },
      "outputs": [],
      "source": [
        "import math\n",
        "\n",
        "index = math.floor(x.shape[0]*0.8)\n",
        "\n",
        "y_train = y[:index]\n",
        "x_train = x[:index,:]\n",
        "\n",
        "y_test = y[index:]\n",
        "x_test = x[index:,:]\n",
        "\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "  x_train, y_train, x_test, y_test = x_train.cuda(), y_train.cuda(), x_test.cuda(), y_test.cuda()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yNG1s69YB2jg",
        "outputId": "986a5153-cccc-4fcc-9358-72a305a8af33"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([13440, 4]) torch.Size([13440, 1]) torch.Size([3360, 4]) torch.Size([3360, 1])\n"
          ]
        }
      ],
      "source": [
        "print(x_train.shape,y_train.shape,x_test.shape,y_test.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "logit normal transform"
      ],
      "metadata": {
        "id": "3WsjZsRnqDIm"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IkqiWA-u12T-"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "class Sigmoid(Transform):\n",
        "    def __init__(self, temperature=1, eps=1e-6, learn_temperature=False):\n",
        "        super().__init__()\n",
        "        self.eps = eps\n",
        "        if learn_temperature:\n",
        "            self.temperature = nn.Parameter(torch.Tensor([temperature]))\n",
        "        else:\n",
        "            self.temperature = torch.Tensor([temperature]).to(device)\n",
        "\n",
        "    def forward(self, inputs, context=None):\n",
        "        inputs = self.temperature * inputs\n",
        "        outputs = torch.sigmoid(inputs)\n",
        "        logabsdet = torchutils.sum_except_batch(\n",
        "            torch.log(self.temperature) - F.softplus(-inputs) - F.softplus(inputs)\n",
        "        )\n",
        "        return outputs, logabsdet\n",
        "\n",
        "    def inverse(self, inputs, context=None):\n",
        "        if torch.min(inputs) < 0 or torch.max(inputs) > 1:\n",
        "            raise InputOutsideDomain()\n",
        "\n",
        "        inputs = torch.clamp(inputs, self.eps, 1 - self.eps).to(device)\n",
        "\n",
        "        outputs = (1 / self.temperature) * (torch.log(inputs) - torch.log1p(-inputs))\n",
        "        logabsdet = -torchutils.sum_except_batch(\n",
        "            torch.log(self.temperature)\n",
        "            - F.softplus(-self.temperature * outputs)\n",
        "            - F.softplus(self.temperature * outputs)\n",
        "        )\n",
        "        return outputs, logabsdet\n",
        "\n",
        "\n",
        "class Logit(InverseTransform):\n",
        "    def __init__(self, temperature=1, eps=1e-6):\n",
        "        super().__init__(Sigmoid(temperature=temperature, eps=eps))\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "linear spline"
      ],
      "metadata": {
        "id": "XHha87ZkqGKj"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hqAg6oLOFpGf"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "from nflows.transforms.base import InputOutsideDomain\n",
        "from nflows.utils import torchutils\n",
        "\n",
        "\n",
        "def unconstrained_linear_spline(\n",
        "    inputs, unnormalized_pdf, inverse=False, tail_bound=1.0, tails=\"linear\"\n",
        "):\n",
        "    inside_interval_mask = (inputs >= -tail_bound) & (inputs <= tail_bound)\n",
        "    outside_interval_mask = ~inside_interval_mask\n",
        "\n",
        "    outputs = torch.zeros_like(inputs)\n",
        "    logabsdet = torch.zeros_like(inputs)\n",
        "\n",
        "    if tails == \"linear\":\n",
        "        outputs[outside_interval_mask] = inputs[outside_interval_mask]\n",
        "        logabsdet[outside_interval_mask] = 0\n",
        "    else:\n",
        "        raise RuntimeError(\"{} tails are not implemented.\".format(tails))\n",
        "\n",
        "    if torch.any(inside_interval_mask):\n",
        "        outputs[inside_interval_mask], logabsdet[inside_interval_mask] = linear_spline(\n",
        "            inputs=inputs[inside_interval_mask],\n",
        "            unnormalized_pdf=unnormalized_pdf[inside_interval_mask, :],\n",
        "            inverse=inverse,\n",
        "            left=-tail_bound,\n",
        "            right=tail_bound,\n",
        "            bottom=-tail_bound,\n",
        "            top=tail_bound,\n",
        "        )\n",
        "\n",
        "    return outputs, logabsdet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oVfSY7lBFshp"
      },
      "outputs": [],
      "source": [
        "def linear_spline(\n",
        "    inputs, unnormalized_pdf, inverse=False, left=0.0, right=1.0, bottom=0.0, top=1.0\n",
        "):\n",
        "    \"\"\"\n",
        "    Reference:\n",
        "    > Müller et al., Neural Importance Sampling, arXiv:1808.03856, 2018.\n",
        "    \"\"\"\n",
        "    if torch.min(inputs) < left or torch.max(inputs) > right:\n",
        "        raise InputOutsideDomain()\n",
        "\n",
        "    if inverse:\n",
        "        inputs = (inputs - bottom) / (top - bottom)\n",
        "    else:\n",
        "        inputs = (inputs - left) / (right - left)\n",
        "\n",
        "    num_bins = unnormalized_pdf.size(-1)\n",
        "\n",
        "    pdf = F.softmax(unnormalized_pdf, dim=-1)\n",
        "\n",
        "    cdf = torch.cumsum(pdf, dim=-1)\n",
        "    cdf[..., -1] = 1.0\n",
        "    cdf = F.pad(cdf, pad=(1, 0), mode=\"constant\", value=0.0)\n",
        "\n",
        "    if inverse:\n",
        "        inv_bin_idx = torchutils.searchsorted(cdf, inputs)\n",
        "\n",
        "        bin_boundaries = (\n",
        "            torch.linspace(0, 1, num_bins + 1)\n",
        "            .view([1] * inputs.dim() + [-1])\n",
        "            .expand(*inputs.shape, -1)\n",
        "        ).to(device)\n",
        "\n",
        "        slopes = (cdf[..., 1:] - cdf[..., :-1]) / (\n",
        "            bin_boundaries[..., 1:] - bin_boundaries[..., :-1]\n",
        "        )\n",
        "        offsets = cdf[..., 1:] - slopes * bin_boundaries[..., 1:]\n",
        "\n",
        "        inv_bin_idx = inv_bin_idx.unsqueeze(-1)\n",
        "        input_slopes = slopes.gather(-1, inv_bin_idx)[..., 0]\n",
        "        input_offsets = offsets.gather(-1, inv_bin_idx)[..., 0]\n",
        "\n",
        "        outputs = (inputs - input_offsets) / input_slopes\n",
        "        outputs = torch.clamp(outputs, 0, 1)\n",
        "\n",
        "        logabsdet = -torch.log(input_slopes)\n",
        "    else:\n",
        "        bin_pos = inputs * num_bins\n",
        "\n",
        "        bin_idx = torch.floor(bin_pos).long()\n",
        "        bin_idx[bin_idx >= num_bins] = num_bins - 1\n",
        "\n",
        "        alpha = bin_pos - bin_idx.float()\n",
        "\n",
        "        input_pdfs = pdf.gather(-1, bin_idx[..., None])[..., 0]\n",
        "\n",
        "        outputs = cdf.gather(-1, bin_idx[..., None])[..., 0]\n",
        "        outputs += alpha * input_pdfs\n",
        "        outputs = torch.clamp(outputs, 0, 1)\n",
        "\n",
        "        bin_width = 1.0 / num_bins\n",
        "        logabsdet = torch.log(input_pdfs) - np.log(bin_width)\n",
        "\n",
        "    if inverse:\n",
        "        outputs = outputs * (right - left) + left\n",
        "    else:\n",
        "        outputs = outputs * (top - bottom) + bottom\n",
        "\n",
        "    return outputs, logabsdet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "-_K5Nu-I-dVO"
      },
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "class MaskedUnconstrainedPiecewiseLinearAutoregressiveTransform(AutoregressiveTransform):\n",
        "    def __init__(\n",
        "        self,\n",
        "        num_bins,\n",
        "        features,\n",
        "        hidden_features,\n",
        "        context_features=None,\n",
        "        num_blocks=2,\n",
        "        use_residual_blocks=True,\n",
        "        random_mask=False,\n",
        "        activation=F.relu,\n",
        "        dropout_probability=0.0,\n",
        "        use_batch_norm=False,\n",
        "        tail_bound=1.0,\n",
        "        tail = 'linear',\n",
        "    ):\n",
        "        self.num_bins = num_bins\n",
        "        self.features = features\n",
        "        self.tail_bound = tail_bound\n",
        "        self.tail = tail\n",
        "        made = made_module.MADE(\n",
        "            features=features,\n",
        "            hidden_features=hidden_features,\n",
        "            context_features=context_features,\n",
        "            num_blocks=num_blocks,\n",
        "            output_multiplier=self._output_dim_multiplier(),\n",
        "            use_residual_blocks=use_residual_blocks,\n",
        "            random_mask=random_mask,\n",
        "            activation=activation,\n",
        "            dropout_probability=dropout_probability,\n",
        "            use_batch_norm=use_batch_norm,\n",
        "        )\n",
        "        super().__init__(made)\n",
        "\n",
        "    def _output_dim_multiplier(self):\n",
        "        return self.num_bins\n",
        "\n",
        "    def _elementwise(self, inputs, autoregressive_params, inverse=False):\n",
        "        batch_size = inputs.shape[0]\n",
        "\n",
        "        unnormalized_pdf = autoregressive_params.view(\n",
        "            batch_size, self.features, self._output_dim_multiplier()\n",
        "        )\n",
        "        \n",
        "        outputs, logabsdet = unconstrained_linear_spline(\n",
        "            inputs=inputs, unnormalized_pdf=unnormalized_pdf, inverse=inverse, tail_bound=self.tail_bound, tails=self.tail\n",
        "        )\n",
        "\n",
        "        return outputs, torchutils.sum_except_batch(logabsdet)\n",
        "\n",
        "    def _elementwise_forward(self, inputs, autoregressive_params):\n",
        "        return self._elementwise(inputs, autoregressive_params)\n",
        "\n",
        "    def _elementwise_inverse(self, inputs, autoregressive_params):\n",
        "        return self._elementwise(inputs, autoregressive_params, inverse=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "cubic spline"
      ],
      "metadata": {
        "id": "8DU8rFeaqN3f"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0nGfVKj0KQz-"
      },
      "outputs": [],
      "source": [
        "from nflows.transforms.splines.cubic import unconstrained_cubic_spline\n",
        "\n",
        "class MaskedUnconstrainedPiecewiseCubicAutoregressiveTransform(AutoregressiveTransform):\n",
        "    def __init__(\n",
        "        self,\n",
        "        num_bins,\n",
        "        features,\n",
        "        hidden_features,\n",
        "        context_features=None,\n",
        "        num_blocks=2,\n",
        "        use_residual_blocks=True,\n",
        "        random_mask=False,\n",
        "        activation=F.relu,\n",
        "        dropout_probability=0.0,\n",
        "        use_batch_norm=False,\n",
        "    ):\n",
        "        self.num_bins = num_bins\n",
        "        self.features = features\n",
        "        made = made_module.MADE(\n",
        "            features=features,\n",
        "            hidden_features=hidden_features,\n",
        "            context_features=context_features,\n",
        "            num_blocks=num_blocks,\n",
        "            output_multiplier=self._output_dim_multiplier(),\n",
        "            use_residual_blocks=use_residual_blocks,\n",
        "            random_mask=random_mask,\n",
        "            activation=activation,\n",
        "            dropout_probability=dropout_probability,\n",
        "            use_batch_norm=use_batch_norm,\n",
        "        )\n",
        "        super(MaskedUnconstrainedPiecewiseCubicAutoregressiveTransform, self).__init__(made)\n",
        "\n",
        "    def _output_dim_multiplier(self):\n",
        "        return self.num_bins * 2 + 2\n",
        "\n",
        "    def _elementwise(self, inputs, autoregressive_params, inverse=False):\n",
        "        batch_size = inputs.shape[0]\n",
        "\n",
        "        transform_params = autoregressive_params.view(\n",
        "            batch_size, self.features, self.num_bins * 2 + 2\n",
        "        )\n",
        "\n",
        "        unnormalized_widths = transform_params[..., : self.num_bins]\n",
        "        unnormalized_heights = transform_params[..., self.num_bins : 2 * self.num_bins]\n",
        "        derivatives = transform_params[..., 2 * self.num_bins :]\n",
        "        unnorm_derivatives_left = derivatives[..., 0][..., None]\n",
        "        unnorm_derivatives_right = derivatives[..., 1][..., None]\n",
        "\n",
        "        if hasattr(self.autoregressive_net, \"hidden_features\"):\n",
        "            unnormalized_widths /= np.sqrt(self.autoregressive_net.hidden_features)\n",
        "            unnormalized_heights /= np.sqrt(self.autoregressive_net.hidden_features)\n",
        "\n",
        "        outputs, logabsdet = unconstrained_cubic_spline(\n",
        "            inputs=inputs,\n",
        "            unnormalized_widths=unnormalized_widths,\n",
        "            unnormalized_heights=unnormalized_heights,\n",
        "            unnorm_derivatives_left=unnorm_derivatives_left,\n",
        "            unnorm_derivatives_right=unnorm_derivatives_right,\n",
        "            inverse=inverse,\n",
        "            tail_bound=1.0,\n",
        "            tails=\"linear\",\n",
        "        )\n",
        "        return outputs, torchutils.sum_except_batch(logabsdet)\n",
        "\n",
        "    def _elementwise_forward(self, inputs, autoregressive_params):\n",
        "        return self._elementwise(inputs, autoregressive_params)\n",
        "\n",
        "    def _elementwise_inverse(self, inputs, autoregressive_params):\n",
        "        return self._elementwise(inputs, autoregressive_params, inverse=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "base distribution"
      ],
      "metadata": {
        "id": "h-dWYECYqQN5"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Db5hH0sID5Z7"
      },
      "outputs": [],
      "source": [
        "model = nn.Sequential(\n",
        "          nn.Linear(4, 512),\n",
        "          nn.ReLU(),\n",
        "          nn.Linear(512, 512),\n",
        "          nn.ReLU(),\n",
        "          nn.Linear(512, 2)\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uo6vIglb_MXh"
      },
      "outputs": [],
      "source": [
        "num_layers = 5\n",
        "base_dist = ConditionalDiagonalNormal(shape=[1],context_encoder=model)\n",
        "#base_dist = StandardNormal(shape=[1])\n",
        "\n",
        "transforms = []\n",
        "\n",
        "#transforms.append(Logit(eps=1e-3))\n",
        "#logit needs a cuda version\n",
        "for _ in range(num_layers):\n",
        "    transforms.append(ReversePermutation(features=1))\n",
        "    #transforms.append(MaskedPiecewiseRationalQuadraticAutoregressiveTransform(features=1,hidden_features=64,context_features=4,num_bins=10,tails = \"linear\"))\n",
        "    #transforms.append(MaskedPiecewiseQuadraticAutoregressiveTransform(features=1,hidden_features=256,context_features=4,num_bins=10,tails = \"linear\"))\n",
        "    #transforms.append(MaskedAffineAutoregressiveTransform(features=1,hidden_features=256,context_features=4))\n",
        "    #transforms.append(MaskedUMNNAutoregressiveTransform(features=1,hidden_features=512,context_features=4,integrand_net_layers=[256, 256, 256]))\n",
        "    #transforms.append(MaskedUnconstrainedPiecewiseLinearAutoregressiveTransform(features=1,hidden_features=256,context_features=4,num_bins=10,tail_bound=1.0,tail = 'linear'))\n",
        "    transforms.append(MaskedUnconstrainedPiecewiseCubicAutoregressiveTransform(features=1,hidden_features=256,context_features=4,num_bins=10))\n",
        "\n",
        "transform = CompositeTransform(transforms)\n",
        "\n",
        "flow = Flow(transform, base_dist)\n",
        "if torch.cuda.is_available():\n",
        "  flow = flow.cuda()\n",
        "\n",
        "#optimizer = optim.Adam(flow.parameters(),lr=1e-3)\n",
        "optimizer = optim.Adam(flow.parameters(),lr=1e-4)\n",
        "scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[300,600], gamma=0.3)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DaFHFA4mAQfu",
        "outputId": "8b2a1ebf-d52f-4944-c9c2-0e6f31d6c3b1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "iteration 0 : 4.316174030303955\n",
            "iteration 10 : 0.30816659331321716\n",
            "iteration 20 : -0.16254831850528717\n",
            "iteration 30 : -0.40090006589889526\n",
            "iteration 40 : -0.52266526222229\n",
            "iteration 50 : -0.5761312246322632\n",
            "iteration 60 : -0.6520320773124695\n",
            "iteration 70 : -0.6509710550308228\n",
            "iteration 80 : -0.7001732587814331\n",
            "iteration 90 : -0.6699402332305908\n",
            "iteration 100 : -0.7307687401771545\n",
            "iteration 110 : -0.5880226492881775\n",
            "iteration 120 : -0.6920303106307983\n",
            "iteration 130 : -0.6274837255477905\n",
            "iteration 140 : -0.6343744993209839\n",
            "iteration 150 : -0.7400908470153809\n",
            "iteration 160 : -0.7739614844322205\n",
            "iteration 170 : -0.6412115693092346\n",
            "iteration 180 : -0.7610052227973938\n",
            "iteration 190 : -0.80299973487854\n",
            "iteration 200 : -0.7626189589500427\n",
            "iteration 210 : -0.741364598274231\n",
            "iteration 220 : -0.746520459651947\n",
            "iteration 230 : -0.8133448362350464\n",
            "iteration 240 : -0.8212624788284302\n",
            "iteration 250 : -0.3700930178165436\n",
            "iteration 260 : -0.5978741645812988\n",
            "iteration 270 : -0.7119569778442383\n",
            "iteration 280 : -0.7638779878616333\n",
            "iteration 290 : -0.7678177356719971\n",
            "iteration 300 : -0.7508945465087891\n",
            "iteration 310 : -0.8042075037956238\n",
            "iteration 320 : -0.8243792057037354\n",
            "iteration 330 : -0.836532473564148\n",
            "iteration 340 : -0.846065878868103\n",
            "iteration 350 : -0.853455126285553\n",
            "iteration 360 : -0.8600254654884338\n",
            "iteration 370 : -0.865541934967041\n",
            "iteration 380 : -0.8673473596572876\n",
            "iteration 390 : -0.8646907806396484\n",
            "iteration 400 : -0.7176947593688965\n",
            "iteration 410 : -0.7745863199234009\n",
            "iteration 420 : -0.8143675327301025\n",
            "iteration 430 : -0.8346801400184631\n",
            "iteration 440 : -0.8497965931892395\n",
            "iteration 450 : -0.8626810312271118\n",
            "iteration 460 : -0.872045636177063\n",
            "iteration 470 : -0.8571828007698059\n",
            "iteration 480 : -0.8727409243583679\n",
            "iteration 490 : -0.8747173547744751\n",
            "iteration 500 : -0.881097137928009\n",
            "iteration 510 : -0.8180832862854004\n",
            "iteration 520 : -0.8364372849464417\n",
            "iteration 530 : -0.873160719871521\n",
            "iteration 540 : -0.8830645680427551\n",
            "iteration 550 : -0.8632269501686096\n",
            "iteration 560 : -0.7100268006324768\n",
            "iteration 570 : -0.7795568704605103\n",
            "iteration 580 : -0.8179764747619629\n",
            "iteration 590 : -0.8445873856544495\n",
            "iteration 600 : -0.8631468415260315\n",
            "iteration 610 : -0.8680980205535889\n",
            "iteration 620 : -0.8718639612197876\n",
            "iteration 630 : -0.8753377795219421\n",
            "iteration 640 : -0.8785560727119446\n",
            "iteration 650 : -0.8816401958465576\n",
            "iteration 660 : -0.8846266865730286\n",
            "iteration 670 : -0.8876006007194519\n",
            "iteration 680 : -0.8906515836715698\n",
            "iteration 690 : -0.8937064409255981\n",
            "iteration 700 : -0.8967309594154358\n",
            "iteration 710 : -0.8998087644577026\n",
            "iteration 720 : -0.9025523066520691\n",
            "iteration 730 : -0.9049345850944519\n",
            "iteration 740 : -0.9069094061851501\n",
            "iteration 750 : -0.9088620543479919\n",
            "iteration 760 : -0.9106212854385376\n",
            "iteration 770 : -0.9124078750610352\n",
            "iteration 780 : -0.9090989232063293\n",
            "iteration 790 : -0.9106115102767944\n",
            "iteration 800 : -0.9134814739227295\n",
            "iteration 810 : -0.9163721203804016\n",
            "iteration 820 : -0.9173038005828857\n",
            "iteration 830 : -0.9190891981124878\n",
            "iteration 840 : -0.9210375547409058\n",
            "iteration 850 : -0.9223110675811768\n",
            "iteration 860 : -0.9238413572311401\n",
            "iteration 870 : -0.9235579371452332\n",
            "iteration 880 : -0.9142647981643677\n",
            "iteration 890 : -0.9219720959663391\n",
            "iteration 900 : -0.9210161566734314\n",
            "iteration 910 : -0.9174928665161133\n",
            "iteration 920 : -0.9216204285621643\n",
            "iteration 930 : -0.9272332191467285\n",
            "iteration 940 : -0.9297204613685608\n",
            "iteration 950 : -0.9310462474822998\n",
            "iteration 960 : -0.9319032430648804\n",
            "iteration 970 : -0.9267440438270569\n",
            "iteration 980 : -0.9240534901618958\n",
            "iteration 990 : -0.9266373515129089\n"
          ]
        }
      ],
      "source": [
        "num_iter = 1000\n",
        "for i in range(num_iter):\n",
        "    optimizer.zero_grad()\n",
        "    loss = -flow.log_prob(inputs=y_train, context=x_train).mean()\n",
        "    if i%10 == 0:\n",
        "      print('iteration',i,':',loss.item())\n",
        "    loss.backward()\n",
        "    optimizer.step()\n",
        "    scheduler.step()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wFO1pA2LA703"
      },
      "outputs": [],
      "source": [
        "import properscoring as ps\n",
        "y_true = y_test.cpu().numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zEHsnfg2cKuH",
        "outputId": "676a5387-8b45-45ff-caf8-81001b8f99bb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(3360, 1000)\n",
            "0.09110336122361047\n"
          ]
        }
      ],
      "source": [
        "score = 0\n",
        "samples = []\n",
        "with torch.no_grad():\n",
        "  for i in range(y_true.shape[0]):\n",
        "    sample = flow.sample(1000,context=x_test[i,:].reshape((-1,4))).cpu().numpy()\n",
        "    sample = sample.squeeze()\n",
        "    score += ps.crps_ensemble(y_true[i,0] , sample)\n",
        "    samples.append(list(sample))\n",
        "samples = np.array(samples)\n",
        "print(samples.shape)\n",
        "print(score/y_true.shape[0])"
      ]
    }
  ]
}