More tests for implicit quantiles

Author: Jan Beitner

pytorch_forecasting/metrics/distributions.py

@@ -443,8 +443,8 @@ def __init__(
         self,
         quantiles: List[float] = [0.02, 0.1, 0.25, 0.5, 0.75, 0.9, 0.98],
         input_size: Optional[int] = 16,
-        hidden_size: Optional[int] = 64,
-        n_loss_samples: Optional[int] = 16,
+        hidden_size: Optional[int] = 32,
+        n_loss_samples: Optional[int] = 64,
     ) -> None:
         """
         Args:
@@ -460,6 +460,14 @@ def __init__(
         self.distribution_arguments = list(range(int(input_size)))
         self.n_loss_samples = n_loss_samples
 
+    def sample(self, y_pred, n_samples: int) -> torch.Tensor:
+        eps = 1e-3
+        # for a couple of random quantiles (excl. 0 and 1 as they would lead to infinities)
+        quantiles = torch.rand(size=(n_samples,), device=y_pred.device).clamp(eps, 1 - eps)
+        # make prediction
+        samples = self.to_quantiles(y_pred, quantiles=quantiles)
+        return samples
+
     def loss(self, y_pred: torch.Tensor, y_actual: torch.Tensor) -> torch.Tensor:
         """
         Calculate negative likelihood
@@ -478,7 +486,7 @@ def loss(self, y_pred: torch.Tensor, y_actual: torch.Tensor) -> torch.Tensor:
         pred_quantiles = self.to_quantiles(y_pred, quantiles=quantiles)
         # and calculate quantile loss
         errors = y_actual[..., None] - pred_quantiles
-        loss = torch.fmax(quantiles[None] * errors, (quantiles[None] - 1) * errors).mean(dim=-1)
+        loss = 2 * torch.fmax(quantiles[None] * errors, (quantiles[None] - 1) * errors).mean(dim=-1)
         return loss
 
     def rescale_parameters(
@@ -492,9 +500,7 @@ def to_prediction(self, y_pred: torch.Tensor, n_samples: int = 100) -> torch.Ten
             return self.to_quantiles(y_pred, quantiles=[0.5]).squeeze(-1)
         else:
             # for a couple of random quantiles (excl. 0 and 1 as they would lead to infinities) make prediction
-            eps = 1e-3
-            quantiles = torch.rand(size=(n_samples,), device=y_pred.device).clamp(eps, 1 - eps)
-            return self.to_quantiles(y_pred, quantiles=quantiles).mean(-1)
+            return self.sample(y_pred, n_samples=n_samples).mean(-1)
 
     def to_quantiles(self, y_pred: torch.Tensor, quantiles: List[float] = None) -> torch.Tensor:
         """
@@ -518,14 +524,16 @@ def to_quantiles(self, y_pred: torch.Tensor, quantiles: List[float] = None) -> t
         scale = y_pred[..., -1][..., None]
 
         # predict quantiles
-        predictions = self.quantile_network(x, quantiles)
+        if y_pred.requires_grad:
+            predictions = self.quantile_network(x, quantiles)
+        else:
+            with torch.no_grad():
+                predictions = self.quantile_network(x, quantiles)
         # rescale output
         predictions = loc + predictions * scale
         # transform output if required
         if self._transformation is not None:
             transform = TorchNormalizer.get_transform(self._transformation)["reverse"]
             predictions = transform(predictions)
 
-        if not y_pred.requires_grad:
-            predictions = predictions.detach()
         return predictions

pytorch_forecasting/metrics/quantile.py

@@ -32,7 +32,7 @@ def loss(self, y_pred: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
         for i, q in enumerate(self.quantiles):
             errors = target - y_pred[..., i]
             losses.append(torch.max((q - 1) * errors, q * errors).unsqueeze(-1))
-        losses = torch.cat(losses, dim=2)
+        losses = 2 * torch.cat(losses, dim=2)
 
         return losses
 

pytorch_forecasting/models/base_model.py

@@ -409,7 +409,6 @@ def training_step(self, batch, batch_idx):
         """
         x, y = batch
         log, out = self.step(x, y, batch_idx)
-        log.update(self.create_log(x, y, out, batch_idx))
         return log
 
     def training_epoch_end(self, outputs):

pytorch_forecasting/models/temporal_fusion_transformer/__init__.py

@@ -535,7 +535,7 @@ def epoch_end(self, outputs):
         """
         run at epoch end for training or validation
         """
-        if self.log_interval > 0:
+        if self.log_interval > 0 and not self.training:
             self.log_interpretation(outputs)
 
     def interpret_output(

tests/test_metrics.py

@@ -234,4 +234,4 @@ def test_ImplicitQuantileNetworkDistributionLoss():
     assert quantiles.size(1) == n_timesteps
 
     point_prediction = loss.to_prediction(pred, n_samples=None)
-    assert point_prediction.ndim == loss.to_prediction(pred, n_samples=100).ndim - 1
+    assert point_prediction.ndim == loss.to_prediction(pred, n_samples=100).ndim

tests/test_models/test_deepar.py

@@ -11,6 +11,7 @@
 from pytorch_forecasting.data.encoders import GroupNormalizer
 from pytorch_forecasting.metrics import (
     BetaDistributionLoss,
+    ImplicitQuantileNetworkDistributionLoss,
     LogNormalDistributionLoss,
     MultivariateNormalDistributionLoss,
     NegativeBinomialDistributionLoss,
@@ -121,6 +122,9 @@ def _integration(
                 lags={"volume": [2], "discount": [2]},
             )
         ),
+        dict(
+            loss=ImplicitQuantileNetworkDistributionLoss(hidden_size=8),
+        ),
         dict(
             loss=MultivariateNormalDistributionLoss(),
         ),