Merge pull request #172 from jdb78/fix/allow-new-series-ids-in-test-data

Re-encode group ids by dataset to identify series

Author: jdb78

pytorch_forecasting/data/timeseries.py

@@ -353,6 +353,24 @@ def _set_target_normalizer(self, data: pd.DataFrame):
             self.target_normalizer, (TorchNormalizer, NaNLabelEncoder)
         ), f"target_normalizer has to be either None or of class TorchNormalizer but found {self.target_normalizer}"
 
+    @property
+    def _group_ids_mapping(self) -> Dict[str, str]:
+        """
+        Mapping of group id names to group ids used to identify series in dataset -
+        group ids can also be used for target normalizer.
+        The former can change from training to validation and test dataset while the later must not.
+        """
+        return {name: f"__group_id__{name}" for name in self.group_ids}
+
+    @property
+    def _group_ids(self) -> List[str]:
+        """
+        Group ids used to identify series in dataset.
+
+        See :py:meth:`~TimeSeriesDataSet._group_ids_mapping` for details.
+        """
+        return list(self._group_ids_mapping.values())
+
     def _validate_data(self, data: pd.DataFrame):
         """
         Validate that data will not cause hick-ups later on.
@@ -403,9 +421,19 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
         Returns:
             pd.DataFrame: pre-processed dataframe
         """
+        # encode group ids - this encoding
+        for name, group_name in self._group_ids_mapping.items():
+            self.categorical_encoders[group_name] = NaNLabelEncoder().fit(data[name].to_numpy().reshape(-1))
+            data[group_name] = self.transform_values(name, data[name], inverse=False, group_id=True)
 
         # encode categoricals
-        for name in set(self.categoricals + self.group_ids):
+        if isinstance(
+            self.target_normalizer, GroupNormalizer
+        ):  # if we use a group normalizer, group_ids must be encoded as well
+            group_ids_to_encode = self.group_ids
+        else:
+            group_ids_to_encode = []
+        for name in set(group_ids_to_encode + self.categoricals):
             allow_nans = name in self.dropout_categoricals
             if name in self.variable_groups:  # fit groups
                 columns = self.variable_groups[name]
@@ -430,7 +458,7 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
                         self.categorical_encoders[name] = self.categorical_encoders[name].fit(data[name])
 
         # encode them
-        for name in set(self.flat_categoricals + self.group_ids):
+        for name in set(group_ids_to_encode + self.flat_categoricals):
             data[name] = self.transform_values(name, data[name], inverse=False)
 
         # save special variables
@@ -472,6 +500,10 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
                 data[self.target], scales = self.target_normalizer.transform(data[self.target], data, return_norm=True)
             elif isinstance(self.target_normalizer, NaNLabelEncoder):
                 data[self.target] = self.target_normalizer.transform(data[self.target])
+                data["__target__"] = data[
+                    self.target
+                ]  # overwrite target because it requires encoding (continuous targets should not be normalized)
+                scales = "no target scales available for categorical target"
             else:
                 data[self.target], scales = self.target_normalizer.transform(data[self.target], return_norm=True)
 
@@ -488,6 +520,8 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
 
         if self.target in self.reals:
             self.scalers[self.target] = self.target_normalizer
+        else:
+            self.categorical_encoders[self.target] = self.target_normalizer
 
         # rescale continuous variables apart from target
         for name in self.reals:
@@ -515,7 +549,12 @@ def _preprocess_data(self, data: pd.DataFrame) -> pd.DataFrame:
         return data
 
     def transform_values(
-        self, name: str, values: Union[pd.Series, torch.Tensor, np.ndarray], data: pd.DataFrame = None, inverse=False
+        self,
+        name: str,
+        values: Union[pd.Series, torch.Tensor, np.ndarray],
+        data: pd.DataFrame = None,
+        inverse=False,
+        group_id: bool = False,
     ) -> np.ndarray:
         """
         Scale and encode values.
@@ -526,12 +565,16 @@ def transform_values(
             data (pd.DataFrame, optional): extra data used for scaling (e.g. dataframe with groups columns).
                 Defaults to None.
             inverse (bool, optional): if to conduct inverse transformation. Defaults to False.
+            group_id (bool, optional): If the passed name refers to a group id (different encoders are used for these).
+                Defaults to False.
 
         Returns:
             np.ndarray: (de/en)coded/(de)scaled values
         """
+        if group_id:
+            name = self._group_ids_mapping[name]
         # remaining categories
-        if name in set(self.flat_categoricals + self.group_ids):
+        if name in set(self.flat_categoricals + self.group_ids + self._group_ids):
             name = self.variable_to_group_mapping.get(name, name)  # map name to encoder
             encoder = self.categorical_encoders[name]
             if encoder is None:
@@ -575,7 +618,7 @@ def _data_to_tensors(self, data: pd.DataFrame) -> Dict[str, torch.Tensor]:
                 time index
         """
 
-        index = torch.tensor(data[self.group_ids].to_numpy(np.long), dtype=torch.long)
+        index = torch.tensor(data[self._group_ids].to_numpy(np.long), dtype=torch.long)
         time = torch.tensor(data["__time_idx__"].to_numpy(np.long), dtype=torch.long)
 
         categorical = torch.tensor(data[self.flat_categoricals].to_numpy(np.long), dtype=torch.long)
@@ -735,7 +778,7 @@ def _construct_index(self, data: pd.DataFrame, predict_mode: bool) -> pd.DataFra
         Returns:
             pd.DataFrame: index dataframe
         """
-        g = data.groupby(self.group_ids, observed=True)
+        g = data.groupby(self._group_ids, observed=True)
 
         df_index_first = g["__time_idx__"].transform("nth", 0).to_frame("time_first")
         df_index_last = g["__time_idx__"].transform("nth", -1).to_frame("time_last")
@@ -797,10 +840,10 @@ def _construct_index(self, data: pd.DataFrame, predict_mode: bool) -> pd.DataFra
 
         # check that all groups/series have at least one entry in the index
         if not group_ids.isin(df_index.group_id).all():
-            missing_groups = data.loc[~group_ids.isin(df_index.group_id), self.group_ids].drop_duplicates()
+            missing_groups = data.loc[~group_ids.isin(df_index.group_id), self._group_ids].drop_duplicates()
             # decode values
-            for name in missing_groups.columns:
-                missing_groups[name] = self.transform_values(name, missing_groups[name], inverse=True)
+            for name, id in self._group_ids_mapping.items():
+                missing_groups[id] = self.transform_values(name, missing_groups[id], inverse=True, group_id=True)
             warnings.warn(
                 "Min encoder length and/or min_prediction_idx and/or min prediction length is too large for "
                 f"{len(missing_groups)} series/groups which therefore are not present in the dataset index. "
@@ -1210,7 +1253,7 @@ def x_to_index(self, x: Dict[str, torch.Tensor]) -> pd.DataFrame:
         for id in self.group_ids:
             index_data[id] = x["groups"][:, self.group_ids.index(id)].cpu()
             # decode if possible
-            index_data[id] = self.transform_values(id, index_data[id], inverse=True)
+            index_data[id] = self.transform_values(id, index_data[id], inverse=True, group_id=True)
         index = pd.DataFrame(index_data)
         return index
 

tests/conftest.py

@@ -11,6 +11,18 @@
 from pytorch_forecasting.data.examples import get_stallion_data  # isort:skip
 
 
+# for vscode debugging: https://stackoverflow.com/a/62563106/14121677
+if os.getenv("_PYTEST_RAISE", "0") != "0":
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_exception_interact(call):
+        raise call.excinfo.value
+
+    @pytest.hookimpl(tryfirst=True)
+    def pytest_internalerror(excinfo):
+        raise excinfo.value
+
+
 @pytest.fixture
 def test_data():
     data = get_stallion_data()

tests/test_data.py

@@ -373,10 +373,48 @@ def test_categorical_target(test_data):
         min_encoder_length=1,
     )
 
-    x, y = next(iter(dataset.to_dataloader()))
+    _, y = next(iter(dataset.to_dataloader()))
     assert y.dtype is torch.long, "target must be of type long"
 
 
 def test_pickle(test_dataset):
     pickle.dumps(test_dataset)
     pickle.dumps(test_dataset.to_dataloader())
+
+
+@pytest.mark.parametrize(
+    "kwargs",
+    [
+        {},
+        dict(
+            target_normalizer=GroupNormalizer(
+                groups=["agency", "sku"], log_scale=True, scale_by_group=True, log_zero_value=1.0
+            ),
+        ),
+    ],
+)
+def test_new_group_ids(test_data, kwargs):
+    """Test for new group ids in dataset"""
+    train_agency = test_data["agency"].iloc[0]
+    train_dataset = TimeSeriesDataSet(
+        test_data[lambda x: x.agency == train_agency],
+        time_idx="time_idx",
+        target="volume",
+        group_ids=["agency", "sku"],
+        max_encoder_length=5,
+        max_prediction_length=2,
+        min_prediction_length=1,
+        min_encoder_length=1,
+        categorical_encoders=dict(agency=NaNLabelEncoder(add_nan=True), sku=NaNLabelEncoder(add_nan=True)),
+        **kwargs,
+    )
+
+    # test sampling from training dataset
+    next(iter(train_dataset.to_dataloader()))
+
+    # create test dataset with group ids that have not been observed before
+    test_dataset = TimeSeriesDataSet.from_dataset(train_dataset, test_data)
+
+    # check that we can iterate through dataset without error
+    for _ in iter(test_dataset.to_dataloader()):
+        pass