From d1a31599ea7d43fd9606ca36869706e8960092d2 Mon Sep 17 00:00:00 2001 From: Ralf Date: Mon, 17 Apr 2023 14:19:16 +0200 Subject: [PATCH] [FSTORE-752] Online type consistency issue (#1310) --- .../FeatureGroupInputValidation.java | 11 +- .../TestFeatureGroupInputValidation.java | 396 +++++++++++++++++- 2 files changed, 402 insertions(+), 5 deletions(-) diff --git a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeatureGroupInputValidation.java b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeatureGroupInputValidation.java index b08a58cf5d..1cb312b0dd 100644 --- a/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeatureGroupInputValidation.java +++ b/hopsworks-common/src/main/java/io/hops/hopsworks/common/featurestore/featuregroup/FeatureGroupInputValidation.java @@ -45,16 +45,18 @@ public class FeatureGroupInputValidation { @EJB - private FeaturestoreInputValidation featureStoreInputValidation; + protected FeaturestoreInputValidation featureStoreInputValidation; @EJB - private OnlineFeaturegroupController onlineFeaturegroupController; + protected OnlineFeaturegroupController onlineFeaturegroupController; public FeatureGroupInputValidation() { } // for testing - public FeatureGroupInputValidation(FeaturestoreInputValidation featureStoreInputValidation) { + public FeatureGroupInputValidation(FeaturestoreInputValidation featureStoreInputValidation, + OnlineFeaturegroupController onlineFeaturegroupController) { this.featureStoreInputValidation = featureStoreInputValidation; + this.onlineFeaturegroupController = onlineFeaturegroupController; } /** @@ -164,7 +166,8 @@ public void verifyOnlineOfflineTypeMatch(FeaturegroupDTO featuregroupDTO) throws if ((offlineType.startsWith("array") || offlineType.startsWith("struct") || - offlineType.startsWith("binary")) && + offlineType.startsWith("binary") || + offlineType.startsWith("map")) && (onlineType.startsWith("varbinary") || onlineType.equals("blob"))) { continue; } diff --git a/hopsworks-common/src/test/io/hops/hopsworks/common/featurestore/utils/TestFeatureGroupInputValidation.java b/hopsworks-common/src/test/io/hops/hopsworks/common/featurestore/utils/TestFeatureGroupInputValidation.java index 18e96f3714..4b80752d54 100644 --- a/hopsworks-common/src/test/io/hops/hopsworks/common/featurestore/utils/TestFeatureGroupInputValidation.java +++ b/hopsworks-common/src/test/io/hops/hopsworks/common/featurestore/utils/TestFeatureGroupInputValidation.java @@ -19,6 +19,9 @@ import io.hops.hopsworks.common.featurestore.feature.FeatureGroupFeatureDTO; import io.hops.hopsworks.common.featurestore.featuregroup.FeatureGroupInputValidation; import io.hops.hopsworks.common.featurestore.featuregroup.cached.CachedFeaturegroupDTO; +import io.hops.hopsworks.common.featurestore.featuregroup.ondemand.OnDemandFeaturegroupDTO; +import io.hops.hopsworks.common.featurestore.featuregroup.online.OnlineFeaturegroupController; +import io.hops.hopsworks.common.featurestore.featuregroup.stream.StreamFeatureGroupDTO; import io.hops.hopsworks.exceptions.FeaturestoreException; import io.hops.hopsworks.persistence.entity.featurestore.featuregroup.cached.TimeTravelFormat; import org.apache.commons.lang.StringUtils; @@ -34,7 +37,7 @@ public class TestFeatureGroupInputValidation { private FeatureGroupInputValidation featureGroupInputValidation = - new FeatureGroupInputValidation(new FeaturestoreInputValidation()); + new FeatureGroupInputValidation(new FeaturestoreInputValidation(), new OnlineFeaturegroupController()); List features; @@ -174,4 +177,395 @@ public void testVerifyAndGetNewFeaturesIfMissingType() throws Exception { thrown.expect(FeaturestoreException.class); featureGroupInputValidation.verifyAndGetNewFeatures(features, newSchema); } + + @Test + public void testVerifyOnlineOfflineTypeMatchCheckCachedFeatureGroupDTO() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "Integer"); + featureGroupFeatureDTO.setOnlineType("bad_type"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + thrown.expect(FeaturestoreException.class); + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchCheckStreamFeatureGroupDTO() throws Exception { + // Arrange + StreamFeatureGroupDTO featuregroupDTO = new StreamFeatureGroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "Integer"); + featureGroupFeatureDTO.setOnlineType("bad_type"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + thrown.expect(FeaturestoreException.class); + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchCheckOnDemandFeaturegroupDTO() throws Exception { + // Arrange + OnDemandFeaturegroupDTO featuregroupDTO = new OnDemandFeaturegroupDTO(); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "Integer"); + featureGroupFeatureDTO.setOnlineType("bad_type"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchCheckOffline() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(false); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "Integer"); + featureGroupFeatureDTO.setOnlineType("bad_type"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchBoolean() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "BOOLEAN"); + featureGroupFeatureDTO.setOnlineType("TINYINT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchTinyint() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "TINYINT"); + featureGroupFeatureDTO.setOnlineType("TINYINT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchSmallint() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "SMALLINT"); + featureGroupFeatureDTO.setOnlineType("SMALLINT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchInt() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "INT"); + featureGroupFeatureDTO.setOnlineType("INT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchIntTinyint() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "INT"); + featureGroupFeatureDTO.setOnlineType("TINYINT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchIntSmallint() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "INT"); + featureGroupFeatureDTO.setOnlineType("SMALLINT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchBigint() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "BIGINT"); + featureGroupFeatureDTO.setOnlineType("BIGINT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchFloat() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "FLOAT"); + featureGroupFeatureDTO.setOnlineType("FLOAT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchDouble() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "DOUBLE"); + featureGroupFeatureDTO.setOnlineType("DOUBLE"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchDecimal() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "DECIMAL"); + featureGroupFeatureDTO.setOnlineType("DECIMAL"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchTimestamp() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "TIMESTAMP"); + featureGroupFeatureDTO.setOnlineType("TIMESTAMP"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchDate() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "DATE"); + featureGroupFeatureDTO.setOnlineType("DATE"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchStringVarchar() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "STRING"); + featureGroupFeatureDTO.setOnlineType("VARCHAR"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchStringText() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "STRING"); + featureGroupFeatureDTO.setOnlineType("TEXT"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchArrayVarbinary() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "ARRAY"); + featureGroupFeatureDTO.setOnlineType("VARBINARY"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchArrayBlob() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "ARRAY"); + featureGroupFeatureDTO.setOnlineType("BLOB"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchStructVarbinary() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "STRUCT"); + featureGroupFeatureDTO.setOnlineType("VARBINARY"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchStructBlob() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "STRUCT"); + featureGroupFeatureDTO.setOnlineType("BLOB"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchBinaryVarbinary() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "BINARY"); + featureGroupFeatureDTO.setOnlineType("VARBINARY"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchBinaryBlob() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "BINARY"); + featureGroupFeatureDTO.setOnlineType("BLOB"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchMapVarbinary() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "MAP"); + featureGroupFeatureDTO.setOnlineType("VARBINARY"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } + + @Test + public void testVerifyOnlineOfflineTypeMatchMapBlob() throws Exception { + // Arrange + CachedFeaturegroupDTO featuregroupDTO = new CachedFeaturegroupDTO(); + featuregroupDTO.setOnlineEnabled(true); + List newSchema = new ArrayList<>(); + FeatureGroupFeatureDTO featureGroupFeatureDTO = new FeatureGroupFeatureDTO("part_param", "MAP"); + featureGroupFeatureDTO.setOnlineType("BLOB"); + newSchema.add(featureGroupFeatureDTO); + featuregroupDTO.setFeatures(newSchema); + + // Act + featureGroupInputValidation.verifyOnlineOfflineTypeMatch(featuregroupDTO); + } }