[HWORKS-934] Use more precise statistics (#1681)

logicalclocks · Feb 13, 2024 · 8d1964d · 8d1964d
1 parent a2e2529
commit 8d1964d
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 20 deletions.
diff --git a/...va/io/hops/hopsworks/api/featurestore/statistics/FeatureDescriptiveStatisticsBuilder.java b/...va/io/hops/hopsworks/api/featurestore/statistics/FeatureDescriptiveStatisticsBuilder.java
@@ -134,21 +134,21 @@ public FeatureDescriptiveStatistics buildFromDeequJson(JSONObject statsJson) {
       fds.setCount(statsJson.getLong("count"));
     }
     if (statsJson.has("completeness")) {
-      fds.setCompleteness(statsJson.getDouble("completeness"));
+      fds.setCompleteness(Float.valueOf(statsJson.getString("completeness")));
     }
     if (statsJson.has("approximateNumDistinctValues")) {
       fds.setApproxNumDistinctValues(statsJson.getLong("approximateNumDistinctValues"));
     }
 
     // commmon for all data types if exact_uniqueness is enabled
     if (statsJson.has("uniqueness")) {
-      fds.setUniqueness(statsJson.getDouble("uniqueness"));
+      fds.setUniqueness(Float.valueOf(statsJson.getString("uniqueness")));
     }
     if (statsJson.has("entropy")) {
-      fds.setEntropy(statsJson.getDouble("entropy"));
+      fds.setEntropy(Float.valueOf(statsJson.getString("entropy")));
     }
     if (statsJson.has("distinctness")) {
-      fds.setDistinctness(statsJson.getDouble("distinctness"));
+      fds.setDistinctness(Float.valueOf(statsJson.getString("distinctness")));
     }
     if (statsJson.has("exactNumDistinctValues")) {
       fds.setExactNumDistinctValues(statsJson.getLong("exactNumDistinctValues"));

diff --git a/...ava/io/hops/hopsworks/common/featurestore/statistics/FeatureDescriptiveStatisticsDTO.java b/...ava/io/hops/hopsworks/common/featurestore/statistics/FeatureDescriptiveStatisticsDTO.java
@@ -33,7 +33,7 @@ public class FeatureDescriptiveStatisticsDTO extends RestDTO<FeatureDescriptiveS
 
   // for any feature type
   private Long count;
-  private Double completeness;
+  private Float completeness;
   private Long numNonNullValues;
   private Long numNullValues;
   private Long approxNumDistinctValues;
@@ -47,9 +47,9 @@ public class FeatureDescriptiveStatisticsDTO extends RestDTO<FeatureDescriptiveS
   private List<Double> percentiles;
 
   // with exact uniqueness
-  private Double distinctness;
-  private Double entropy;
-  private Double uniqueness;
+  private Float distinctness;
+  private Float entropy;
+  private Float uniqueness;
   private Long exactNumDistinctValues;
 
   // histogram, correlations, kll <- from hdfs file

diff --git a/...ps/hopsworks/persistence/entity/featurestore/statistics/FeatureDescriptiveStatistics.java b/...ps/hopsworks/persistence/entity/featurestore/statistics/FeatureDescriptiveStatistics.java
@@ -66,7 +66,7 @@ public class FeatureDescriptiveStatistics implements Serializable {
 
   @Basic
   @Column(name = "completeness")
-  private Double completeness;
+  private Float completeness;
 
   @Basic
   @Column(name = "num_non_null_values")
@@ -111,15 +111,15 @@ public class FeatureDescriptiveStatistics implements Serializable {
 
   @Basic
   @Column(name = "distinctness")
-  private Double distinctness;
+  private Float distinctness;
 
   @Basic
   @Column(name = "entropy")
-  private Double entropy;
+  private Float entropy;
 
   @Basic
   @Column(name = "uniqueness")
-  private Double uniqueness;
+  private Float uniqueness;
 
   @Basic
   @Column(name = "exact_num_distinct_values")
@@ -213,11 +213,11 @@ public void setPercentiles(List<Double> percentiles) {
     this.percentiles = percentiles;
   }
 
-  public Double getCompleteness() {
+  public Float getCompleteness() {
     return completeness;
   }
 
-  public void setCompleteness(Double completeness) {
+  public void setCompleteness(Float completeness) {
     this.completeness = completeness;
   }
 
@@ -237,27 +237,27 @@ public void setNumNullValues(Long numNullValues) {
     this.numNullValues = numNullValues;
   }
 
-  public Double getDistinctness() {
+  public Float getDistinctness() {
     return distinctness;
   }
 
-  public void setDistinctness(Double distinctness) {
+  public void setDistinctness(Float distinctness) {
     this.distinctness = distinctness;
   }
 
-  public Double getEntropy() {
+  public Float getEntropy() {
     return entropy;
   }
 
-  public void setEntropy(Double entropy) {
+  public void setEntropy(Float entropy) {
     this.entropy = entropy;
   }
 
-  public Double getUniqueness() {
+  public Float getUniqueness() {
     return uniqueness;
   }
 
-  public void setUniqueness(Double uniqueness) {
+  public void setUniqueness(Float uniqueness) {
     this.uniqueness = uniqueness;
   }