From edc1d2759f61c20bf8fe19bbf5569e80a9b1cf51 Mon Sep 17 00:00:00 2001 From: anton Date: Tue, 16 Jan 2024 10:47:09 +0100 Subject: [PATCH] FAIRSPC-29: fixed numeric filter with forced casting to numeric (Postgresql) type to avoid inaccurate casting of PreparedStatement causing a performance issue --- .../src/metadata/views/MetadataView.js | 3 +- .../src/metadata/views/MetadataViewAPI.js | 1 + .../src/metadata/views/metadataViewUtils.js | 1 + .../saturn/services/views/ViewFilter.java | 1 + .../services/views/ViewStoreReader.java | 33 ++++++++++++++----- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/projects/mercury/src/metadata/views/MetadataView.js b/projects/mercury/src/metadata/views/MetadataView.js index 32b4b835b8..d8bad2f1d3 100644 --- a/projects/mercury/src/metadata/views/MetadataView.js +++ b/projects/mercury/src/metadata/views/MetadataView.js @@ -13,7 +13,7 @@ import MetadataViewContext from "./MetadataViewContext"; import BreadcrumbsContext from "../../common/contexts/BreadcrumbsContext"; import {getLocationContextFromString, getMetadataViewNameFromString} from "../../search/searchUtils"; import type {MetadataViewEntity} from "./metadataViewUtils"; -import {getMetadataViewsPath, ofBooleanValueType, ofRangeValueType, RESOURCES_VIEW} from "./metadataViewUtils"; +import {getMetadataViewsPath, ofBooleanValueType, ofRangeValueType, ofNumericValueType, RESOURCES_VIEW} from "./metadataViewUtils"; import MetadataViewActiveFacetFilters from "./MetadataViewActiveFacetFilters"; import MetadataViewInformationDrawer from "./MetadataViewInformationDrawer"; import {useSingleSelection} from "../../file/UseSelection"; @@ -87,6 +87,7 @@ export const MetadataView = (props: MetadataViewProperties) => { const setFilterValues = (type: ValueType, filter: MetadataViewFilter, values: any[]) => { if (ofRangeValueType(type)) { [filter.min, filter.max] = values; + filter.numericValue = ofNumericValueType(type); } else if (ofBooleanValueType(type)) { filter.booleanValue = values.length > 0 ? values[0] : null; } else { diff --git a/projects/mercury/src/metadata/views/MetadataViewAPI.js b/projects/mercury/src/metadata/views/MetadataViewAPI.js index 2c87ab39af..8490fdbec0 100644 --- a/projects/mercury/src/metadata/views/MetadataViewAPI.js +++ b/projects/mercury/src/metadata/views/MetadataViewAPI.js @@ -13,6 +13,7 @@ export type MetadataViewFilter = { max: any; prefix: string; booleanValue?: boolean; + numericValue?: boolean; } export type MetadataViewFacetValue = { diff --git a/projects/mercury/src/metadata/views/metadataViewUtils.js b/projects/mercury/src/metadata/views/metadataViewUtils.js index d93936103e..01ab68af9a 100644 --- a/projects/mercury/src/metadata/views/metadataViewUtils.js +++ b/projects/mercury/src/metadata/views/metadataViewUtils.js @@ -21,3 +21,4 @@ export const getMetadataViewsPath = (viewName: string) => { export const ofRangeValueType: boolean = (type: ValueType) => type === 'Number' || type === 'Date'; export const ofBooleanValueType: boolean = (type: ValueType) => type === 'Boolean'; +export const ofNumericValueType: boolean = (type: ValueType) => type === 'Number'; diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewFilter.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewFilter.java index 294ecb9c48..4c0f04b338 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewFilter.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewFilter.java @@ -18,6 +18,7 @@ public class ViewFilter { Object min; Object max; Boolean booleanValue; + Boolean numericValue; String prefix; /** * Used internally for filtering on resource location. diff --git a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewStoreReader.java b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewStoreReader.java index f9a5f2dc3b..499a25e773 100644 --- a/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewStoreReader.java +++ b/projects/saturn/src/main/java/io/fairspace/saturn/services/views/ViewStoreReader.java @@ -1,19 +1,21 @@ package io.fairspace.saturn.services.views; -import io.fairspace.saturn.config.*; -import io.fairspace.saturn.config.ViewsConfig.*; +import io.fairspace.saturn.config.Config; +import io.fairspace.saturn.config.ViewsConfig.ColumnType; +import io.fairspace.saturn.config.ViewsConfig.View; import io.fairspace.saturn.services.search.FileSearchRequest; import io.fairspace.saturn.services.search.SearchResultDTO; -import io.fairspace.saturn.vocabulary.*; +import io.fairspace.saturn.vocabulary.FS; import lombok.SneakyThrows; -import lombok.extern.slf4j.*; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import java.sql.*; -import java.time.*; -import java.util.*; +import java.time.Instant; import java.util.Date; -import java.util.stream.*; +import java.util.*; +import java.util.stream.Collectors; +import java.util.stream.Stream; import static io.fairspace.saturn.config.ViewsConfig.ColumnType.Date; import static io.fairspace.saturn.services.views.Table.idColumn; @@ -193,13 +195,19 @@ String sqlConstraint(String fieldName, ViewFilter filter, List values) { .collect(Collectors.joining(", "))); } var constraints = new ArrayList(); + // NOTE: for NUMERIC filter, turned out, PreparedStatement may cast them as real or double, + // what causes a dramatic slow down for the query execution (up to x100). + // As a workaround, we force PrepareStatement to send the request with numeric type (the same as it is defined + // in the database schema) if (filter.getMin() != null) { values.add(filter.getMin()); - constraints.add(fieldName + " >= ?"); + var constraint = filter.numericValue ? " >= ?::numeric" : " >= ?"; + constraints.add(fieldName + constraint); } if (filter.getMax() != null) { values.add(filter.getMax()); - constraints.add(fieldName + " <= ?"); + var constraint = filter.numericValue ? " <= ?::numeric" : " <= ?"; + constraints.add(fieldName + constraint); } if (filter.getPrefix() != null && !filter.getPrefix().isBlank()) { // Use view label instead of id for prefix filters @@ -506,9 +514,13 @@ public List>> retrieveRows( throw new IllegalArgumentException("View not supported: " + view); } // Fetch rows with columns from the view table + var viewStart = Instant.now().toEpochMilli(); var rows = this.retrieveViewTableRows(view, filters, offset, limit); + var viewQueryTime = Instant.now().toEpochMilli() - viewStart; + System.out.println("+++++ VIEW TIME RETRIEVAL: " + viewQueryTime + " ++++++"); // Add items from join tables if (includeJoinedViews) { + var joinStart = Instant.now().toEpochMilli(); for (var joinView : viewConfig.join) { for (var row : rows) { var id = (String) row.get(view).stream().findFirst().orElseThrow().getValue(); @@ -522,6 +534,9 @@ public List>> retrieveRows( } } } + var joinQueryTime = Instant.now().toEpochMilli() - joinStart; + var numOfJoinCalls = viewConfig.join.size() * rows.size(); + System.out.println("----- JOIN TIME RETRIEVAL: " + joinQueryTime + " ----- for " + numOfJoinCalls + " retrievals"); } return rows; } catch (SQLTimeoutException e) {