From 867236f14492956e1605767fac67dcc8e284637f Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Wed, 24 Jan 2024 15:53:27 +0100 Subject: [PATCH] Implement Select filter value auto-completion for SQL mode (#3277) --- .../conquery/commands/ManagerNode.java | 10 +- .../conquery/io/storage/NamespaceStorage.java | 5 +- .../io/storage/NamespacedStorage.java | 7 +- .../conquery/io/storage/WorkerStorage.java | 3 +- .../conquery/mode/ManagerProvider.java | 34 +++- .../bakdata/conquery/mode/StorageHandler.java | 12 ++ .../mode/cluster/ClusterManagerProvider.java | 2 +- .../mode/cluster/ClusterStorageHandler.java | 22 +++ .../mode/local/LocalManagerProvider.java | 12 +- .../mode/local/LocalNamespaceHandler.java | 5 +- .../mode/local/SqlStorageHandler.java | 46 ++++++ .../conquery/models/config/StoreFactory.java | 3 +- .../models/config/XodusStoreFactory.java | 6 +- .../conquery/models/datasets/Column.java | 11 +- .../conquery/models/error/ConqueryError.java | 3 +- .../conquery/models/error/ErrorMessages.java | 4 +- .../models/worker/DatasetRegistry.java | 6 +- .../models/worker/DistributedNamespace.java | 16 +- .../models/worker/LocalNamespace.java | 143 +++-------------- .../conquery/models/worker/Namespace.java | 148 +++++++++++++----- .../sql/conquery/SqlExecutionManager.java | 8 +- .../sql/execution/SqlExecutionService.java | 46 +++++- .../api/StoredQueriesProcessorTest.java | 2 +- .../tests/FilterAutocompleteTest.java | 8 +- .../io/AbstractSerializationTest.java | 2 +- .../io/jackson/serializer/IdRefrenceTest.java | 2 +- .../util/NonPersistentStoreFactory.java | 3 +- 27 files changed, 350 insertions(+), 219 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/StorageHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/local/SqlStorageHandler.java diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index 8baff15832..c7b649fe1f 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -20,11 +20,13 @@ import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.mode.Manager; +import com.bakdata.conquery.mode.StorageHandler; import com.bakdata.conquery.models.auth.AuthorizationController; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.models.worker.Worker; import com.bakdata.conquery.resources.ResourcesProvider; import com.bakdata.conquery.resources.admin.AdminServlet; @@ -244,19 +246,21 @@ public void loadNamespaces() { ExecutorService loaders = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); + DatasetRegistry registry = getDatasetRegistry(); // Namespaces load their storage themselves, so they can inject Namespace relevant objects into stored objects - final Collection namespaceStorages = getConfig().getStorage().discoverNamespaceStorages(); + StorageHandler storageHandler = registry.getStorageHandler(); + final Collection namespaceStorages = getConfig().getStorage().discoverNamespaceStorages(storageHandler); for (NamespaceStorage namespaceStorage : namespaceStorages) { loaders.submit(() -> { - getDatasetRegistry().createNamespace(namespaceStorage); + registry.createNamespace(namespaceStorage); }); } loaders.shutdown(); while (!loaders.awaitTermination(1, TimeUnit.MINUTES)) { - final int coundLoaded = getDatasetRegistry().getDatasets().size(); + final int coundLoaded = registry.getDatasets().size(); log.debug("Waiting for Worker namespaces to load. {} are already finished. {} pending.", coundLoaded, namespaceStorages.size() - coundLoaded); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index 96723346d7..5fa5e50969 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.io.storage.xodus.stores.KeyIncludingStore; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; +import com.bakdata.conquery.mode.StorageHandler; import com.bakdata.conquery.models.config.StoreFactory; import com.bakdata.conquery.models.datasets.PreviewConfig; import com.bakdata.conquery.models.datasets.concepts.StructureNode; @@ -40,8 +41,8 @@ public class NamespaceStorage extends NamespacedStorage { protected SingletonStore primaryDictionary; - public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator) { - super(storageFactory, pathName, validator); + public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator, StorageHandler storageHandler) { + super(storageFactory, pathName, validator, storageHandler); } public EncodedDictionary getPrimaryDictionary() { diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index 5970577065..4fe3f60e24 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.io.storage.xodus.stores.KeyIncludingStore; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; +import com.bakdata.conquery.mode.StorageHandler; import com.bakdata.conquery.models.config.StoreFactory; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; @@ -52,6 +53,9 @@ public abstract class NamespacedStorage extends ConqueryStorage { @Getter private final StoreFactory storageFactory; + @Getter + private final StorageHandler storageHandler; + @Getter private final Validator validator; @@ -62,10 +66,11 @@ public abstract class NamespacedStorage extends ConqueryStorage { protected IdentifiableStore imports; protected IdentifiableStore> concepts; - public NamespacedStorage(StoreFactory storageFactory, String pathName, Validator validator) { + public NamespacedStorage(StoreFactory storageFactory, String pathName, Validator validator, StorageHandler storageHandler) { this.pathName = pathName; this.storageFactory = storageFactory; this.validator = validator; + this.storageHandler = storageHandler; } public void openStores(ObjectMapper objectMapper) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java index 12d0550cce..fe6130307d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.io.storage.xodus.stores.KeyIncludingStore; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; +import com.bakdata.conquery.mode.cluster.ClusterStorageHandler; import com.bakdata.conquery.models.config.StoreFactory; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.events.Bucket; @@ -28,7 +29,7 @@ public class WorkerStorage extends NamespacedStorage { private IdentifiableStore cBlocks; public WorkerStorage(StoreFactory storageFactory, Validator validator, String pathName) { - super(storageFactory, pathName, validator); + super(storageFactory, pathName, validator, new ClusterStorageHandler()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java index 77765ac315..c82eab5779 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java @@ -3,11 +3,16 @@ import javax.validation.Validator; import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.mode.cluster.ClusterStorageHandler; +import com.bakdata.conquery.mode.local.SqlStorageHandler; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.sql.execution.SqlExecutionService; import io.dropwizard.setup.Environment; /** @@ -27,16 +32,39 @@ static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig return new InternalObjectMapperCreator(config, validator); } - static DatasetRegistry createDatasetRegistry(NamespaceHandler namespaceHandler, ConqueryConfig config, - InternalObjectMapperCreator creator) { + static DatasetRegistry createDistributedDatasetRegistry( + NamespaceHandler namespaceHandler, + ConqueryConfig config, + InternalObjectMapperCreator creator + ) { + ClusterStorageHandler storageHandler = new ClusterStorageHandler(); + return createDatasetRegistry(namespaceHandler, creator, storageHandler, config); + } + + static DatasetRegistry createLocalDatasetRegistry( + NamespaceHandler namespaceHandler, + ConqueryConfig config, + InternalObjectMapperCreator creator, + SqlExecutionService sqlExecutionService + ) { + SqlStorageHandler storageHandler = new SqlStorageHandler(sqlExecutionService); + return createDatasetRegistry(namespaceHandler, creator, storageHandler, config); + } + private static DatasetRegistry createDatasetRegistry( + NamespaceHandler namespaceHandler, + InternalObjectMapperCreator creator, + StorageHandler storageHandler, + ConqueryConfig config + ) { final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), config.getIndex().getEmptyLabel()); DatasetRegistry datasetRegistry = new DatasetRegistry<>( config.getCluster().getEntityBucketSize(), config, creator, namespaceHandler, - indexService + indexService, + storageHandler ); MetaStorage storage = new MetaStorage(config.getStorage(), datasetRegistry); datasetRegistry.setMetaStorage(storage); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/StorageHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/StorageHandler.java new file mode 100644 index 0000000000..f9f9780127 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/StorageHandler.java @@ -0,0 +1,12 @@ +package com.bakdata.conquery.mode; + +import java.util.stream.Stream; + +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.datasets.Column; + +public interface StorageHandler { + + Stream lookupColumnValues(NamespaceStorage namespaceStorage, Column column); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java index 085db3eb29..ec25ecab4d 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java @@ -27,7 +27,7 @@ public ClusterManager provideManager(ConqueryConfig config, Environment environm final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); final ClusterState clusterState = new ClusterState(); final NamespaceHandler namespaceHandler = new ClusterNamespaceHandler(clusterState, config, creator); - final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + final DatasetRegistry datasetRegistry = ManagerProvider.createDistributedDatasetRegistry(namespaceHandler, config, creator); creator.init(datasetRegistry); final ClusterConnectionManager connectionManager = diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageHandler.java new file mode 100644 index 0000000000..d2579a7af4 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageHandler.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.mode.cluster; + +import java.util.stream.Stream; + +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.StorageHandler; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.ImportColumn; +import com.bakdata.conquery.models.events.stores.root.StringStore; + +public class ClusterStorageHandler implements StorageHandler { + + @Override + public Stream lookupColumnValues(NamespaceStorage namespaceStorage, Column column) { + return namespaceStorage.getAllImports().stream() + .filter(imp -> imp.getTable().equals(column.getTable())) + .flatMap(imp -> { + final ImportColumn importColumn = imp.getColumns()[column.getPosition()]; + return ((StringStore) importColumn.getTypeDescription()).iterateValues(); + }); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index 20cf172849..6214b22374 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -19,6 +19,8 @@ import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect; import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; +import com.bakdata.conquery.sql.execution.SqlExecutionService; import io.dropwizard.setup.Environment; import org.jooq.DSLContext; @@ -34,8 +36,14 @@ public DelegateManager provideManager(ConqueryConfig config, Env DSLContext dslContext = DslContextFactory.create(sqlConnectorConfig); SqlDialect sqlDialect = createSqlDialect(sqlConnectorConfig, dslContext); SqlContext sqlContext = new SqlContext(sqlConnectorConfig, sqlDialect); - NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, sqlContext); - DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + + SqlExecutionService sqlExecutionService = new SqlExecutionService( + sqlDialect.getDSLContext(), + ResultSetProcessorFactory.create(sqlDialect) + ); + + NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, sqlContext, sqlExecutionService); + DatasetRegistry datasetRegistry = ManagerProvider.createLocalDatasetRegistry(namespaceHandler, config, creator, sqlExecutionService); creator.init(datasetRegistry); return new DelegateManager<>( diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 836b5ef1a5..ce79a76c6c 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.sql.SqlContext; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; +import com.bakdata.conquery.sql.execution.SqlExecutionService; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor @@ -20,16 +21,18 @@ public class LocalNamespaceHandler implements NamespaceHandler { private final ConqueryConfig config; private final InternalObjectMapperCreator mapperCreator; private final SqlContext sqlContext; + private final SqlExecutionService sqlExecutionService; @Override public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, IndexService indexService) { NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator, indexService); - ExecutionManager executionManager = new SqlExecutionManager(sqlContext, metaStorage); + ExecutionManager executionManager = new SqlExecutionManager(sqlContext, sqlExecutionService, metaStorage); return new LocalNamespace( namespaceData.getPreprocessMapper(), namespaceData.getCommunicationMapper(), namespaceStorage, executionManager, + sqlExecutionService, namespaceData.getJobManager(), namespaceData.getFilterSearch(), namespaceData.getIndexService(), diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlStorageHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlStorageHandler.java new file mode 100644 index 0000000000..5e3141f1b0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlStorageHandler.java @@ -0,0 +1,46 @@ +package com.bakdata.conquery.mode.local; + +import java.util.stream.Stream; + +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.StorageHandler; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.sql.execution.SqlExecutionService; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Select; +import org.jooq.impl.DSL; + +@Slf4j +public class SqlStorageHandler implements StorageHandler { + + private final SqlExecutionService sqlExecutionService; + private final DSLContext dslContext; + + public SqlStorageHandler(SqlExecutionService sqlExecutionService) { + this.sqlExecutionService = sqlExecutionService; + this.dslContext = sqlExecutionService.getDslContext(); + } + + @Override + public Stream lookupColumnValues(NamespaceStorage namespaceStorage, Column column) { + Select> columValuesQuery = dslContext.selectDistinct(DSL.field(DSL.name(column.getName()))) + .from(DSL.table(DSL.name(column.getTable().getName()))); + return queryForDistinctValues(columValuesQuery); + } + + private Stream queryForDistinctValues(Select> columValuesQuery) { + try { + return sqlExecutionService.fetchStream(columValuesQuery) + .map(record -> record.get(0, String.class)) + // the database might return null or a blank string as a distinct value + .filter(value -> value != null && !value.isBlank()); + } + catch (Exception e) { + log.error("Expecting exactly 1 column in Result when querying for distinct values of a column. Query: {}.", columValuesQuery, e); + } + return Stream.empty(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index b987030c87..32c937a701 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.io.storage.WorkerStorage; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; +import com.bakdata.conquery.mode.StorageHandler; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; @@ -37,7 +38,7 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") public interface StoreFactory { - Collection discoverNamespaceStorages(); + Collection discoverNamespaceStorages(StorageHandler storageHandler); Collection discoverWorkerStorages(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 4e28a84f8c..2aa54ed50e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -38,6 +38,7 @@ import com.bakdata.conquery.io.storage.xodus.stores.StoreInfo; import com.bakdata.conquery.io.storage.xodus.stores.WeakCachedStore; import com.bakdata.conquery.io.storage.xodus.stores.XodusStore; +import com.bakdata.conquery.mode.StorageHandler; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; @@ -93,7 +94,6 @@ @CPSType(id = "XODUS", base = StoreFactory.class) public class XodusStoreFactory implements StoreFactory { - /** * The store names are created by hand here because the abstraction of {@link BigStore} * creates two stores. Defining the expected stores like this, does not require a lot or complicated logic. @@ -198,8 +198,8 @@ public ExecutorService getReaderExecutorService() { Multimaps.synchronizedSetMultimap(MultimapBuilder.hashKeys().hashSetValues().build()); @Override - public Collection discoverNamespaceStorages() { - return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath, getValidator()), NAMESPACE_STORES); + public Collection discoverNamespaceStorages(StorageHandler storageHandler) { + return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath, getValidator(), storageHandler), NAMESPACE_STORES); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index f7303d091b..3b5ceefdca 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -13,7 +13,6 @@ import com.bakdata.conquery.models.dictionary.Dictionary; import com.bakdata.conquery.models.dictionary.MapDictionary; import com.bakdata.conquery.models.events.MajorTypeId; -import com.bakdata.conquery.models.events.stores.root.StringStore; import com.bakdata.conquery.models.identifiable.IdMutex; import com.bakdata.conquery.models.identifiable.Labeled; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; @@ -156,16 +155,10 @@ public TrieSearch createTrieSearch(IndexConfig config, NamespaceS final TrieSearch search = new TrieSearch<>(suffixLength, config.getSearchSplitChars()); - storage.getAllImports().stream() - .filter(imp -> imp.getTable().equals(getTable())) - .flatMap(imp -> { - final ImportColumn importColumn = imp.getColumns()[getPosition()]; - - return ((StringStore) importColumn.getTypeDescription()).iterateValues(); - }) + storage.getStorageHandler() + .lookupColumnValues(storage, this) .map(value -> new FrontendValue(value, value)) .onClose(() -> log.debug("DONE processing values for {}", getId())) - .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); diff --git a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java index eb02b756e2..225e0d629e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java +++ b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.error; -import java.sql.SQLException; import java.util.Set; import java.util.UUID; @@ -256,7 +255,7 @@ public String getMessageTemplate(ErrorMessages errorMessages) { @CPSType(base = ConqueryError.class, id = "CQ_SQL_ERROR") @RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) public static class SqlError extends ConqueryError { - private final SQLException error; + private final Exception error; @Override public String getMessageTemplate(ErrorMessages errorMessages) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java b/backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java index a7225ffe63..027c05368c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java +++ b/backend/src/main/java/com/bakdata/conquery/models/error/ErrorMessages.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.error; -import java.sql.SQLException; - import c10n.annotations.De; import c10n.annotations.En; import com.bakdata.conquery.models.forms.util.Alignment; @@ -60,5 +58,5 @@ public interface ErrorMessages { @En("Something went wrong while querying the database: ${0}.") @De("Etwas ist beim Anfragen des Servers fehlgeschlagen: ${0}.") - String sqlError(SQLException error); + String sqlError(Exception error); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index b0dcbc1a38..4230316a7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.mode.StorageHandler; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; @@ -51,14 +52,17 @@ public class DatasetRegistry extends IdResolveContext imple @Getter @Setter private MetaStorage metaStorage; + private final NamespaceHandler namespaceHandler; private final IndexService indexService; + @Getter + private final StorageHandler storageHandler; public N createNamespace(Dataset dataset, Validator validator) throws IOException { // Prepare empty storage - NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName(), validator); + NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName(), validator, storageHandler); final ObjectMapper persistenceMapper = internalObjectMapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); datasetStorage.openStores(persistenceMapper); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java index 4ef1a5d78f..eb59d79385 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java @@ -23,14 +23,22 @@ @Slf4j @Getter @ToString(onlyExplicitlyIncluded = true) -public class DistributedNamespace extends LocalNamespace { +public class DistributedNamespace extends Namespace { private final WorkerHandler workerHandler; private final DistributedExecutionManager executionManager; - public DistributedNamespace(ObjectMapper preprocessMapper, ObjectMapper communicationMapper, NamespaceStorage storage, DistributedExecutionManager executionManager, - JobManager jobManager, FilterSearch filterSearch, IndexService indexService, List injectables, - WorkerHandler workerHandler) { + public DistributedNamespace( + ObjectMapper preprocessMapper, + ObjectMapper communicationMapper, + NamespaceStorage storage, + DistributedExecutionManager executionManager, + JobManager jobManager, + FilterSearch filterSearch, + IndexService indexService, + List injectables, + WorkerHandler workerHandler + ) { super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables); this.executionManager = executionManager; this.workerHandler = workerHandler; diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java index d77b93adff..5bab2bad96 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -1,141 +1,34 @@ package com.bakdata.conquery.models.worker; -import java.io.IOException; -import java.util.ArrayList; import java.util.List; -import java.util.NoSuchElementException; -import java.util.function.Function; import com.bakdata.conquery.io.jackson.Injectable; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.PreviewConfig; -import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; -import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; -import com.bakdata.conquery.models.jobs.SimpleJob; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.FilterSearch; +import com.bakdata.conquery.sql.execution.SqlExecutionService; import com.fasterxml.jackson.databind.ObjectMapper; -import lombok.AccessLevel; import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -@Slf4j @Getter -@ToString(onlyExplicitlyIncluded = true) -@RequiredArgsConstructor -public class LocalNamespace extends IdResolveContext implements Namespace { - - private final ObjectMapper preprocessMapper; - private final ObjectMapper communicationMapper; - @ToString.Include - private final NamespaceStorage storage; - - private final ExecutionManager executionManager; - - // TODO: 01.07.2020 FK: This is not used a lot, as NamespacedMessages are highly convoluted and hard to decouple as is. - private final JobManager jobManager; - - private final FilterSearch filterSearch; - - private final IndexService indexService; - - // Jackson's injectables that are available when deserializing requests (see PathParamInjector) or items from the storage - private final List injectables; - - - @Override - public Dataset getDataset() { - return storage.getDataset(); - } - - @Override - public void close() { - try { - jobManager.close(); - } - catch (Exception e) { - log.error("Unable to close namespace jobmanager of {}", this, e); - } - - try { - log.info("Closing namespace storage of {}", getStorage().getDataset().getId()); - storage.close(); - } - catch (IOException e) { - log.error("Unable to close namespace storage of {}.", this, e); - } - } - - @Override - public void remove() { - try { - jobManager.close(); - } - catch (Exception e) { - log.error("Unable to close namespace jobmanager of {}", this, e); - } - - log.info("Removing namespace storage of {}", getStorage().getDataset().getId()); - storage.removeStorage(); - } - - @Override - public CentralRegistry getCentralRegistry() { - return getStorage().getCentralRegistry(); - } - - @Override - public int getNumberOfEntities() { - return getStorage().getPrimaryDictionary().getSize(); - } - - @Override - public void updateInternToExternMappings() { - storage.getAllConcepts().stream() - .flatMap(c -> c.getConnectors().stream()) - .flatMap(con -> con.getSelects().stream()) - .filter(MappableSingleColumnSelect.class::isInstance) - .map(MappableSingleColumnSelect.class::cast) - .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s::loadMapping))); - - storage.getSecondaryIds().stream() - .filter(desc -> desc.getMapping() != null) - .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s.getMapping()::init))); - } - - @Override - public void clearIndexCache() { - indexService.evictCache(); - } - - @Override - public PreviewConfig getPreviewConfig() { - return getStorage().getPreviewConfig(); - } - - @Override - public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException { - if (!this.getDataset().getId().equals(dataset)) { - throw new NoSuchElementException("Wrong dataset: '" + dataset + "' (expected: '" + this.getDataset().getId() + "')"); - } - return storage.getCentralRegistry(); - } - - @Override - public CentralRegistry getMetaRegistry() { - throw new UnsupportedOperationException(); - } - - @Override - public ExecutionManager getExecutionManager() { - return executionManager; +public class LocalNamespace extends Namespace { + + private final SqlExecutionService sqlExecutionService; + + public LocalNamespace( + ObjectMapper preprocessMapper, + ObjectMapper communicationMapper, + NamespaceStorage storage, + ExecutionManager executionManager, + SqlExecutionService sqlExecutionService, + JobManager jobManager, + FilterSearch filterSearch, + IndexService indexService, + List injectables + ) { + super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables); + this.sqlExecutionService = sqlExecutionService; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java index e4f16697e0..bc4bd608a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.models.worker; -import java.io.Closeable; +import java.io.IOException; import java.util.List; import java.util.NoSuchElementException; @@ -8,52 +8,118 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.Identifiable; -import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.jobs.SimpleJob; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.FilterSearch; import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@ToString(onlyExplicitlyIncluded = true) +@RequiredArgsConstructor +public abstract class Namespace extends IdResolveContext { + + private final ObjectMapper preprocessMapper; + + private final ObjectMapper communicationMapper; + + @ToString.Include + private final NamespaceStorage storage; + + private final ExecutionManager executionManager; + + // TODO: 01.07.2020 FK: This is not used a lot, as NamespacedMessages are highly convoluted and hard to decouple as is. + private final JobManager jobManager; + + private final FilterSearch filterSearch; + + private final IndexService indexService; + + // Jackson's injectables that are available when deserializing requests (see PathParamInjector) or items from the storage + private final List injectables; + + public Dataset getDataset() { + return storage.getDataset(); + } + + public void close() { + try { + jobManager.close(); + } + catch (Exception e) { + log.error("Unable to close namespace jobmanager of {}", this, e); + } + + try { + log.info("Closing namespace storage of {}", getStorage().getDataset().getId()); + storage.close(); + } + catch (IOException e) { + log.error("Unable to close namespace storage of {}.", this, e); + } + } + + public void remove() { + try { + jobManager.close(); + } + catch (Exception e) { + log.error("Unable to close namespace jobmanager of {}", this, e); + } + + log.info("Removing namespace storage of {}", getStorage().getDataset().getId()); + storage.removeStorage(); + } + + public CentralRegistry getCentralRegistry() { + return getStorage().getCentralRegistry(); + } + + public int getNumberOfEntities() { + return getStorage().getPrimaryDictionary().getSize(); + } + + public void updateInternToExternMappings() { + storage.getAllConcepts().stream() + .flatMap(c -> c.getConnectors().stream()) + .flatMap(con -> con.getSelects().stream()) + .filter(MappableSingleColumnSelect.class::isInstance) + .map(MappableSingleColumnSelect.class::cast) + .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s::loadMapping))); + + storage.getSecondaryIds().stream() + .filter(desc -> desc.getMapping() != null) + .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s.getMapping()::init))); + } + + public void clearIndexCache() { + indexService.evictCache(); + } + + public PreviewConfig getPreviewConfig() { + return getStorage().getPreviewConfig(); + } + + @Override + public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException { + if (!this.getDataset().getId().equals(dataset)) { + throw new NoSuchElementException("Wrong dataset: '" + dataset + "' (expected: '" + this.getDataset().getId() + "')"); + } + return storage.getCentralRegistry(); + } + + @Override + public CentralRegistry getMetaRegistry() { + throw new UnsupportedOperationException(); + } -public interface Namespace extends Injectable, Closeable { - - Dataset getDataset(); - - void remove(); - - CentralRegistry getCentralRegistry(); - - int getNumberOfEntities(); - - void updateInternToExternMappings(); - - void clearIndexCache(); - - PreviewConfig getPreviewConfig(); - - CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException; - - CentralRegistry getMetaRegistry(); - - ExecutionManager getExecutionManager(); - - ObjectMapper getPreprocessMapper(); - - ObjectMapper getCommunicationMapper(); - - NamespaceStorage getStorage(); - - JobManager getJobManager(); - - FilterSearch getFilterSearch(); - - IndexService getIndexService(); - - List getInjectables(); - - & NamespacedId, T extends Identifiable> T resolve(ID id); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index 58c9e3d969..680ad81dad 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -17,7 +17,6 @@ import com.bakdata.conquery.sql.SqlContext; import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; -import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import com.bakdata.conquery.sql.execution.SqlExecutionResult; import com.bakdata.conquery.sql.execution.SqlExecutionService; @@ -29,13 +28,10 @@ public class SqlExecutionManager implements ExecutionManager { private final SqlExecutionService executionService; private final SqlConverter converter; - public SqlExecutionManager(final SqlContext context, MetaStorage metaStorage) { + public SqlExecutionManager(final SqlContext context, SqlExecutionService sqlExecutionService, MetaStorage metaStorage) { SqlDialect sqlDialect = context.getSqlDialect(); this.metaStorage = metaStorage; - this.executionService = new SqlExecutionService( - sqlDialect.getDSLContext(), - ResultSetProcessorFactory.create(sqlDialect) - ); + this.executionService = sqlExecutionService; this.converter = new SqlConverter(sqlDialect, context.getConfig()); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java index 45534e32cf..cf73d6ff07 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.execution; import java.sql.Connection; +import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -8,6 +9,7 @@ import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @@ -15,9 +17,14 @@ import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conquery.SqlManagedQuery; import com.google.common.base.Stopwatch; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.Select; +import org.jooq.exception.DataAccessException; @RequiredArgsConstructor @Slf4j @@ -26,7 +33,9 @@ public class SqlExecutionService { private static final int PID_COLUMN_INDEX = 1; private static final int VALUES_OFFSET_INDEX = 2; + @Getter private final DSLContext dslContext; + private final ResultSetProcessor resultSetProcessor; public SqlExecutionResult execute(SqlManagedQuery sqlQuery) { @@ -37,12 +46,42 @@ public SqlExecutionResult execute(SqlManagedQuery sqlQuery) { return result; } + public Result fetch(Select query) { + log.debug("Executing query: \n{}", query); + try { + return dslContext.fetch(query); + } + catch (DataAccessException exception) { + throw new ConqueryError.SqlError(exception); + } + } + + /** + * Executes the query and returns the results as a Stream. + *

+ * Note: The returned Stream is resourceful. It must be closed by the caller, because it contains a reference to an open {@link ResultSet} + * and {@link PreparedStatement}. + * + * @param query The query to be executed. + * @return A Stream of query results. + */ + public Stream fetchStream(Select query) { + log.debug("Executing query: \n{}", query); + try { + return dslContext.fetchStream(query); + } + catch (DataAccessException exception) { + throw new ConqueryError.SqlError(exception); + } + } + private SqlExecutionResult createStatementAndExecute(SqlManagedQuery sqlQuery, Connection connection) { String sqlString = sqlQuery.getSqlQuery().getSql(); List> resultTypes = sqlQuery.getSqlQuery().getResultInfos().stream().map(ResultInfo::getType).collect(Collectors.toList()); log.debug("Executing query: \n{}", sqlString); + try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sqlString)) { int columnCount = resultSet.getMetaData().getColumnCount(); @@ -51,12 +90,9 @@ private SqlExecutionResult createStatementAndExecute(SqlManagedQuery sqlQuery, C return new SqlExecutionResult(columnNames, resultTable); } - catch (SQLException e) { - throw new ConqueryError.SqlError(e); - } // not all DB vendors throw SQLExceptions - catch (RuntimeException e) { - throw new ConqueryError.SqlError(new SQLException(e)); + catch (SQLException | RuntimeException e) { + throw new ConqueryError.SqlError(e); } } diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index de8b43a227..ceed909a25 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -60,7 +60,7 @@ public class StoredQueriesProcessorTest { public static final AuthorizationController AUTHORIZATION_CONTROLLER = new AuthorizationController(STORAGE, new DevelopmentAuthorizationConfig()); public static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final DatasetRegistry datasetRegistry = new DatasetRegistry<>(0, CONFIG, null, null, null); + private static final DatasetRegistry datasetRegistry = new DatasetRegistry<>(0, CONFIG, null, null, null, null); private static final QueryProcessor processor = new QueryProcessor(datasetRegistry, STORAGE, CONFIG); private static final Dataset DATASET_0 = new Dataset() {{ diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java index 48c2c39ada..6892260c8d 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java @@ -10,6 +10,7 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalInt; +import java.util.Set; import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; @@ -47,6 +48,11 @@ public class FilterAutocompleteTest extends IntegrationTest.Simple implements Pr "b,lbl-6,ov-6" }; + @Override + public Set forModes() { + return Set.of(StandaloneSupport.Mode.WORKER, StandaloneSupport.Mode.SQL); + } + @Override public void execute(StandaloneSupport conquery) throws Exception { //read test specification @@ -158,4 +164,4 @@ public void execute(StandaloneSupport conquery) throws Exception { .containsExactly("", "aaa", "a", "baaa", "aab", "b", "f", "fm", "m", "mf"); } } -} \ No newline at end of file +} diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index e09081772f..2e0a9d36e7 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -41,7 +41,7 @@ public void before() { InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, validator); final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), "emptyDefaultLabel"); final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler(new ClusterState(), config, creator); - datasetRegistry = new DatasetRegistry<>(0, config, null, clusterNamespaceHandler, indexService); + datasetRegistry = new DatasetRegistry<>(0, config, null, clusterNamespaceHandler, indexService, null); metaStorage = new MetaStorage(new NonPersistentStoreFactory(), datasetRegistry); datasetRegistry.setMetaStorage(metaStorage); creator.init(datasetRegistry); diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java index 69f77b30f9..86334a2f02 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java @@ -39,7 +39,7 @@ public void testListReferences() throws IOException { registry.register(dataset); registry.register(table); - final DatasetRegistry datasetRegistry = new DatasetRegistry<>(0, null, null, null, null); + final DatasetRegistry datasetRegistry = new DatasetRegistry<>(0, null, null, null, null, null); final MetaStorage metaStorage = new MetaStorage(new NonPersistentStoreFactory(),datasetRegistry); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index 93d58023b7..ca9b74af6c 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.io.storage.StoreMappings; import com.bakdata.conquery.io.storage.WorkerStorage; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; +import com.bakdata.conquery.mode.StorageHandler; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; @@ -68,7 +69,7 @@ public class NonPersistentStoreFactory implements StoreFactory { @Override - public Collection discoverNamespaceStorages() { + public Collection discoverNamespaceStorages(StorageHandler storageHandler) { return Collections.emptyList(); }