diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java index 313eab5f96..6051d23a9c 100644 --- a/backend/src/main/java/com/bakdata/conquery/Conquery.java +++ b/backend/src/main/java/com/bakdata/conquery/Conquery.java @@ -1,12 +1,9 @@ package com.bakdata.conquery; +import jakarta.validation.Validator; + import ch.qos.logback.classic.Level; -import com.bakdata.conquery.commands.DistributedStandaloneCommand; -import com.bakdata.conquery.commands.ManagerNode; -import com.bakdata.conquery.commands.MigrateCommand; -import com.bakdata.conquery.commands.PreprocessorCommand; -import com.bakdata.conquery.commands.RecodeStoreCommand; -import com.bakdata.conquery.commands.ShardNode; +import com.bakdata.conquery.commands.*; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.metrics.prometheus.PrometheusBundle; @@ -22,7 +19,6 @@ import io.dropwizard.core.ConfiguredBundle; import io.dropwizard.core.setup.Bootstrap; import io.dropwizard.core.setup.Environment; -import jakarta.validation.Validator; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -51,7 +47,7 @@ public void initialize(Bootstrap bootstrap) { // main config file is json bootstrap.setConfigurationFactoryFactory(JsonConfigurationFactory::new); - bootstrap.addCommand(new ShardNode()); + bootstrap.addCommand(new ShardCommand()); bootstrap.addCommand(new PreprocessorCommand()); bootstrap.addCommand(new DistributedStandaloneCommand(this)); bootstrap.addCommand(new RecodeStoreCommand()); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 1dbb9753d1..92d2b9b463 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -39,6 +39,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; +import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.metrics.ExecutionMetrics; @@ -108,7 +109,6 @@ public class QueryProcessor { private Validator validator; - public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { final Collection allQueries = storage.getAllExecutions(); @@ -294,14 +294,13 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject su public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, ExternalUpload upload) { final Namespace namespace = datasetRegistry.get(dataset.getId()); - final CQExternal.ResolveStatistic statistic = CQExternal.resolveEntities( + final EntityResolver.ResolveStatistic statistic = namespace.getEntityResolver().resolveEntities( upload.getValues(), upload.getFormat(), namespace.getStorage().getIdMapping(), config.getIdColumns(), config.getLocale().getDateReader(), - upload.isOneRowPerEntity(), - true + upload.isOneRowPerEntity() ); // Resolving nothing is a problem thus we fail. diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java index b698e7b26a..95f9e1c94b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendSelect.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.apiv1.frontend; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; -import com.bakdata.conquery.models.types.ResultType; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Builder; import lombok.Data; @@ -15,7 +14,7 @@ public class FrontendSelect { private SelectId id; private String label; private String description; - private ResultType resultType; + private String resultType; @JsonProperty("default") private Boolean isDefault; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 48466e2dd8..2ba0d02a0b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; @@ -11,7 +10,6 @@ import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.CQElement; @@ -21,7 +19,6 @@ import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -33,16 +30,12 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo; import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.util.DateReader; -import com.bakdata.conquery.util.io.IdColumnUtil; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonView; import com.google.common.collect.Streams; import io.dropwizard.validation.ValidationMethod; import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; import lombok.AccessLevel; -import lombok.Data; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -55,8 +48,6 @@ @NoArgsConstructor public class CQExternal extends CQElement { - private static final String FORMAT_EXTRA = "EXTRA"; - /** * Describes the format of {@code values}, how to extract data from each row: *

@@ -118,7 +109,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { final String[] extraHeaders = Streams.zip( Arrays.stream(headers), format.stream(), - (header, format) -> format.equals(FORMAT_EXTRA) ? header : null + (header, format) -> format.equals(EntityResolverUtil.FORMAT_EXTRA) ? header : null ) .filter(Objects::nonNull) .toArray(String[]::new); @@ -180,78 +171,18 @@ private ExternalNode> createExternalNodeForList(QueryPlanContext co ); } - /** - * For each row try and collect all dates. - * - * @return Row -> Dates - */ - private static CDateSet[] readDates(String[][] values, List format, DateReader dateReader) { - final CDateSet[] out = new CDateSet[values.length]; - - final List dateFormats = format.stream() - .map(CQExternal::resolveDateFormat) - // Don't use Stream#toList to preserve null-values - .collect(Collectors.toList()); - - - /* - If no format is provided, put empty dates into output. - This indicates that no date context was provided and - the entries are not restricted by any date restriction, - but can also don't contribute to any date aggregation. - */ - if (dateFormats.stream().allMatch(Objects::isNull)) { - // Initialize empty - for (int row = 0; row < values.length; row++) { - out[row] = CDateSet.createEmpty(); - } - return out; - } - - for (int row = 1; row < values.length; row++) { - try { - final CDateSet dates = CDateSet.createEmpty(); - - // Collect all specified dates into a single set. - for (int col = 0; col < dateFormats.size(); col++) { - final DateFormat dateFormat = dateFormats.get(col); - - if (dateFormat == null) { - continue; - } - dateFormat.readDates(values[row][col], dateReader, dates); - } - - if (dates.isEmpty()) { - continue; - } - - if (out[row] == null) { - out[row] = CDateSet.createEmpty(); - } - - out[row].addAll(dates); - } - catch (Exception e) { - log.warn("Failed to parse Date from {}", row, e); - } - } - - return out; - } - @Override public void resolve(QueryResolveContext context) { headers = values[0]; - final ResolveStatistic resolved = - resolveEntities(values, format, - context.getNamespace().getStorage().getIdMapping(), - context.getConfig().getIdColumns(), - context.getConfig().getLocale().getDateReader(), - onlySingles, - context.getConfig().getSqlConnectorConfig().isEnabled() - ); + final EntityResolver.ResolveStatistic resolved = context.getNamespace().getEntityResolver().resolveEntities( + values, + format, + context.getNamespace().getStorage().getIdMapping(), + context.getConfig().getIdColumns(), + context.getConfig().getLocale().getDateReader(), + onlySingles + ); if (resolved.getResolved().isEmpty()) { throw new ConqueryError.ExternalResolveEmptyError(); @@ -277,158 +208,6 @@ public void resolve(QueryResolveContext context) { extra = resolved.getExtra(); } - @Data - public static class ResolveStatistic { - - @JsonIgnore - private final Map resolved; - - /** - * Entity -> Column -> Values - */ - @JsonIgnore - private final Map>> extra; - - private final List unreadableDate; - private final List unresolvedId; - - } - - /** - * Helper method to try and resolve entities in values using the specified format. - */ - public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List format, EntityIdMap mapping, IdColumnConfig idColumnConfig, @NotNull DateReader dateReader, boolean onlySingles, boolean isInSqlMode) { - final Map resolved = new HashMap<>(); - - final List unresolvedDate = new ArrayList<>(); - final List unresolvedId = new ArrayList<>(); - - // extract dates from rows - final CDateSet[] rowDates = readDates(values, format, dateReader); - - // Extract extra data from rows by Row, to be collected into by entities - // Row -> Column -> Value - final Map[] extraDataByRow = readExtras(values, format); - - final List> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers()); - - // We will not be able to resolve anything... - if (readers.isEmpty()) { - return new ResolveStatistic(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), List.of(values)); - } - - // Entity -> Column -> Values - final Map>> extraDataByEntity = new HashMap<>(); - - // ignore the first row, because this is the header - for (int rowNum = 1; rowNum < values.length; rowNum++) { - - final String[] row = values[rowNum]; - - if (rowDates[rowNum] == null) { - unresolvedDate.add(row); - continue; - } - - // TODO proper implementation of EntityIdMap#resolve for SQL mode - String resolvedId = isInSqlMode - ? String.valueOf(row[0]) - : tryResolveId(row, readers, mapping); - - if (resolvedId == null) { - unresolvedId.add(row); - continue; - } - - //read the dates from the row - resolved.put(resolvedId, rowDates[rowNum]); - - // Entity was resolved for row so we collect the data. - if (extraDataByRow[rowNum] != null) { - - for (Map.Entry entry : extraDataByRow[rowNum].entrySet()) { - extraDataByEntity.computeIfAbsent(resolvedId, (ignored) -> new HashMap<>()) - .computeIfAbsent(entry.getKey(), (ignored) -> new ArrayList<>()) - .add(entry.getValue()); - } - } - } - - if (onlySingles) { - // Check that there is at most one value per entity and per column - final boolean alright = extraDataByEntity.values().stream() - .map(Map::values) - .flatMap(Collection::stream) - .allMatch(l -> l.size() <= 1); - if (!alright) { - throw new ConqueryError.ExternalResolveOnePerRowError(); - } - } - - return new ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId); - } - - /** - * Try to extract a {@link com.bakdata.conquery.models.identifiable.mapping.EntityIdMap.ExternalId} from the row, - * then try to map it to an internal {@link com.bakdata.conquery.models.query.entity.Entity} - */ - private static String tryResolveId(String[] row, List> readers, EntityIdMap mapping) { - String resolvedId = null; - - for (Function reader : readers) { - final EntityIdMap.ExternalId externalId = reader.apply(row); - - if (externalId == null) { - continue; - } - - String innerResolved = mapping.resolve(externalId); - - if (innerResolved == null) { - continue; - } - - // Only if all resolvable ids agree on the same entity, do we return the id. - if (resolvedId != null && !innerResolved.equals(resolvedId)) { - log.error("`{}` maps to different Entities", (Object) row); - continue; - } - - resolvedId = innerResolved; - } - return resolvedId; - } - - /** - * Try and extract Extra data from input to be returned as extra-data in output. - *

- * Line -> ( Column -> Value ) - */ - private static Map[] readExtras(String[][] values, List format) { - final String[] names = values[0]; - final Map[] extrasByRow = new Map[values.length]; - - - for (int line = 1; line < values.length; line++) { - for (int col = 0; col < format.size(); col++) { - if (!format.get(col).equals(FORMAT_EXTRA)) { - continue; - } - - - if (extrasByRow[line] == null) { - extrasByRow[line] = new HashMap<>(names.length); - } - - extrasByRow[line].put(names[col], values[line][col]); - } - } - - - return extrasByRow; - } - - @Override public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { return new RequiredEntities(valuesResolved.keySet()); @@ -441,7 +220,7 @@ public List getResultInfos() { } List resultInfos = new ArrayList<>(); for (int col = 0; col < format.size(); col++) { - if (!format.get(col).equals(FORMAT_EXTRA)) { + if (!format.get(col).equals(EntityResolverUtil.FORMAT_EXTRA)) { continue; } @@ -484,15 +263,4 @@ public boolean isHeadersUnique() { return false; } - /** - * Try to resolve a date format, return nothing if not possible. - */ - private static DateFormat resolveDateFormat(String name) { - try { - return DateFormat.valueOf(name); - } - catch (IllegalArgumentException e) { - return null; // Does not exist - } - } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java new file mode 100644 index 0000000000..c93f1e3375 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolver.java @@ -0,0 +1,51 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import java.util.Collections; +import java.util.List; +import java.util.Map; + +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.config.IdColumnConfig; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.util.DateReader; +import com.fasterxml.jackson.annotation.JsonIgnore; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import lombok.Data; + +public interface EntityResolver { + + /** + * Helper method to try and resolve entities in values using the specified format. + */ + ResolveStatistic resolveEntities( + @NotEmpty String[][] values, + @NotEmpty List format, + EntityIdMap mapping, + IdColumnConfig idColumnConfig, + @NotNull DateReader dateReader, + boolean onlySingles + ); + + @Data + class ResolveStatistic { + + @JsonIgnore + private final Map resolved; + + /** + * Entity -> Column -> Values + */ + @JsonIgnore + private final Map>> extra; + + private final List unreadableDate; + private final List unresolvedId; + + public static ResolveStatistic forEmptyReaders(String[][] values) { + return new ResolveStatistic(Collections.emptyMap(), Collections.emptyMap(), Collections.emptyList(), List.of(values)); + } + + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java new file mode 100644 index 0000000000..3a9be46a25 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/EntityResolverUtil.java @@ -0,0 +1,179 @@ +package com.bakdata.conquery.apiv1.query.concept.specific.external; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.error.ConqueryError; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.ExternalId; +import com.bakdata.conquery.util.DateReader; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class EntityResolverUtil { + + public static final String FORMAT_EXTRA = "EXTRA"; + + /** + * For each row try and collect all dates. + * + * @return Row -> Dates + */ + public static CDateSet[] readDates(String[][] values, List format, DateReader dateReader) { + final CDateSet[] out = new CDateSet[values.length]; + + final List dateFormats = format.stream() + .map(EntityResolverUtil::resolveDateFormat) + // Don't use Stream#toList to preserve null-values + .collect(Collectors.toList()); + + /* + If no format is provided, put empty dates into output. + This indicates that no date context was provided and + the entries are not restricted by any date restriction, + but can also don't contribute to any date aggregation. + */ + if (dateFormats.stream().allMatch(Objects::isNull)) { + // Initialize empty + for (int row = 0; row < values.length; row++) { + out[row] = CDateSet.createEmpty(); + } + return out; + } + + for (int row = 1; row < values.length; row++) { + try { + final CDateSet dates = CDateSet.createEmpty(); + + // Collect all specified dates into a single set. + for (int col = 0; col < dateFormats.size(); col++) { + final DateFormat dateFormat = dateFormats.get(col); + + if (dateFormat == null) { + continue; + } + dateFormat.readDates(values[row][col], dateReader, dates); + } + + if (dates.isEmpty()) { + continue; + } + + if (out[row] == null) { + out[row] = CDateSet.createEmpty(); + } + + out[row].addAll(dates); + } + catch (Exception e) { + log.warn("Failed to parse Date from {}", row, e); + } + } + + return out; + } + + public static void collectExtraData(Map[] extraDataByRow, int rowNum, Map>> extraDataByEntity, String resolvedId) { + if (extraDataByRow[rowNum] != null) { + for (Map.Entry entry : extraDataByRow[rowNum].entrySet()) { + extraDataByEntity.computeIfAbsent(resolvedId, (ignored) -> new HashMap<>()) + .computeIfAbsent(entry.getKey(), (ignored) -> new ArrayList<>()) + .add(entry.getValue()); + } + } + } + + public static void verifyOnlySingles(boolean onlySingles, Map>> extraDataByEntity) { + if (!onlySingles) { + return; + } + // Check that there is at most one value per entity and per column + final boolean alright = extraDataByEntity.values().stream() + .map(Map::values) + .flatMap(Collection::stream) + .allMatch(l -> l.size() <= 1); + if (!alright) { + throw new ConqueryError.ExternalResolveOnePerRowError(); + } + } + + /** + * Try to extract a {@link ExternalId} from the row, + * then try to map it to an internal {@link com.bakdata.conquery.models.query.entity.Entity} + */ + public static String tryResolveId(String[] row, List> readers, EntityIdMap mapping) { + String resolvedId = null; + + for (Function reader : readers) { + final ExternalId externalId = reader.apply(row); + + if (externalId == null) { + continue; + } + + String innerResolved = mapping.resolve(externalId); + + if (innerResolved == null) { + continue; + } + + // Only if all resolvable ids agree on the same entity, do we return the id. + if (resolvedId != null && !innerResolved.equals(resolvedId)) { + log.error("`{}` maps to different Entities", (Object) row); + continue; + } + + resolvedId = innerResolved; + } + return resolvedId; + } + + /** + * Try and extract Extra data from input to be returned as extra-data in output. + *

+ * Line -> ( Column -> Value ) + */ + public static Map[] readExtras(String[][] values, List format) { + final String[] names = values[0]; + final Map[] extrasByRow = new Map[values.length]; + + + for (int line = 1; line < values.length; line++) { + for (int col = 0; col < format.size(); col++) { + if (!format.get(col).equals(FORMAT_EXTRA)) { + continue; + } + + + if (extrasByRow[line] == null) { + extrasByRow[line] = new HashMap<>(names.length); + } + + extrasByRow[line].put(names[col], values[line][col]); + } + } + + + return extrasByRow; + } + + /** + * Try to resolve a date format, return nothing if not possible. + */ + private static DateFormat resolveDateFormat(String name) { + try { + return DateFormat.valueOf(name); + } + catch (IllegalArgumentException e) { + return null; // Does not exist + } + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/commands/CollectEntitiesCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/CollectEntitiesCommand.java deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java index 1a9a0ab165..8e368fc189 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java @@ -26,7 +26,7 @@ public class DistributedStandaloneCommand extends ServerCommand private final Conquery conquery; private ClusterManager manager; - private ManagerNode managerNode = new ManagerNode(); + private final ManagerNode managerNode = new ManagerNode(); private final List shardNodes = new Vector<>(); // TODO clean up the command structure, so we can use the Environment from EnvironmentCommand @@ -104,7 +104,7 @@ public void startStandalone(Environment environment, Namespace namespace, Conque clone = config.withStorage(((XodusStoreFactory) config.getStorage()).withDirectory(managerDir)); } - sc.run(environment, namespace, clone); + sc.run(clone, environment); return sc; })); } diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index 004048186d..d1c5686040 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -46,7 +46,7 @@ import org.glassfish.jersey.internal.inject.AbstractBinder; /** - * Central node of Conquery. Hosts the frontend, api, meta data and takes care of query distribution to + * Central node of Conquery. Hosts the frontend, api, metadata and takes care of query distribution to * {@link ShardNode}s and respectively the {@link Worker}s hosted on them. The {@link ManagerNode} can also * forward queries or results to statistic backends. Finally, it collects the results of queries for access over the api. */ @@ -113,7 +113,7 @@ public void run(Manager manager) throws InterruptedException { // Create AdminServlet first to make it available to the realms admin = new AdminServlet(this); - authController = new AuthorizationController(getStorage(), config, environment, admin); + authController = new AuthorizationController(getMetaStorage(), config, environment, admin); environment.lifecycle().manage(authController); // Register default components for the admin interface @@ -145,14 +145,14 @@ public void run(Manager manager) throws InterruptedException { private void registerTasks(Manager manager, Environment environment, ConqueryConfig config) { environment.admin().addTask(formScanner); environment.admin().addTask( - new QueryCleanupTask(getStorage(), Duration.of( + new QueryCleanupTask(getMetaStorage(), Duration.of( config.getQueries().getOldQueriesTime().getQuantity(), config.getQueries().getOldQueriesTime().getUnit().toChronoUnit() ))); - environment.admin().addTask(new PermissionCleanupTask(getStorage())); + environment.admin().addTask(new PermissionCleanupTask(getMetaStorage())); manager.getAdminTasks().forEach(environment.admin()::addTask); - environment.admin().addTask(new ReloadMetaStorageTask(getStorage())); + environment.admin().addTask(new ReloadMetaStorageTask(getMetaStorage())); final ShutdownTask shutdown = new ShutdownTask(); environment.admin().addTask(shutdown); @@ -164,7 +164,7 @@ private void configureApiServlet(ConqueryConfig config, DropwizardResourceConfig jerseyConfig.register(new AbstractBinder() { @Override protected void configure() { - bind(getStorage()).to(MetaStorage.class); + bind(getMetaStorage()).to(MetaStorage.class); bind(getDatasetRegistry()).to(DatasetRegistry.class); } }); @@ -203,7 +203,7 @@ public void customizeApiObjectMapper(ObjectMapper objectMapper) { injectableValues.add(Validator.class, getValidator()); getDatasetRegistry().injectInto(objectMapper); - getStorage().injectInto(objectMapper); + getMetaStorage().injectInto(objectMapper); getConfig().injectInto(objectMapper); } @@ -219,10 +219,10 @@ public ObjectMapper createInternalObjectMapper(Class viewClass) private void loadMetaStorage() { log.info("Opening MetaStorage"); - getStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class)); + getMetaStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class)); log.info("Loading MetaStorage"); - getStorage().loadData(); - log.info("MetaStorage loaded {}", getStorage()); + getMetaStorage().loadData(); + log.info("MetaStorage loaded {}", getMetaStorage()); } @SneakyThrows(InterruptedException.class) @@ -236,7 +236,7 @@ public void loadNamespaces() { final Collection namespaceStorages = getConfig().getStorage().discoverNamespaceStorages(); for (NamespaceStorage namespaceStorage : namespaceStorages) { loaders.submit(() -> { - registry.createNamespace(namespaceStorage); + registry.createNamespace(namespaceStorage, getMetaStorage()); }); } @@ -262,16 +262,16 @@ public void stop() throws Exception { provider.close(); } catch (Exception e) { - log.error(provider + " could not be closed", e); + log.error("{} could not be closed", provider, e); } } try { - getStorage().close(); + getMetaStorage().close(); } catch (Exception e) { - log.error("{} could not be closed", getStorage(), e); + log.error("{} could not be closed", getMetaStorage(), e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java new file mode 100644 index 0000000000..bf52085427 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardCommand.java @@ -0,0 +1,39 @@ +package com.bakdata.conquery.commands; + +import java.util.Collections; + +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.util.commands.NoOpConquery; +import io.dropwizard.core.cli.ServerCommand; +import io.dropwizard.core.server.DefaultServerFactory; +import io.dropwizard.core.setup.Bootstrap; +import io.dropwizard.core.setup.Environment; +import net.sourceforge.argparse4j.inf.Namespace; + +/** + * Command to run conquery as a shard node. + */ +public class ShardCommand extends ServerCommand { + + public ShardCommand() { + super(new NoOpConquery(), "shard", "Connects this instance as a ShardNode to a running ManagerNode."); + } + + @Override + protected void run(Bootstrap bootstrap, Namespace namespace, ConqueryConfig configuration) throws Exception { + bootstrap.addBundle(new ShardNode()); + + super.run(bootstrap, namespace, configuration); + } + + @Override + protected void run(Environment environment, Namespace namespace, ConqueryConfig configuration) throws Exception { + /* + Clear application connectors for a shard, before building the server, + as we only expose the metrics through the admin connector. + */ + ((DefaultServerFactory)configuration.getServerFactory()).setApplicationConnectors(Collections.emptyList()); + + super.run(environment, namespace, configuration); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index 2293959fec..26e41c8945 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -7,6 +7,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; @@ -37,14 +38,13 @@ import com.fasterxml.jackson.databind.DeserializationConfig; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationConfig; +import io.dropwizard.core.ConfiguredBundle; import io.dropwizard.core.setup.Environment; import io.dropwizard.lifecycle.Managed; import io.dropwizard.util.Duration; -import jakarta.validation.Validator; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import net.sourceforge.argparse4j.inf.Namespace; import org.apache.mina.core.RuntimeIoException; import org.apache.mina.core.future.ConnectFuture; import org.apache.mina.core.service.IoHandler; @@ -61,10 +61,12 @@ */ @Slf4j @Getter -public class ShardNode extends ConqueryCommand implements IoHandler, Managed { +public class ShardNode implements ConfiguredBundle, IoHandler, Managed { public static final String DEFAULT_NAME = "shard-node"; + private final String name; + private NioSocketConnector connector; private ConnectFuture future; private JobManager jobManager; @@ -82,12 +84,12 @@ public ShardNode() { } public ShardNode(String name) { - super(name, "Connects this instance as a ShardNode to a running ManagerNode."); + this.name = name; } @Override - protected void run(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception { + public void run(ConqueryConfig config, Environment environment) throws Exception { this.environment = environment; this.config = config; @@ -343,7 +345,7 @@ private void connectToCluster() { future.cancel(); // Sleep thirty seconds then retry. - TimeUnit.SECONDS.sleep(30); + TimeUnit.SECONDS.sleep(config.getCluster().getConnectRetryTimeout().toSeconds()); } catch (RuntimeIoException e) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java index 0023d97757..e08b026eea 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jackson/serializer/MetaIdReferenceDeserializer.java @@ -4,18 +4,14 @@ import java.util.InputMismatchException; import java.util.Optional; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.ids.Id; import com.bakdata.conquery.models.identifiable.ids.IdUtil; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; -import com.fasterxml.jackson.databind.BeanDescription; -import com.fasterxml.jackson.databind.BeanProperty; -import com.fasterxml.jackson.databind.DeserializationContext; -import com.fasterxml.jackson.databind.JavaType; -import com.fasterxml.jackson.databind.JsonDeserializer; -import com.fasterxml.jackson.databind.JsonMappingException; +import com.fasterxml.jackson.databind.*; import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.deser.SettableBeanProperty; import com.fasterxml.jackson.databind.jsontype.TypeDeserializer; @@ -42,7 +38,7 @@ public T deserialize(JsonParser parser, DeserializationContext ctxt) throws IOEx ID id = ctxt.readValue(parser, idClass); try { - final CentralRegistry centralRegistry = CentralRegistry.get(ctxt); + final CentralRegistry centralRegistry = MetaStorage.get(ctxt).getCentralRegistry(); // Not all Components have registries, we leave it up to the validator to be angry. if (centralRegistry == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index d8c12a95b8..d0cf222cef 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -11,13 +11,9 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; -import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; -import com.bakdata.conquery.models.identifiable.ids.specific.UserId; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.identifiable.ids.specific.*; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -34,8 +30,6 @@ public class MetaStorage extends ConqueryStorage implements Injectable { protected final CentralRegistry centralRegistry = new CentralRegistry(); private final StoreFactory storageFactory; - @Getter - protected final DatasetRegistry datasetRegistry; private IdentifiableStore executions; private IdentifiableStore formConfigs; private IdentifiableStore authUser; @@ -47,8 +41,8 @@ public void openStores(ObjectMapper mapper) { authRole = storageFactory.createRoleStore(centralRegistry, "meta", this, mapper); authGroup = storageFactory.createGroupStore(centralRegistry, "meta", this, mapper); // Executions depend on users - executions = storageFactory.createExecutionsStore(centralRegistry, datasetRegistry, "meta", mapper); - formConfigs = storageFactory.createFormConfigStore(centralRegistry, datasetRegistry, "meta", mapper); + executions = storageFactory.createExecutionsStore(centralRegistry, "meta", mapper); + formConfigs = storageFactory.createFormConfigStore(centralRegistry, "meta", mapper); } @@ -196,4 +190,8 @@ public void addFormConfig(FormConfig formConfig) { public MutableInjectableValues inject(MutableInjectableValues values) { return values.add(MetaStorage.class, this); } + + public static MetaStorage get(DeserializationContext ctxt) throws JsonMappingException { + return (MetaStorage) ctxt.findInjectableValue(MetaStorage.class.getName(), null, null); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index f84b4dc544..7b29656e6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -17,7 +17,6 @@ import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import jakarta.validation.Validator; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -32,7 +31,7 @@ public class NamespaceStorage extends NamespacedStorage { protected CachedStore entity2Bucket; - public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator) { + public NamespaceStorage(StoreFactory storageFactory, String pathName) { super(storageFactory, pathName); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index a4a4c07d54..a9a7378760 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ImportId; import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; +import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import lombok.Getter; @@ -56,7 +57,8 @@ public NamespacedStorage(StoreFactory storageFactory, String pathName) { } public void openStores(ObjectMapper objectMapper) { - + // Before we start to parse the stores we need to replace the injected value for the IdResolveContext (from DatasetRegistry to this centralRegistry) + new SingletonNamespaceCollection(centralRegistry).injectInto(objectMapper); dataset = storageFactory.createDatasetStore(pathName, objectMapper); secondaryIds = storageFactory.createSecondaryIdDescriptionStore(centralRegistry, pathName, objectMapper); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java index f4a402adce..53ada11ccc 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java @@ -24,6 +24,7 @@ public class DelegateManager implements Manager { ConqueryConfig config; Environment environment; DatasetRegistry datasetRegistry; + MetaStorage storage; ImportHandler importHandler; StorageListener storageListener; Supplier> nodeProvider; @@ -42,7 +43,7 @@ public void stop() throws Exception { } @Override - public MetaStorage getStorage() { - return datasetRegistry.getMetaStorage(); + public MetaStorage getMetaStorage() { + return storage; } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java index 64d255dad8..8fdcf63076 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java @@ -23,13 +23,12 @@ @RequiredArgsConstructor public class InternalObjectMapperCreator { private final ConqueryConfig config; + private final MetaStorage storage; private final Validator validator; private DatasetRegistry datasetRegistry = null; - private MetaStorage storage = null; public void init(DatasetRegistry datasetRegistry) { this.datasetRegistry = datasetRegistry; - this.storage = datasetRegistry.getMetaStorage(); } public ObjectMapper createInternalObjectMapper(@Nullable Class viewClass) { diff --git a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java index bef1a3c444..f8de4d3035 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java @@ -10,9 +10,9 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; +import io.dropwizard.core.setup.Environment; import io.dropwizard.lifecycle.Managed; import io.dropwizard.servlets.tasks.Task; -import io.dropwizard.core.setup.Environment; /** * A manager provides the implementations that differ by running mode. @@ -27,5 +27,6 @@ public interface Manager extends Managed { List getAdminTasks(); InternalObjectMapperCreator getInternalObjectMapperCreator(); JobManager getJobManager(); - MetaStorage getStorage(); + + MetaStorage getMetaStorage(); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java index c25f63a08e..fe45f4ecbe 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java @@ -23,8 +23,8 @@ static JobManager newJobManager(ConqueryConfig config) { return new JobManager(JOB_MANAGER_NAME, config.isFailOnError()); } - static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig config, Validator validator) { - return new InternalObjectMapperCreator(config, validator); + static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig config, MetaStorage metaStorage, Validator validator) { + return new InternalObjectMapperCreator(config, metaStorage, validator); } static DatasetRegistry createDatasetRegistry( @@ -33,16 +33,13 @@ static DatasetRegistry createDatasetRegistry( InternalObjectMapperCreator creator ) { final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), config.getIndex().getEmptyLabel()); - DatasetRegistry datasetRegistry = new DatasetRegistry<>( + return new DatasetRegistry<>( config.getCluster().getEntityBucketSize(), config, creator, namespaceHandler, indexService ); - MetaStorage storage = new MetaStorage(config.getStorage(), datasetRegistry); - datasetRegistry.setMetaStorage(storage); - return datasetRegistry; } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java index 8f3e4eb1e4..fef5334580 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java @@ -4,6 +4,7 @@ import java.util.List; import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; @@ -32,6 +33,7 @@ public interface NamespaceHandler { static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalObjectMapperCreator mapperCreator, IndexService indexService) { List injectables = new ArrayList<>(); injectables.add(indexService); + ObjectMapper persistenceMapper = mapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); ObjectMapper communicationMapper = mapperCreator.createInternalObjectMapper(View.InternalCommunication.class); ObjectMapper preprocessMapper = mapperCreator.createInternalObjectMapper(null); @@ -40,8 +42,9 @@ static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final C injectables.forEach(i -> i.injectInto(communicationMapper)); injectables.forEach(i -> i.injectInto(preprocessMapper)); - // Open and load the stores - storage.openStores(persistenceMapper); + + // Each store needs its own mapper because each injects its own registry + storage.openStores(Jackson.copyMapperAndInjectables(persistenceMapper)); storage.loadData(); JobManager jobManager = new JobManager(storage.getDataset().getName(), config.isFailOnError()); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java new file mode 100644 index 0000000000..8aa3d7c720 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterEntityResolver.java @@ -0,0 +1,83 @@ +package com.bakdata.conquery.mode.cluster; + +import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.collectExtraData; +import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.readDates; +import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.tryResolveId; +import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.verifyOnlySingles; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver; +import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil; +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.config.IdColumnConfig; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.ExternalId; +import com.bakdata.conquery.util.DateReader; +import com.bakdata.conquery.util.io.IdColumnUtil; +import jakarta.validation.constraints.NotEmpty; + +public class ClusterEntityResolver implements EntityResolver { + + @Override + public ResolveStatistic resolveEntities( + @NotEmpty String[][] values, + List format, + EntityIdMap mapping, + IdColumnConfig idColumnConfig, + DateReader dateReader, + boolean onlySingles + ) { + final Map resolved = new HashMap<>(); + final List unresolvedDate = new ArrayList<>(); + final List unresolvedId = new ArrayList<>(); + + // extract dates from rows + final CDateSet[] rowDates = readDates(values, format, dateReader); + + // Extract extra data from rows by Row, to be collected into by entities + // Row -> Column -> Value + final Map[] extraDataByRow = EntityResolverUtil.readExtras(values, format); + + final List> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers()); + + // We will not be able to resolve anything... + if (readers.isEmpty()) { + return EntityResolver.ResolveStatistic.forEmptyReaders(values); + } + + // Entity -> Column -> Values + final Map>> extraDataByEntity = new HashMap<>(); + + // ignore the first row, because this is the header + for (int rowNum = 1; rowNum < values.length; rowNum++) { + + final String[] row = values[rowNum]; + + if (rowDates[rowNum] == null) { + unresolvedDate.add(row); + continue; + } + + String resolvedId = tryResolveId(row, readers, mapping); + + if (resolvedId == null) { + unresolvedId.add(row); + continue; + } + + // read the dates from the row + resolved.put(resolvedId, rowDates[rowNum]); + + // Entity was resolved for row, so we collect the data. + collectExtraData(extraDataByRow, rowNum, extraDataByEntity, resolvedId); + } + + verifyOnlySingles(onlySingles, extraDataByEntity); + return new EntityResolver.ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java index 1579acf868..cafc855713 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.function.Supplier; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.*; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; @@ -19,7 +20,8 @@ public class ClusterManagerProvider implements ManagerProvider { public ClusterManager provideManager(ConqueryConfig config, Environment environment) { final JobManager jobManager = ManagerProvider.newJobManager(config); - final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); + final MetaStorage storage = new MetaStorage(config.getStorage()); + final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, storage, environment.getValidator()); final ClusterState clusterState = new ClusterState(); final NamespaceHandler namespaceHandler = new ClusterNamespaceHandler(clusterState, config, creator); final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); @@ -35,7 +37,7 @@ public ClusterManager provideManager(ConqueryConfig config, Environment environm final DelegateManager delegate = - new DelegateManager<>(config, environment, datasetRegistry, importHandler, extension, nodeProvider, adminTasks, creator, jobManager); + new DelegateManager<>(config, environment, datasetRegistry, storage, importHandler, extension, nodeProvider, adminTasks, creator, jobManager); environment.healthChecks().register("cluster", new ClusterHealthCheck(clusterState)); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java index acf04a20fe..617f166d72 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java @@ -37,6 +37,7 @@ public DistributedNamespace createNamespace(NamespaceStorage storage, final Meta namespaceData.getJobManager(), namespaceData.getFilterSearch(), namespaceData.getIndexService(), + new ClusterEntityResolver(), namespaceData.getInjectables(), workerHandler ); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java index 36356fa9fc..18e725f1d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java @@ -31,5 +31,4 @@ public WorkerInformation getWorker(final WorkerId workerId, final DatasetId id) .flatMap(ns -> ns.getWorkers().getOptional(workerId)) .orElseThrow(() -> new NoSuchElementException("Unknown worker worker '%s' for dataset '%s'".formatted(workerId, id))); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index 762a754eb8..41d3895cf3 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.function.Supplier; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.DelegateManager; import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.mode.ManagerProvider; @@ -32,15 +33,18 @@ public LocalManagerProvider(SqlDialectFactory dialectFactory) { public DelegateManager provideManager(ConqueryConfig config, Environment environment) { - InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); - NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory); - DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + final MetaStorage storage = new MetaStorage(config.getStorage()); + final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, storage, environment.getValidator()); + final NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory); + final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + creator.init(datasetRegistry); return new DelegateManager<>( config, environment, datasetRegistry, + storage, new FailingImportHandler(), new LocalStorageListener(), EMPTY_NODE_PROVIDER, diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index b437423e30..a0f7f23c2d 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -20,6 +20,7 @@ import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; import com.bakdata.conquery.sql.execution.SqlExecutionResult; import com.bakdata.conquery.sql.execution.SqlExecutionService; @@ -48,11 +49,13 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto DSLContext dslContext = dslContextWrapper.getDslContext(); SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect()); - SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(sqlDialect)); + ResultSetProcessor resultSetProcessor = ResultSetProcessorFactory.create(config, sqlDialect); + SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor); NodeConversions nodeConversions = new NodeConversions(idColumns, sqlDialect, dslContext, databaseConfig, sqlExecutionService); SqlConverter sqlConverter = new SqlConverter(nodeConversions); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); + SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(idColumns, dslContext, sqlDialect, sqlExecutionService); return new LocalNamespace( namespaceData.getPreprocessMapper(), @@ -64,6 +67,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto namespaceData.getJobManager(), namespaceData.getFilterSearch(), namespaceData.getIndexService(), + sqlEntityResolver, namespaceData.getInjectables() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java new file mode 100644 index 0000000000..681889ec86 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/SqlEntityResolver.java @@ -0,0 +1,206 @@ +package com.bakdata.conquery.mode.local; + +import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.collectExtraData; +import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.readDates; +import static com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil.verifyOnlySingles; +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.table; +import static org.jooq.impl.DSL.val; +import static org.jooq.impl.DSL.when; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver; +import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolverUtil; +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.config.IdColumnConfig; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.ExternalId; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import com.bakdata.conquery.sql.execution.SqlExecutionService; +import com.bakdata.conquery.util.DateReader; +import com.bakdata.conquery.util.io.IdColumnUtil; +import jakarta.validation.constraints.NotEmpty; +import lombok.RequiredArgsConstructor; +import org.jooq.CommonTableExpression; +import org.jooq.DSLContext; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.Record; +import org.jooq.Record2; +import org.jooq.Record3; +import org.jooq.Select; +import org.jooq.SelectConditionStep; +import org.jooq.Table; + +@RequiredArgsConstructor +public class SqlEntityResolver implements EntityResolver { + + private static final Name IS_RESOLVED_ALIAS = name("is_resolved"); + private static final Name UNRESOLVED_CTE = name("ids_unresolved"); + public static final String ROW_INDEX = "rowIndex"; + + private final IdColumnConfig idColumns; + private final DSLContext context; + private final SqlDialect dialect; + private final SqlExecutionService executionService; + + @Override + public ResolveStatistic resolveEntities( + @NotEmpty String[][] values, + List format, + EntityIdMap mapping, + IdColumnConfig idColumnConfig, + DateReader dateReader, + boolean onlySingles + ) { + final Map resolved = new HashMap<>(); + final List unresolvedDate = new ArrayList<>(); + final List unresolvedId = new ArrayList<>(); + + // extract dates from rows + final CDateSet[] rowDates = readDates(values, format, dateReader); + + // Extract extra data from rows by Row, to be collected into by entities + // Row -> Column -> Value + final Map[] extraDataByRow = EntityResolverUtil.readExtras(values, format); + + final List> readers = IdColumnUtil.getIdReaders(format, idColumnConfig.getIdMappers()); + + // We will not be able to resolve anything... + if (readers.isEmpty()) { + return EntityResolver.ResolveStatistic.forEmptyReaders(values); + } + + // Entity -> Column -> Values + final Map>> extraDataByEntity = new HashMap<>(); + + // all IDs of this map had at least a matching reader + final Map resolvedIdsMap = resolveIds(values, readers); + + // ignore the first row, because this is the header + for (int rowNum = 1; rowNum < values.length; rowNum++) { + + final String[] row = values[rowNum]; + + final IdResolveInfo idResolveInfo = resolvedIdsMap.get(rowNum); + if (idResolveInfo == null) { + // row had no matching reader + unresolvedId.add(row); + continue; + } + + // external ID could not be resolved internally + if (!idResolveInfo.isResolved()) { + unresolvedDate.add(row); + continue; + } + + final String resolvedId = idResolveInfo.externalId(); + + if (rowDates[rowNum] == null) { + unresolvedDate.add(row); + continue; + } + + // read the dates from the row + resolved.put(resolvedId, rowDates[rowNum]); + + // Entity was resolved for row, so we collect the data. + collectExtraData(extraDataByRow, rowNum, extraDataByEntity, resolvedId); + } + + verifyOnlySingles(onlySingles, extraDataByEntity); + return new EntityResolver.ResolveStatistic(resolved, extraDataByEntity, unresolvedDate, unresolvedId); + + } + + /** + * Create a SQL query like this + * 

+	 *     {@code
+	 *      with "ids_unresolved" as (select 1   as "row",
+	 *                                      '1'  as "primary_id"
+	 *                                -- will select more ids here via union all)
+	 * 		select "row",
+	 * 		       "primary_id",
+	 * 		       case
+	 * 		           when "entities"."id" is not null then true
+	 * 		           else false
+	 * 		           end as "is_resolved"
+	 * 		from "entities"
+	 * 		         join "ids_unresolved"
+	 * 		              on "primary_id" = "entities"."id"
+	 * 		where "primary_id" = "pid"
+	 *     }
+	 *
+ *

+ * For each ID, that had a matching reader, it will return an entry in the map with row number -> IdResolveInfo. + */ + private Map resolveIds(String[][] values, List> readers) { + + CommonTableExpression unresolvedCte = createUnresolvedCte(values, readers); + + Field rowIndex = field(name(ROW_INDEX), Integer.class); + Field externalPrimaryColumn = field(name(SharedAliases.PRIMARY_COLUMN.getAlias()), String.class); + Field innerPrimaryColumn = field(name(idColumns.findPrimaryIdColumn().getField()), String.class); + Field isResolved = when(innerPrimaryColumn.isNotNull(), val(true)) + .otherwise(false) + .as(IS_RESOLVED_ALIAS); + + Table allIdsTable = table(name(idColumns.getTable())); + SelectConditionStep> resolveIdsQuery = + context.with(unresolvedCte) + .select(rowIndex, externalPrimaryColumn, isResolved) + .from(dialect.getFunctionProvider().innerJoin(allIdsTable, unresolvedCte, List.of(externalPrimaryColumn.eq(innerPrimaryColumn)))) + .where(externalPrimaryColumn.eq(innerPrimaryColumn)); + + return executionService.fetchStream(resolveIdsQuery) + .collect(Collectors.toMap( + record -> record.get(rowIndex), + record -> new IdResolveInfo(record.get(externalPrimaryColumn), record.get(isResolved)) + )); + } + + private CommonTableExpression createUnresolvedCte(String[][] values, List> readers) { + + List>> selects = new ArrayList<>(values.length); + for (int i = 1; i < values.length; i++) { + + final String[] row = values[i]; + + String resolvedId = null; + for (Function reader : readers) { + final ExternalId externalId = reader.apply(row); + resolvedId = externalId.getId(); + } + + // no matching reader found + if (resolvedId == null) { + continue; + } + + Field rowIndex = val(i).as(ROW_INDEX); + Field externalPrimaryColumn = val(resolvedId).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Select> externalIdSelect = context.select(rowIndex, externalPrimaryColumn) + // some dialects can't just select static values without FROM clause + .from(dialect.getFunctionProvider().getNoOpTable()); + + selects.add(externalIdSelect); + } + + return UNRESOLVED_CTE.as(selects.stream().reduce(Select::unionAll).orElseThrow(IllegalStateException::new)); + } + + + private record IdResolveInfo(String externalId, boolean isResolved) { + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java index c98582ea00..c71c7fde1c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.models.config; import java.net.InetAddress; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; import io.dropwizard.core.Configuration; import io.dropwizard.util.Duration; import io.dropwizard.validation.PortRange; -import jakarta.validation.Valid; -import jakarta.validation.constraints.Min; -import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; @@ -26,6 +26,7 @@ public class ClusterConfig extends Configuration { private int entityBucketSize = 1000; private Duration idleTimeOut = Duration.minutes(5); + private Duration connectRetryTimeout = Duration.seconds(30); /** * Amount of backpressure before jobs can volunteer to block to send messages to their shards. diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java index 6c74ce0cc1..a708564a40 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ColumnConfig.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.ExternalId; import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; import com.fasterxml.jackson.annotation.JsonAlias; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; @@ -34,18 +35,18 @@ @JsonIgnoreProperties(value = {"resolvable"}) // for backwards compatibility public class ColumnConfig { - public EntityIdMap.ExternalId read(String value) { + public ExternalId read(String value) { if (Strings.isNullOrEmpty(value)) { return null; } if (getLength() == -1 || getPad() == null) { - return new EntityIdMap.ExternalId(getName(), value); + return new ExternalId(getName(), value); } String padded = StringUtils.leftPad(value, getLength(), getPad()); - return new EntityIdMap.ExternalId(getName(), padded); + return new ExternalId(getName(), padded); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java index d49d6fcbb7..9c8d1202f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java @@ -44,7 +44,7 @@ public class IdColumnConfig { /** * Relevant in SQL-Mode, used as AllIdsTable for CQExternal and CQYes. */ - private String table; + private String table = "entities"; /** * List of resolvable and printable ids. @@ -56,7 +56,7 @@ public class IdColumnConfig { private List ids = List.of( ColumnConfig.builder() .name("ID") - .field("result") + .field("pid") .label(Map.of(Locale.ROOT, "result")) .primaryId(true) .print(true) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java index 8629074ccd..07f13b1082 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/PluginConfig.java @@ -3,11 +3,10 @@ import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSBase; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import freemarker.core.Environment; @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @CPSBase public interface PluginConfig { - public default void initialize(ManagerNode managerNode){}; + default void initialize(ManagerNode managerNode){} } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java index f79907fa0a..78ec8aecea 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/StoreFactory.java @@ -12,11 +12,7 @@ import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.Import; -import com.bakdata.conquery.models.datasets.PreviewConfig; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.*; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.events.Bucket; @@ -27,7 +23,6 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.annotation.JsonTypeInfo; @@ -67,9 +62,9 @@ public interface StoreFactory { SingletonStore createStructureStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper); // MetaStorage - IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper); + IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper); - IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper); + IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper); IdentifiableStore createUserStore(CentralRegistry centralRegistry, String pathName, MetaStorage storage, ObjectMapper objectMapper); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index ae74ba9f23..7529e92934 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -52,7 +52,6 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.bakdata.conquery.util.io.ConqueryMDC; @@ -191,7 +190,7 @@ public ExecutorService getReaderExecutorService() { @Override public Collection discoverNamespaceStorages() { - return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath, getValidator()), NAMESPACE_STORES); + return loadNamespacedStores("dataset_", (storePath) -> new NamespaceStorage(this, storePath), NAMESPACE_STORES); } @Override @@ -258,22 +257,22 @@ public SingletonStore createDatasetStore(String pathName, ObjectMapper @Override public IdentifiableStore createSecondaryIdDescriptionStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SECONDARY_IDS, objectMapper), centralRegistry); } @Override public IdentifiableStore createInternToExternMappingStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, INTERN_TO_EXTERN, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, INTERN_TO_EXTERN, objectMapper), centralRegistry); } @Override public IdentifiableStore createSearchIndexStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SEARCH_INDEX, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, SEARCH_INDEX, objectMapper), centralRegistry); } @Override public SingletonStore createPreviewStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, ENTITY_PREVIEW, centralRegistry.injectIntoNew(objectMapper))); + return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, ENTITY_PREVIEW, objectMapper)); } @Override @@ -283,27 +282,27 @@ public CachedStore createEntity2BucketStore(String pathName, Ob @Override public IdentifiableStore createTableStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, TABLES, objectMapper), centralRegistry); } @Override public IdentifiableStore> createConceptStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, CONCEPTS, objectMapper), centralRegistry); } @Override public IdentifiableStore createImportStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, IMPORTS, objectMapper), centralRegistry); } @Override public IdentifiableStore createCBlockStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, C_BLOCKS, objectMapper), centralRegistry); } @Override public IdentifiableStore createBucketStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS, centralRegistry.injectIntoNew(objectMapper)), centralRegistry); + return StoreMappings.identifiable(createStore(findEnvironment(pathName), validator, BUCKETS, objectMapper), centralRegistry); } @Override @@ -332,17 +331,17 @@ public SingletonStore createWorkerToBucketsStore(String path @Override public SingletonStore createStructureStore(String pathName, CentralRegistry centralRegistry, ObjectMapper objectMapper) { - return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE, centralRegistry.injectIntoNew(objectMapper))); + return StoreMappings.singleton(createStore(findEnvironment(pathName), validator, STRUCTURE, objectMapper)); } @Override - public IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, datasetRegistry.injectInto(centralRegistry.injectIntoNew(objectMapper))), centralRegistry); + public IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "executions")), validator, EXECUTIONS, objectMapper), centralRegistry); } @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) { - return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, datasetRegistry.injectInto(centralRegistry.injectIntoNew(objectMapper))), centralRegistry); + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { + return StoreMappings.identifiable(createStore(findEnvironment(resolveSubDir(pathName, "formConfigs")), validator, FORM_CONFIG, objectMapper), centralRegistry); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java index 07e3dd23b8..721fc2c920 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java @@ -1,10 +1,23 @@ package com.bakdata.conquery.models.datasets.concepts; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; -import com.bakdata.conquery.apiv1.frontend.*; +import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; +import com.bakdata.conquery.apiv1.frontend.FrontendList; +import com.bakdata.conquery.apiv1.frontend.FrontendNode; +import com.bakdata.conquery.apiv1.frontend.FrontendRoot; +import com.bakdata.conquery.apiv1.frontend.FrontendSecondaryId; +import com.bakdata.conquery.apiv1.frontend.FrontendSelect; +import com.bakdata.conquery.apiv1.frontend.FrontendTable; +import com.bakdata.conquery.apiv1.frontend.FrontendValidityDate; +import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; @@ -159,7 +172,7 @@ public FrontendSelect createSelect(Select select) { .id(select.getId()) .label(select.getLabel()) .description(select.getDescription()) - .resultType(select.getResultType()) + .resultType(select.getResultType().typeInfo()) .isDefault(select.isDefault()) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java index 4cd7c460a1..50a1316eff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/DistinctSelect.java @@ -9,6 +9,8 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.AllValuesAggregator; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.conversion.model.select.DistinctSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; @CPSType(id = "DISTINCT", base = Select.class) @@ -29,4 +31,9 @@ public Aggregator createAggregator() { public ResultType getResultType() { return new ResultType.ListT(super.getResultType()); } + + @Override + public SelectConverter createConverter() { + return new DistinctSelectConverter(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java index 2d1ff6246c..05b694ebb9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java @@ -12,6 +12,8 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DateUnionAggregator; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.conversion.model.select.DateUnionSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Getter; @@ -53,4 +55,9 @@ public Aggregator createAggregator() { public ResultType getResultType() { return new ResultType.ListT<>(ResultType.DateRangeT.INSTANCE); } + + @Override + public SelectConverter createConverter() { + return new DateUnionSelectConverter(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java index 8e32141534..e82f3311c3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java @@ -12,6 +12,8 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator; import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.conversion.model.select.DurationSumSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Getter; @@ -45,7 +47,6 @@ public List getRequiredColumns() { @Override public Aggregator createAggregator() { - // TODO fix this for 2 columns return new DurationSumAggregator(getColumn()); } @@ -53,4 +54,9 @@ public Aggregator createAggregator() { public ResultType getResultType() { return ResultType.IntegerT.INSTANCE; } + + @Override + public SelectConverter createConverter() { + return new DurationSumSelectConverter(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java index c23469d152..5875e89264 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/ConceptTreeCache.java @@ -1,27 +1,29 @@ package com.bakdata.conquery.models.datasets.concepts.tree; -import java.util.HashMap; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.util.CalculatedValue; import com.fasterxml.jackson.annotation.JsonIgnore; -import lombok.Getter; +import lombok.Data; +import lombok.RequiredArgsConstructor; /** * Cache for ConceptTree index searches. */ +@RequiredArgsConstructor +@Data public class ConceptTreeCache { /** * Statistics for Cache. */ - @Getter private int hits; /** * Statistics for Cache. */ - @Getter private int misses; @JsonIgnore @@ -29,15 +31,12 @@ public class ConceptTreeCache { /** * Store of all cached values. + * + * @implNote ConcurrentHashMap does not allow null values, but we want to have null values in the map. So we wrap the values in Optional. */ - @JsonIgnore - private final Map cached; + private final Map> cached = new ConcurrentHashMap<>();; - public ConceptTreeCache(TreeConcept treeConcept) { - this.treeConcept = treeConcept; - cached = new HashMap<>(); - } /** * If id is already in cache, use that. If not calculate it by querying treeConcept. If rowMap was not used to query, cache the response. @@ -48,7 +47,7 @@ public ConceptTreeChild findMostSpecificChild(String value, CalculatedValue streamValues() { return Arrays.stream(values).filter(Objects::nonNull); } + public void setParent(Bucket bucket) { + // not used + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java index 93db593245..1558fa67a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java @@ -31,9 +31,7 @@ public interface ColumnStore { * @implNote BackReference set here because Jackson does not support for fields in interfaces and abstract classes see also https://github.com/FasterXML/jackson-databind/issues/3304 */ @JsonBackReference - default void setParent(Bucket bucket) { - - } + void setParent(Bucket bucket); /** @@ -92,13 +90,6 @@ default long estimateMemoryConsumptionBytes() { @JsonIgnore int getLines(); - /** - * Bytes required to store auxilary data. - */ - default long estimateTypeSizeBytes() { - return 0; - } - /** * Create an empty store that's only a description of the transformation. */ @@ -115,6 +106,7 @@ static T emptyCopy(T store) { */ T select(int[] starts, int[] lengths); + @JsonIgnore void setNull(int event); diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java index 9c72287b4e..c116f9142b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/CompoundDateRangeStore.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.DateRangeStore; import com.bakdata.conquery.models.events.stores.root.DateStore; +import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -89,6 +90,7 @@ public DateStore getEndStore() { } @Override + @JsonBackReference public void setParent(@NonNull Bucket bucket) { parent = bucket; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java index fbaeadc9a8..dcef9bc254 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/DirectDateRangeStore.java @@ -2,6 +2,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.primitive.IntegerDateStore; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.DateRangeStore; @@ -29,6 +30,10 @@ public DirectDateRangeStore(DateStore minStore, DateStore maxStore) { this.maxStore = maxStore; } + public void setParent(Bucket bucket) { + // not used + } + @Override public int getLines() { // they can be unaligned, if one of them is empty. diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java index c3d38474d8..c66faaae0e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/MoneyIntStore.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.events.stores.specific; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.IntegerStore; import com.bakdata.conquery.models.events.stores.root.MoneyStore; @@ -61,4 +62,8 @@ public void setNull(int event) { public final boolean has(int event) { return numberType.has(event); } + + public void setParent(Bucket bucket) { + // not used + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java index 1173ac01bb..d441f771fd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/QuarterDateRangeStore.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.common.QuarterUtils; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.DateRangeStore; import com.bakdata.conquery.models.events.stores.root.IntegerStore; @@ -75,4 +76,8 @@ public CDateRange getDateRange(int event) { return CDateRange.of(begin, CDate.ofLocalDate(end)); } + + public void setParent(Bucket bucket) { + // not used + } } \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java index 559457c0d8..ab26940bb8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/RebasingIntegerStore.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.events.stores.specific; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.IntegerStore; import jakarta.validation.constraints.NotNull; @@ -68,4 +69,8 @@ public boolean has(int event) { public void setNull(int event) { store.setNull(event); } + + public void setParent(Bucket bucket) { + // not used + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java index a7f368ba6f..802d1d6b92 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/specific/ScaledDecimalStore.java @@ -4,6 +4,7 @@ import java.math.BigInteger; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.DecimalStore; import com.bakdata.conquery.models.events.stores.root.IntegerStore; @@ -75,4 +76,8 @@ public static BigDecimal scale(int scale, long value) { public boolean has(int event) { return subType.has(event); } + + public void setParent(Bucket bucket) { + // not used + } } \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java index f03b7a69b4..ae74f0de63 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/CentralRegistry.java @@ -5,12 +5,8 @@ import java.util.concurrent.ConcurrentMap; import java.util.function.Function; -import com.bakdata.conquery.io.jackson.Injectable; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.error.ConqueryError.ExecutionCreationResolveError; import com.bakdata.conquery.models.identifiable.ids.Id; -import com.bakdata.conquery.models.worker.IdResolveContext; -import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonMappingException; import lombok.NoArgsConstructor; @@ -22,7 +18,7 @@ @SuppressWarnings({"rawtypes", "unchecked"}) @NoArgsConstructor @ToString(of = "map") -public class CentralRegistry implements Injectable { +public class CentralRegistry { private final IdMap map = new IdMap<>(); private final ConcurrentMap, Function> cacheables = new ConcurrentHashMap<>(); @@ -70,24 +66,8 @@ public synchronized void remove(Identifiable ident) { map.remove(id); } - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - return values.add(CentralRegistry.class, this) - // Possibly overriding mapping for DatasetRegistry - .add(IdResolveContext.class, new SingletonNamespaceCollection(this)); - } - public static CentralRegistry get(DeserializationContext ctxt) throws JsonMappingException { - CentralRegistry result = (CentralRegistry) ctxt.findInjectableValue(CentralRegistry.class.getName(), null, null); - if (result != null) { - return result; - } - - IdResolveContext alternative = (IdResolveContext) ctxt.findInjectableValue(IdResolveContext.class.getName(), null, null); - if (alternative == null) { - return null; - } - return alternative.getMetaRegistry(); + return (CentralRegistry) ctxt.findInjectableValue(CentralRegistry.class.getName(), null, null); } public void clear() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java index 936a4c313f..d19838ffc8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/ids/NamespacedId.java @@ -1,12 +1,13 @@ package com.bakdata.conquery.models.identifiable.ids; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.worker.IdResolveContext; import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.lang3.StringUtils; /** - * Marker interface for {@link Id}s that are loaded via Namespaced CentralRegistry (see {@link com.bakdata.conquery.models.worker.IdResolveContext#findRegistry(DatasetId)}, as opposed to MetaRegistry (see {@link IdResolveContext#getMetaRegistry()}). + * Marker interface for {@link Id}s that are loaded via Namespaced CentralRegistry + * (see {@link com.bakdata.conquery.models.worker.IdResolveContext#findRegistry(DatasetId)}, + * as opposed to Registry in the {@link com.bakdata.conquery.io.storage.MetaStorage} */ public interface NamespacedId { diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java index 83723c0b4b..52b56dd00c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/EntityIdMap.java @@ -172,11 +172,4 @@ public void addInputMapping(String csvEntityId, ExternalId externalEntityId) { } } - @Data - @RequiredArgsConstructor(onConstructor_ = @JsonCreator) - public static class ExternalId { - private final String type; - private final String id; - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java new file mode 100644 index 0000000000..b1f4683797 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/identifiable/mapping/ExternalId.java @@ -0,0 +1,12 @@ +package com.bakdata.conquery.models.identifiable.mapping; + +import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.Data; +import lombok.RequiredArgsConstructor; + +@Data +@RequiredArgsConstructor(onConstructor_ = @JsonCreator) +public class ExternalId { + private final String type; + private final String id; +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java index 0239857c86..692fa2f986 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java @@ -3,6 +3,7 @@ import java.util.Set; import java.util.stream.Collectors; +import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.identifiable.ids.Id; @@ -13,16 +14,12 @@ import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.worker.DistributedNamespace; import com.google.common.collect.Sets; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.NonNull; -import lombok.Setter; +import lombok.*; import lombok.extern.slf4j.Slf4j; /** * Compares the the ids of imports and buckets of a {@link com.bakdata.conquery.models.worker.Worker} with the those - * the {@link com.bakdata.conquery.commands.ManagerNode} assumed the Worker to have and reports an error if there are + * the {@link ManagerNode} assumed the Worker to have and reports an error if there are * inconsistencies. */ @CPSType(id="REPORT_CONSISTENCY", base= NamespacedMessage.class) diff --git a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java index f4ede6725c..52ccbfbea9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java @@ -11,15 +11,12 @@ import c10n.C10N; import com.bakdata.conquery.internationalization.Results; -import com.bakdata.conquery.io.cps.CPSBase; -import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.base.Preconditions; import lombok.AccessLevel; import lombok.EqualsAndHashCode; @@ -29,30 +26,9 @@ import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; -@JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") -@CPSBase @Slf4j public abstract class ResultType { - public String printNullable(PrintSettings cfg, Object f) { - if (f == null) { - return ""; - } - return print(cfg, f); - } - - protected String print(PrintSettings cfg, @NonNull Object f) { - return f.toString(); - } - - public abstract String typeInfo(); - - public abstract T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException; - - protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { - throw new UnsupportedOperationException("ResultType list of type %s not supported for now.".formatted(getClass().getSimpleName())); - } - public static ResultType resolveResultType(MajorTypeId majorTypeId) { return switch (majorTypeId) { case STRING -> StringT.INSTANCE; @@ -65,21 +41,27 @@ public static ResultType resolveResultType(MajorTypeId majorTypeId) { }; } - abstract static class PrimitiveResultType extends ResultType { - @Override - public String typeInfo() { - return this.getClass().getAnnotation(CPSType.class).id(); + public String printNullable(PrintSettings cfg, Object f) { + if (f == null) { + return ""; } + return print(cfg, f); + } - @Override - public String toString() { - return typeInfo(); - } + protected abstract String print(PrintSettings cfg, @NonNull Object f); + + public String toString() { + return typeInfo(); } - @CPSType(id = "BOOLEAN", base = ResultType.class) + public abstract String typeInfo(); + + public abstract T getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException; + + protected abstract List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException; + @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class BooleanT extends PrimitiveResultType { + public static class BooleanT extends ResultType { @Getter(onMethod_ = @JsonCreator) public static final BooleanT INSTANCE = new BooleanT(); @@ -95,16 +77,25 @@ public String print(PrintSettings cfg, Object f) { return (Boolean) f ? "1" : "0"; } + @Override + public String typeInfo() { + return "BOOLEAN"; + } + @Override public Boolean getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getBoolean(resultSet, columnIndex); } + + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getBooleanList(resultSet, columnIndex); + } } - @CPSType(id = "INTEGER", base = ResultType.class) @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class IntegerT extends PrimitiveResultType { + public static class IntegerT extends ResultType { @Getter(onMethod_ = @JsonCreator) public static final IntegerT INSTANCE = new IntegerT(); @@ -116,15 +107,24 @@ public String print(PrintSettings cfg, Object f) { return f.toString(); } + @Override + public String typeInfo() { + return "INTEGER"; + } + @Override public Integer getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getInteger(resultSet, columnIndex); } + + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getIntegerList(resultSet, columnIndex); + } } - @CPSType(id = "NUMERIC", base = ResultType.class) @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class NumericT extends PrimitiveResultType { + public static class NumericT extends ResultType { @Getter(onMethod_ = @JsonCreator) public static final NumericT INSTANCE = new NumericT(); @@ -136,15 +136,24 @@ public String print(PrintSettings cfg, Object f) { return f.toString(); } + @Override + public String typeInfo() { + return "NUMERIC"; + } + @Override public Double getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDouble(resultSet, columnIndex); } + + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getDoubleList(resultSet, columnIndex); + } } - @CPSType(id = "DATE", base = ResultType.class) @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class DateT extends PrimitiveResultType { + public static class DateT extends ResultType { @Getter(onMethod_ = @JsonCreator) public static final DateT INSTANCE = new DateT(); @@ -157,13 +166,23 @@ public String print(PrintSettings cfg, @NonNull Object f) { return print(number, cfg.getDateFormatter()); } + public static String print(Number num, DateTimeFormatter formatter) { + return formatter.format(LocalDate.ofEpochDay(num.intValue())); + } + + @Override + public String typeInfo() { + return "DATE"; + } + @Override public Number getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDate(resultSet, columnIndex); } - public static String print(Number num, DateTimeFormatter formatter) { - return formatter.format(LocalDate.ofEpochDay(num.intValue())); + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getDateList(resultSet, columnIndex); } } @@ -171,39 +190,43 @@ public static String print(Number num, DateTimeFormatter formatter) { * A DateRange is provided by in a query result as two ints in a list, both standing for an epoch day (see {@link LocalDate#toEpochDay()}). * The first int describes the included lower bound of the range. The second int descibes the included upper bound. */ - @CPSType(id = "DATE_RANGE", base = ResultType.class) @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class DateRangeT extends PrimitiveResultType> { + public static class DateRangeT extends ResultType> { @Getter(onMethod_ = @JsonCreator) public static final DateRangeT INSTANCE = new DateRangeT(); @Override public String print(PrintSettings cfg, @NonNull Object f) { - if (!(f instanceof List)) { - throw new IllegalStateException(String.format("Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f)); - } - List list = (List) f; - if (list.size() != 2) { - throw new IllegalStateException("Expected a list with 2 elements, one min, one max. The list was: " + list); - } + Preconditions.checkArgument(f instanceof List, "Expected a List got %s (Type: %s, as string: %s)", f, f.getClass().getName(), f); + + final List list = (List) f; + + Preconditions.checkArgument(list.size() == 2, "Expected a list with 2 elements: one min, one max. The list was: %s", list); + final DateTimeFormatter dateFormat = cfg.getDateFormatter(); final Integer min = (Integer) list.get(0); final Integer max = (Integer) list.get(1); + if (min == null || max == null) { log.warn("Encountered incomplete range, treating it as an open range. Either min or max was null: {}", list); } // Compute minString first because we need it either way - String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : ResultType.DateT.print(min, dateFormat); + final String minString = min == null || min == CDateRange.NEGATIVE_INFINITY ? "-∞" : ResultType.DateT.print(min, dateFormat); if (cfg.isPrettyPrint() && min != null && min.equals(max)) { // If the min and max are the same we print it like a singe date, not a range (only in pretty printing) return minString; } - String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : ResultType.DateT.print(max, dateFormat); + final String maxString = max == null || max == CDateRange.POSITIVE_INFINITY ? "+∞" : ResultType.DateT.print(max, dateFormat); return minString + cfg.getDateRangeSeparator() + maxString; } + @Override + public String typeInfo() { + return "DATE_RANGE"; + } + @Override public List getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getDateRange(resultSet, columnIndex); @@ -215,9 +238,8 @@ public List> getFromResultSetAsList(ResultSet resultSet, int colum } } - @CPSType(id = "STRING", base = ResultType.class) @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class StringT extends PrimitiveResultType { + public static class StringT extends ResultType { @Getter(onMethod_ = @JsonCreator) public static final StringT INSTANCE = new StringT(); @@ -235,9 +257,14 @@ public StringT(BiFunction valueMapper) { @Override protected String print(PrintSettings cfg, @NonNull Object f) { if (valueMapper == null) { - return super.print(cfg, f); + return f.toString(); } - return super.print(cfg, valueMapper.apply(f, cfg)); + return valueMapper.apply(f, cfg); + } + + @Override + public String typeInfo() { + return "STRING"; } @Override @@ -251,9 +278,8 @@ protected List getFromResultSetAsList(ResultSet resultSet, int columnInd } } - @CPSType(id = "MONEY", base = ResultType.class) @NoArgsConstructor(access = AccessLevel.PRIVATE) - public static class MoneyT extends PrimitiveResultType { + public static class MoneyT extends ResultType { @Getter(onMethod_ = @JsonCreator) public static final MoneyT INSTANCE = new MoneyT(); @@ -266,19 +292,27 @@ public String print(PrintSettings cfg, Object f) { return IntegerT.INSTANCE.print(cfg, f); } + @NotNull + public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) { + return new BigDecimal(f.longValue()).movePointLeft(cfg.getCurrency().getDefaultFractionDigits()); + } + + @Override + public String typeInfo() { + return "MONEY"; + } + @Override public BigDecimal getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getMoney(resultSet, columnIndex); } - - @NotNull - public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) { - return new BigDecimal(f.longValue()).movePointLeft(cfg.getCurrency().getDefaultFractionDigits()); + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getMoneyList(resultSet, columnIndex); } } - @CPSType(id = "LIST", base = ResultType.class) @Getter @EqualsAndHashCode(callSuper = false) public static class ListT extends ResultType> { @@ -299,7 +333,7 @@ public String print(PrintSettings cfg, @NonNull Object f) { } // Not sure if this escaping is enough final LocaleConfig.ListFormat listFormat = cfg.getListFormat(); - StringJoiner joiner = listFormat.createListJoiner(); + final StringJoiner joiner = listFormat.createListJoiner(); for (Object obj : (List) f) { joiner.add(listFormat.escapeListElement(elementType.print(cfg, obj))); } @@ -308,21 +342,17 @@ public String print(PrintSettings cfg, @NonNull Object f) { @Override public String typeInfo() { - return this.getClass().getAnnotation(CPSType.class).id() + "[" + elementType.typeInfo() + "]"; + return "LIST[" + elementType.typeInfo() + "]"; } @Override public List getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { - if (elementType.getClass() == DateRangeT.class || elementType.getClass() == StringT.class) { - return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor); - } - // TODO handle all other list types properly - throw new UnsupportedOperationException("Other result type lists not supported for now."); + return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor); } @Override - public String toString() { - return typeInfo(); + protected List> getFromResultSetAsList(final ResultSet resultSet, final int columnIndex, final ResultSetProcessor resultSetProcessor) { + throw new UnsupportedOperationException("Nested lists not supported in SQL mode"); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java index ffcf3e5e04..a52e35667d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.worker; +import java.util.Collection; import java.util.List; import java.util.function.Predicate; @@ -10,19 +11,21 @@ @Data public class ClusterHealthCheck extends HealthCheck { + public static final String HEALTHY_MESSAGE_FMT = "All %d known shards are connected."; private final ClusterState clusterState; @Override protected Result check() throws Exception { + Collection knownShards = clusterState.getShardNodes().values(); final List disconnectedWorkers = - clusterState.getShardNodes().values().stream() - .filter(Predicate.not(ShardNodeInformation::isConnected)) - .map(ShardNodeInformation::toString) - .toList(); + knownShards.stream() + .filter(Predicate.not(ShardNodeInformation::isConnected)) + .map(ShardNodeInformation::toString) + .toList(); if (disconnectedWorkers.isEmpty()){ - return Result.healthy("All known shards are connected."); + return Result.healthy(HEALTHY_MESSAGE_FMT, knownShards.size()); } return Result.unhealthy("The shard(s) %s are no longer connected.".formatted(String.join(",", disconnectedWorkers))); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index afc490de52..64606760ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -10,6 +10,7 @@ import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; +import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; @@ -27,10 +28,8 @@ import com.fasterxml.jackson.annotation.JsonIgnoreType; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.cache.CacheStats; -import jakarta.validation.Validator; import lombok.Getter; import lombok.RequiredArgsConstructor; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -47,30 +46,27 @@ public class DatasetRegistry extends IdResolveContext imple private final InternalObjectMapperCreator internalObjectMapperCreator; - @Getter - @Setter - private MetaStorage metaStorage; - private final NamespaceHandler namespaceHandler; private final IndexService indexService; - public N createNamespace(Dataset dataset, Validator validator) throws IOException { + public N createNamespace(Dataset dataset, MetaStorage metaStorage) throws IOException { // Prepare empty storage - NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName(), validator); + NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName()); final ObjectMapper persistenceMapper = internalObjectMapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); - datasetStorage.openStores(persistenceMapper); + // Each store injects its own IdResolveCtx so each needs its own mapper + datasetStorage.openStores(Jackson.copyMapperAndInjectables((persistenceMapper))); datasetStorage.loadData(); datasetStorage.updateDataset(dataset); datasetStorage.updateIdMapping(new EntityIdMap()); datasetStorage.setPreviewConfig(new PreviewConfig()); datasetStorage.close(); - return createNamespace(datasetStorage); + return createNamespace(datasetStorage, metaStorage); } - public N createNamespace(NamespaceStorage datasetStorage) { + public N createNamespace(NamespaceStorage datasetStorage, MetaStorage metaStorage) { final N namespace = namespaceHandler.createNamespace(datasetStorage, metaStorage, indexService); add(namespace); return namespace; @@ -88,7 +84,6 @@ public void removeNamespace(DatasetId id) { N removed = datasets.remove(id); if (removed != null) { - metaStorage.getCentralRegistry().remove(removed.getDataset()); namespaceHandler.removeNamespace(id, removed); removed.remove(); } @@ -103,12 +98,6 @@ public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementExcep return datasets.get(dataset).getStorage().getCentralRegistry(); } - @Override - public CentralRegistry getMetaRegistry() { - return metaStorage.getCentralRegistry(); - } - - public List getAllDatasets() { return datasets.values().stream().map(Namespace::getStorage).map(NamespaceStorage::getDataset).collect(Collectors.toList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java index 19ecf66d1e..76a5d4664f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.cluster.ClusterEntityResolver; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.Concept; @@ -36,7 +37,6 @@ public class DistributedNamespace extends Namespace { private final WorkerHandler workerHandler; private final DistributedExecutionManager executionManager; - public DistributedNamespace( ObjectMapper preprocessMapper, ObjectMapper communicationMapper, @@ -45,10 +45,11 @@ public DistributedNamespace( JobManager jobManager, FilterSearch filterSearch, IndexService indexService, + ClusterEntityResolver clusterEntityResolver, List injectables, WorkerHandler workerHandler ) { - super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables); + super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, clusterEntityResolver, injectables); this.executionManager = executionManager; this.workerHandler = workerHandler; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java b/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java index 04fec43874..d65f7850c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/IdResolveContext.java @@ -10,7 +10,6 @@ import com.bakdata.conquery.models.identifiable.ids.Id; import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonMappingException; import io.dropwizard.jackson.Jackson; @@ -36,8 +35,6 @@ public MutableInjectableValues inject(MutableInjectableValues values) { } public abstract CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException; - @JsonIgnore - public abstract CentralRegistry getMetaRegistry(); public & NamespacedId, T extends Identifiable> T resolve(ID id) { return findRegistry(id.getDataset()).resolve(id); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java index b21aabf5bd..99fb6340a6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.local.SqlEntityResolver; import com.bakdata.conquery.mode.local.SqlStorageHandler; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.index.IndexService; @@ -37,9 +38,10 @@ public LocalNamespace( JobManager jobManager, FilterSearch filterSearch, IndexService indexService, + SqlEntityResolver sqlEntityResolver, List injectables ) { - super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables); + super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, sqlEntityResolver, injectables); this.dslContextWrapper = dslContextWrapper; this.storageHandler = storageHandler; } @@ -55,7 +57,8 @@ void registerColumnValuesInSearch(Set columns) { try { final Stream stringStream = storageHandler.lookupColumnValues(getStorage(), column); getFilterSearch().registerValues(column, stringStream.collect(Collectors.toSet())); - }catch (Exception e) { + } + catch (Exception e) { log.error("Problem collecting column values for {}", column, e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java index 191265b8b2..dcee848050 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java @@ -6,6 +6,7 @@ import java.util.NoSuchElementException; import java.util.Set; +import com.bakdata.conquery.apiv1.query.concept.specific.external.EntityResolver; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.datasets.Column; @@ -49,6 +50,8 @@ public abstract class Namespace extends IdResolveContext { private final IndexService indexService; + private final EntityResolver entityResolver; + // Jackson's injectables that are available when deserializing requests (see PathParamInjector) or items from the storage private final List injectables; @@ -122,12 +125,6 @@ public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementExcep return storage.getCentralRegistry(); } - @Override - public CentralRegistry getMetaRegistry() { - throw new UnsupportedOperationException(); - } - - /** * Issues a job that initializes the search that is used by the frontend for recommendations in the filter interface of a concept. */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java b/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java index 78bc0c5b52..4f1d3799b3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/SingletonNamespaceCollection.java @@ -8,21 +8,11 @@ @RequiredArgsConstructor public class SingletonNamespaceCollection extends IdResolveContext { - public SingletonNamespaceCollection(CentralRegistry registry) { - this(registry, null); - } - @NonNull private final CentralRegistry registry; - private final CentralRegistry metaRegistry; @Override public CentralRegistry findRegistry(DatasetId dataset) { return registry; } - - @Override - public CentralRegistry getMetaRegistry() { - return metaRegistry; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java index 853830d3de..096554e5c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Worker.java @@ -125,11 +125,6 @@ public MessageToManagerNode transform(NamespaceMessage message) { return new ForwardToNamespace(getInfo().getDataset(), message); } - public ObjectMapper inject(ObjectMapper binaryMapper) { - return new SingletonNamespaceCollection(storage.getCentralRegistry()) - .injectIntoNew(binaryMapper); - } - @Override public void close() { // We do not close the executorService here because it does not belong to this class diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java index 99ddb7e633..cb7f603021 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java @@ -8,7 +8,9 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Supplier; +import jakarta.validation.Validator; +import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.storage.WorkerStorage; import com.bakdata.conquery.models.config.StoreFactory; import com.bakdata.conquery.models.config.ThreadPoolDefinition; @@ -18,14 +20,13 @@ import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; -import jakarta.validation.Validator; import lombok.Getter; import lombok.NonNull; import lombok.Setter; import lombok.extern.slf4j.Slf4j; /** - * {@link com.bakdata.conquery.commands.ShardNode} container of {@link Worker}. + * {@link ShardNode} container of {@link Worker}. * * Each Shard contains one {@link Worker} per {@link Dataset}. */ @@ -119,11 +120,6 @@ public CentralRegistry findRegistry(DatasetId dataset) { return dataset2Worker.get(dataset).getStorage().getCentralRegistry(); } - @Override - public CentralRegistry getMetaRegistry() { - return null; // Workers simply have no MetaRegistry. - } - public void removeWorkerFor(DatasetId dataset) { final Worker worker = dataset2Worker.get(dataset); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java index 40a03b87c4..a41e6e8600 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java @@ -4,6 +4,7 @@ import java.nio.charset.StandardCharsets; import java.util.Collections; +import jakarta.validation.Validator; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.freemarker.Freemarker; @@ -18,35 +19,14 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.resources.admin.rest.AdminConceptsResource; -import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; -import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource; -import com.bakdata.conquery.resources.admin.rest.AdminDatasetsResource; -import com.bakdata.conquery.resources.admin.rest.AdminProcessor; -import com.bakdata.conquery.resources.admin.rest.AdminResource; -import com.bakdata.conquery.resources.admin.rest.AdminTablesResource; -import com.bakdata.conquery.resources.admin.rest.AuthOverviewResource; -import com.bakdata.conquery.resources.admin.rest.GroupResource; -import com.bakdata.conquery.resources.admin.rest.PermissionResource; -import com.bakdata.conquery.resources.admin.rest.RoleResource; -import com.bakdata.conquery.resources.admin.rest.UIProcessor; -import com.bakdata.conquery.resources.admin.rest.UserResource; -import com.bakdata.conquery.resources.admin.ui.AdminUIResource; -import com.bakdata.conquery.resources.admin.ui.AuthOverviewUIResource; -import com.bakdata.conquery.resources.admin.ui.ConceptsUIResource; -import com.bakdata.conquery.resources.admin.ui.DatasetsUIResource; -import com.bakdata.conquery.resources.admin.ui.GroupUIResource; -import com.bakdata.conquery.resources.admin.ui.IndexServiceUIResource; -import com.bakdata.conquery.resources.admin.ui.RoleUIResource; -import com.bakdata.conquery.resources.admin.ui.TablesUIResource; -import com.bakdata.conquery.resources.admin.ui.UserUIResource; +import com.bakdata.conquery.resources.admin.rest.*; +import com.bakdata.conquery.resources.admin.ui.*; import com.bakdata.conquery.resources.admin.ui.model.ConnectorUIResource; import io.dropwizard.core.setup.AdminEnvironment; import io.dropwizard.jersey.DropwizardResourceConfig; import io.dropwizard.jersey.jackson.JacksonMessageBodyProvider; import io.dropwizard.servlets.assets.AssetServlet; import io.dropwizard.views.common.ViewMessageBodyWriter; -import jakarta.validation.Validator; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.glassfish.hk2.utilities.binding.AbstractBinder; @@ -89,7 +69,7 @@ public AdminServlet(ManagerNode manager) { adminProcessor = new AdminProcessor( manager, manager.getConfig(), - manager.getStorage(), + manager.getMetaStorage(), manager.getDatasetRegistry(), manager.getJobManager(), manager.getMaintenanceService(), @@ -101,6 +81,7 @@ public AdminServlet(ManagerNode manager) { manager.getConfig(), manager.getValidator(), manager.getDatasetRegistry(), + manager.getMetaStorage(), manager.getJobManager(), manager.getImportHandler(), manager.getStorageListener() @@ -110,7 +91,7 @@ public AdminServlet(ManagerNode manager) { @Override protected void configure() { bind(manager.getDatasetRegistry()).to(DatasetRegistry.class); - bind(manager.getStorage()).to(MetaStorage.class); + bind(manager.getMetaStorage()).to(MetaStorage.class); bind(manager.getValidator()).to(Validator.class); bind(manager.getJobManager()).to(JobManager.class); bind(manager.getConfig()).to(ConqueryConfig.class); @@ -134,7 +115,7 @@ protected void configure() { bind(adminProcessor).to(AdminProcessor.class); bindAsContract(UIProcessor.class); bind(manager.getDatasetRegistry()).to(DatasetRegistry.class); - bind(manager.getStorage()).to(MetaStorage.class); + bind(manager.getMetaStorage()).to(MetaStorage.class); bind(manager.getConfig()).to(ConqueryConfig.class); } }) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index a8adad0a1e..8248842abc 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -9,6 +9,7 @@ import java.util.Set; import java.util.stream.Collectors; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.ImportHandler; import com.bakdata.conquery.mode.StorageListener; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -59,6 +60,7 @@ public class AdminDatasetProcessor { private final ConqueryConfig config; private final Validator validator; private final DatasetRegistry datasetRegistry; + private final MetaStorage metaStorage; private final JobManager jobManager; private final ImportHandler importHandler; private final StorageListener storageListener; @@ -74,7 +76,7 @@ public synchronized Dataset addDataset(Dataset dataset) throws IOException { throw new WebApplicationException("Dataset already exists", Response.Status.CONFLICT); } - return datasetRegistry.createNamespace(dataset, getValidator()).getDataset(); + return datasetRegistry.createNamespace(dataset, metaStorage).getDataset(); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java index b7ea1916d3..a1b2acf495 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java @@ -5,6 +5,7 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import org.jooq.DSLContext; +import org.jooq.conf.RenderOptionalKeyword; import org.jooq.conf.RenderQuotedNames; import org.jooq.conf.Settings; import org.jooq.impl.DSL; @@ -24,7 +25,9 @@ public static DSLContextWrapper create(DatabaseConfig config, SqlConnectorConfig .withRenderFormatted(connectorConfig.isWithPrettyPrinting()) // enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName() // to prevent any lowercase/uppercase SQL dialect specific identifier naming issues - .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED); + .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED) + // always render "as" keyword for field aliases + .withRenderOptionalAsKeywordForFieldAliases(RenderOptionalKeyword.ON); DSLContext dslContext = DSL.using( hikariDataSource, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java index a1a3186946..eaa8dfaa86 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQYesConverter.java @@ -22,11 +22,7 @@ public Class getConversionClass() { @Override public ConversionContext convert(CQYes cqYes, ConversionContext context) { - ColumnConfig primaryColumnConfig = context.getIdColumns().getIds().stream() - .filter(ColumnConfig::isPrimaryId) - .findFirst() - .orElseThrow(() -> new IllegalStateException("SQL mode requires a primary key column in IdColumnConfig")); - + ColumnConfig primaryColumnConfig = context.getIdColumns().findPrimaryIdColumn(); Field primaryColumn = DSL.field(DSL.name(primaryColumnConfig.getField())); SqlIdColumns ids = new SqlIdColumns(primaryColumn); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index 0ada55f076..85179b7043 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.List; -import java.util.function.Predicate; import java.util.stream.Collectors; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -39,8 +38,13 @@ private Selects collectSelects(CQTableContext tableContext) { List forAggregationFilterStep = tableContext.allSqlSelects().stream() .flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()) - .filter(Predicate.not(SqlSelect::isUniversal)) - .map(sqlSelect -> sqlSelect.qualify(previous.getCteName())) + .map(sqlSelect -> { + // universal selects like an ExistsSelect have no predecessor in preceding CTE + if (sqlSelect.isUniversal()) { + return sqlSelect; + } + return sqlSelect.qualify(previous.getCteName()); + }) .collect(Collectors.toList()); return Selects.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 6cac5ba991..a8fa813a3f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -11,11 +11,9 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.SharedAliases; @@ -99,7 +97,7 @@ private static QueryStep finishConceptConversion(QueryStep predecessor, CQConcep Optional validityDate = predecessorSelects.getValidityDate(); SqlIdColumns ids = predecessorSelects.getIds(); - SelectContext selectContext = SelectContext.create(cqConcept, ids, validityDate, universalTables, context); + SelectContext selectContext = SelectContext.create(cqConcept, ids, validityDate, universalTables, context); List converted = cqConcept.getSelects().stream() .map(select -> select.createConverter().conceptSelect(select, selectContext)) .toList(); @@ -153,7 +151,7 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce getDateRestriction(conversionContext, tablesValidityDate).ifPresent(allSqlFiltersForTable::add); // convert selects - SelectContext selectContext = SelectContext.create(cqTable, ids, tablesValidityDate, connectorTables, conversionContext); + SelectContext selectContext = SelectContext.create(cqTable, ids, tablesValidityDate, connectorTables, conversionContext); List allSelectsForTable = new ArrayList<>(); ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext); allSelectsForTable.add(conceptColumnSelect); @@ -277,7 +275,7 @@ private static Optional getDateRestriction(ConversionContext context )); } - private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { + private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { return cqConcept.getSelects().stream() .filter(select -> select instanceof ConceptColumnSelect) .findFirst() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java index ce9c58994f..2fda1a3295 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptSqlTables.java @@ -4,22 +4,21 @@ import java.util.Map; import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.Getter; @Getter -public class ConceptSqlTables extends ConnectorSqlTables { +public class ConceptSqlTables extends SqlTables { private final List connectorTables; public ConceptSqlTables( - String conceptConnectorLabel, String rootTable, Map cteNameMap, Map predecessorMap, - boolean containsIntervalPacking, List connectorTables ) { - super(conceptConnectorLabel, rootTable, cteNameMap, predecessorMap, containsIntervalPacking); + super(rootTable, cteNameMap, predecessorMap); this.connectorTables = connectorTables; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java index 6f226a4119..fe9443edc1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorSqlTables.java @@ -2,6 +2,7 @@ import java.util.Map; +import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; @@ -20,7 +21,13 @@ public class ConnectorSqlTables extends SqlTables { */ private final boolean withIntervalPacking; + /** + * Corresponding {@link Connector} of these {@link SqlTables}. + */ + private final Connector connector; + public ConnectorSqlTables( + Connector connector, String conceptConnectorLabel, String rootTable, Map cteNameMap, @@ -28,6 +35,7 @@ public ConnectorSqlTables( boolean containsIntervalPacking ) { super(rootTable, cteNameMap, predecessorMap); + this.connector = connector; this.label = conceptConnectorLabel; this.withIntervalPacking = containsIntervalPacking; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 973381cae3..71339400cd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -1,12 +1,14 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.List; +import java.util.Optional; -import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlTables; @@ -14,7 +16,39 @@ import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import com.google.common.base.Preconditions; -class IntervalPackingSelectsCte { +public class IntervalPackingSelectsCte { + + public static QueryStep forSelect( + QueryStep withAggregatedDaterange, + ColumnDateRange daterange, + SqlSelect select, + SqlTables tables, + SqlDialect sqlDialect + ) { + List predecessors = List.of(withAggregatedDaterange); + QueryStep directPredecessor = withAggregatedDaterange; + + // we need an additional predecessor to unnest the validity date if it is a single column range + if (sqlDialect.supportsSingleColumnRanges()) { + String unnestCteName = tables.cteName(ConceptCteStep.UNNEST_DATE); + directPredecessor = sqlDialect.getFunctionProvider().unnestDaterange(daterange, withAggregatedDaterange, unnestCteName); + predecessors = List.of(withAggregatedDaterange, directPredecessor); + } + + Selects predecessorSelects = directPredecessor.getQualifiedSelects(); + Selects selects = Selects.builder() + .ids(predecessorSelects.getIds()) + .sqlSelect(select) + .build(); + + return QueryStep.builder() + .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) + .selects(selects) + .fromTable(QueryStep.toTableLike(directPredecessor.getCteName())) + .groupBy(predecessorSelects.getIds().toFields()) + .predecessors(predecessors) + .build(); + } public static QueryStep forConnector(QueryStep predecessor, CQTableContext cqTableContext) { return create( @@ -40,8 +74,8 @@ public static QueryStep forConcept( } /** - * @param predecessor The preceding query step which must contain an aggregated validity date. - * @param intervalPackingSelects {@link Select}s that require an interval-packed date. + * @param predecessor The preceding query step which must contain an aggregated validity date. + * @param intervalPackingSelects {@link SqlSelect}s which will be part of the returned {@link QueryStep}. * @return A {@link QueryStep} containing converted interval packing selects, like {@link EventDurationSumSelect}, {@link EventDateUnionSelect}, etc. * Returns the given predecessor as is if the given list of interval packing selects is empty. */ @@ -55,20 +89,19 @@ private static QueryStep create( return predecessor; } - Preconditions.checkArgument( - predecessor.getQualifiedSelects().getValidityDate().isPresent(), - "Can't create a IntervalPackingSelectsCte without a validity date present." - ); + Optional validityDate = predecessor.getQualifiedSelects().getValidityDate(); + Preconditions.checkArgument(validityDate.isPresent(), "Can't create a IntervalPackingSelectsCte without a validity date present."); // we need an additional predecessor to unnest the validity date if it is a single column range List predecessors = List.of(); - QueryStep actualPredecessor = predecessor; - if (predecessor.getQualifiedSelects().getValidityDate().get().isSingleColumnRange()) { - actualPredecessor = functionProvider.unnestValidityDate(predecessor, tables.cteName(ConceptCteStep.UNNEST_DATE)); - predecessors = List.of(actualPredecessor); + QueryStep directPredecessor = predecessor; + if (validityDate.get().isSingleColumnRange()) { + String unnestCteName = tables.cteName(ConceptCteStep.UNNEST_DATE); + directPredecessor = functionProvider.unnestDaterange(validityDate.get(), predecessor, unnestCteName); + predecessors = List.of(directPredecessor); } - Selects predecessorSelects = actualPredecessor.getQualifiedSelects(); + Selects predecessorSelects = directPredecessor.getQualifiedSelects(); Selects selects = Selects.builder() .ids(predecessorSelects.getIds()) .sqlSelects(intervalPackingSelects) @@ -77,7 +110,7 @@ private static QueryStep create( return QueryStep.builder() .cteName(tables.cteName(ConceptCteStep.INTERVAL_PACKING_SELECTS)) .selects(selects) - .fromTable(QueryStep.toTableLike(actualPredecessor.getCteName())) + .fromTable(QueryStep.toTableLike(directPredecessor.getCteName())) .groupBy(predecessorSelects.getIds().toFields()) .predecessors(predecessors) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 2a6edc2333..c660248531 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -56,9 +56,10 @@ private static QueryStep.QueryStepBuilder joinWithStratificationTable( List conditions, CQTableContext tableContext ) { - QueryStep stratificationTable = tableContext.getConversionContext().getStratificationTable(); + QueryStep stratificationTableCte = tableContext.getConversionContext().getStratificationTable(); + Table stratificationTable = DSL.table(DSL.name(stratificationTableCte.getCteName())); - Selects stratificationSelects = stratificationTable.getQualifiedSelects(); + Selects stratificationSelects = stratificationTableCte.getQualifiedSelects(); SqlIdColumns stratificationIds = stratificationSelects.getIds(); SqlIdColumns rootTableIds = tableContext.getIds(); List idConditions = stratificationIds.join(rootTableIds); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java index 452a0b147f..5b48c2d3be 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePath.java @@ -54,6 +54,7 @@ private static ConnectorSqlTables createConnectorTables(CQConcept cqConcept, CQT Map cteNameMap = CteStep.createCteNameMap(tableInfo.getMappings().keySet(), conceptConnectorLabel, context.getNameGenerator()); return new ConnectorSqlTables( + cqTable.getConnector(), conceptConnectorLabel, tableInfo.getRootTable(), cteNameMap, @@ -70,11 +71,9 @@ public ConceptSqlTables createConceptTables(QueryStep predecessor) { List connectorSqlTables = this.connectorTableMap.values().stream().toList(); return new ConceptSqlTables( - conceptName, tableInfo.getRootTable(), cteNameMap, tableInfo.getMappings(), - tableInfo.isContainsIntervalPacking(), connectorSqlTables ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index bfe46e4623..aa18433443 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -52,7 +52,7 @@ private QueryStep createPreviousEndStep(IntervalPackingContext context, Aggregat .as(IntervalPacker.PREVIOUS_END_FIELD_NAME); List qualifiedSelects = new ArrayList<>(QualifyingUtil.qualify(context.getCarryThroughSelects(), sourceTableName)); - qualifiedSelects.add(new FieldWrapper<>(previousEnd)); + qualifiedSelects.add(new FieldWrapper<>(previousEnd, daterange.getStart().getName(), daterange.getEnd().getName())); Selects previousEndSelects = buildSelects(ids, daterange, qualifiedSelects, aggregationMode); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java index 84f7f150f7..ef07bd02b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingCteStep.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; +import java.util.HashMap; import java.util.Map; import java.util.Set; @@ -53,4 +54,16 @@ public static Map getMappings(CteStep root, SqlDialect dialect return mappings; } + /** + * Create predecessor mappings for these interval packing {@link CteStep}s based on the default mapping. + */ + public static Map getMappings(SqlDialect dialect) { + if (dialect.supportsSingleColumnRanges()) { + Map mappings = new HashMap<>(); + mappings.put(INTERVAL_COMPLETE, null); // final step directly mapped onto root table + return mappings; + } + return CteStep.getDefaultPredecessorMap(Set.of(values())); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java index 26a9c4eb57..f765f7cd84 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; @@ -127,6 +128,15 @@ public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dat return ColumnDateRange.of(lowerBound, upperBound); } + @Override + public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter) { + String tableName = daterangeSelectOrFilter.getTable().getName(); + if (daterangeSelectOrFilter.getEndColumn() != null) { + return ofStartAndEnd(tableName, daterangeSelectOrFilter.getStartColumn(), daterangeSelectOrFilter.getEndColumn()); + } + return ofStartAndEnd(tableName, daterangeSelectOrFilter.getColumn(), daterangeSelectOrFilter.getColumn()); + } + @Override public ColumnDateRange aggregated(ColumnDateRange columnDateRange) { return ColumnDateRange.of( @@ -150,23 +160,11 @@ public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) } @Override - public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { + public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName) { // HANA does not support single column datemultiranges return predecessor; } - @Override - public Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { - return DSL.field( - "{0}({1}, {2} {3})", - String.class, - DSL.keyword("STRING_AGG"), - stringField, - delimiter, - DSL.orderBy(orderByFields) - ); - } - @Override public Field daterangeStringAggregation(ColumnDateRange columnDateRange) { @@ -330,6 +328,11 @@ private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { endColumn = validityDate.getEndColumn(); } + return ofStartAndEnd(tableName, startColumn, endColumn); + } + + private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { + Field rangeStart = DSL.coalesce( DSL.field(DSL.name(tableName, startColumn.getName()), Date.class), toDateField(MIN_DATE_VALUE) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index 23128c0fc4..96ed3e3f7e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -9,12 +9,12 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.google.common.base.Preconditions; import org.jooq.ArrayAggOrderByStep; import org.jooq.Condition; import org.jooq.DataType; @@ -30,7 +30,8 @@ * * @see PostgreSQL Documentation */ -public class PostgreSqlFunctionProvider implements SqlFunctionProvider { +public class +PostgreSqlFunctionProvider implements SqlFunctionProvider { private static final String INFINITY_DATE_VALUE = "infinity"; private static final String MINUS_INFINITY_DATE_VALUE = "-infinity"; @@ -118,6 +119,15 @@ public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dat return intersection(validityDateRange, restriction); } + @Override + public ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter) { + String tableName = daterangeSelectOrFilter.getTable().getName(); + if (daterangeSelectOrFilter.getEndColumn() != null) { + return ofStartAndEnd(tableName, daterangeSelectOrFilter.getStartColumn(), daterangeSelectOrFilter.getEndColumn()); + } + return ofSingleColumn(tableName, daterangeSelectOrFilter.getColumn()); + } + @Override public ColumnDateRange aggregated(ColumnDateRange columnDateRange) { return ColumnDateRange.of(rangeAgg(columnDateRange)).as(columnDateRange.getAlias()); @@ -141,16 +151,10 @@ public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) } @Override - public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { + public QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName) { - Preconditions.checkArgument( - predecessor.getSelects().getValidityDate().isPresent(), - "Can't create a unnest-CTE without a validity date present." - ); - - Selects predecessorSelects = predecessor.getQualifiedSelects(); - ColumnDateRange validityDate = predecessorSelects.getValidityDate().get(); - ColumnDateRange unnested = ColumnDateRange.of(unnest(validityDate.getRange()).as(validityDate.getAlias())); + ColumnDateRange qualifiedRange = nested.qualify(predecessor.getCteName()); + ColumnDateRange unnested = ColumnDateRange.of(unnest(qualifiedRange.getRange()).as(qualifiedRange.getAlias())); Selects selects = Selects.builder() .ids(predecessor.getQualifiedSelects().getIds()) @@ -164,18 +168,6 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { .build(); } - @Override - public Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { - return DSL.field( - "{0}({1}, {2} {3})", - String.class, - DSL.keyword("string_agg"), - stringField, - delimiter, - DSL.orderBy(orderByFields) - ); - } - @Override public Field daterangeStringAggregation(ColumnDateRange columnDateRange) { Field asMultirange = rangeAgg(columnDateRange); @@ -318,30 +310,21 @@ private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { } private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { - String tableName = validityDate.getConnector().getTable().getName(); - - Field dateRange; - if (validityDate.getEndColumn() != null) { + return ofStartAndEnd(tableName, validityDate.getStartColumn(), validityDate.getEndColumn()); + } + return ofSingleColumn(tableName, validityDate.getColumn()); + } - Field startColumn = DSL.coalesce( - DSL.field(DSL.name(tableName, validityDate.getStartColumn().getName())), - toDateField(MINUS_INFINITY_DATE_VALUE) - ); - Field endColumn = DSL.coalesce( - DSL.field(DSL.name(tableName, validityDate.getEndColumn().getName())), - toDateField(INFINITY_DATE_VALUE) - ); + private ColumnDateRange ofSingleColumn(String tableName, Column column) { - return ColumnDateRange.of(daterange(startColumn, endColumn, "[]")); - } + Field dateRange; - Column validityDateColumn = validityDate.getColumn(); - dateRange = switch (validityDateColumn.getType()) { + dateRange = switch (column.getType()) { // if validityDateColumn is a DATE_RANGE we can make use of Postgres' integrated daterange type, but the upper bound is exclusive by default case DATE_RANGE -> { - Field daterange = DSL.field(DSL.name(validityDateColumn.getName())); + Field daterange = DSL.field(DSL.name(column.getName())); Field startColumn = DSL.coalesce( DSL.function("lower", Date.class, daterange), toDateField(MINUS_INFINITY_DATE_VALUE) @@ -354,19 +337,33 @@ private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { } // if the validity date column is not of daterange type, we construct it manually case DATE -> { - Field column = DSL.field(DSL.name(tableName, validityDate.getColumn().getName()), Date.class); - Field startColumn = DSL.coalesce(column, toDateField(MINUS_INFINITY_DATE_VALUE)); - Field endColumn = DSL.coalesce(column, toDateField(INFINITY_DATE_VALUE)); + Field singleDate = DSL.field(DSL.name(tableName, column.getName()), Date.class); + Field startColumn = DSL.coalesce(singleDate, toDateField(MINUS_INFINITY_DATE_VALUE)); + Field endColumn = DSL.coalesce(singleDate, toDateField(INFINITY_DATE_VALUE)); yield daterange(startColumn, endColumn, "[]"); } default -> throw new IllegalArgumentException( - "Given column type '%s' can't be converted to a proper date restriction.".formatted(validityDateColumn.getType()) + "Given column type '%s' can't be converted to a proper date restriction.".formatted(column.getType()) ); }; return ColumnDateRange.of(dateRange); } + private ColumnDateRange ofStartAndEnd(String tableName, Column startColumn, Column endColumn) { + + Field start = DSL.coalesce( + DSL.field(DSL.name(tableName, startColumn.getName())), + toDateField(MINUS_INFINITY_DATE_VALUE) + ); + Field end = DSL.coalesce( + DSL.field(DSL.name(tableName, endColumn.getName())), + toDateField(INFINITY_DATE_VALUE) + ); + + return ColumnDateRange.of(daterange(start, end, "[]")); + } + private ColumnDateRange ensureIsSingleColumnRange(ColumnDateRange daterange) { return daterange.isSingleColumnRange() ? daterange diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 1d7e09644e..46572215ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; @@ -79,6 +80,8 @@ public interface SqlFunctionProvider { */ ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction); + ColumnDateRange forArbitraryDateRange(DaterangeSelectOrFilter daterangeSelectOrFilter); + ColumnDateRange aggregated(ColumnDateRange columnDateRange); /** @@ -92,13 +95,13 @@ public interface SqlFunctionProvider { ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right); /** - * @param predecessor The predeceasing step containing an aggregated validity date. + * @param predecessor The predeceasing step containing the aggregated {@link ColumnDateRange}. + * @param nested The {@link ColumnDateRange} you want to unnest. + * @param cteName The CTE name of the returned {@link QueryStep}. * @return A QueryStep containing an unnested validity date with 1 row per single daterange for each id. For dialects that don't support single column * multiranges, the given predecessor will be returned as is. */ - QueryStep unnestValidityDate(QueryStep predecessor, String cteName); - - Field stringAggregation(Field stringField, Field delimiter, List> orderByFields); + QueryStep unnestDaterange(ColumnDateRange nested, QueryStep predecessor, String cteName); /** * Aggregates the start and end columns of the validity date of entries into one compound string expression. @@ -140,6 +143,17 @@ public interface SqlFunctionProvider { */ Field yearQuarter(Field dateField); + default Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { + return DSL.field( + "{0}({1}, {2} {3})", + String.class, + DSL.keyword("string_agg"), + stringField, + delimiter, + DSL.orderBy(orderByFields) + ); + } + default Field concat(List> fields) { String concatenated = fields.stream() // if a field is null, the whole concatenation would be null - but we just want to skip this field in this case, @@ -172,34 +186,16 @@ default Condition in(Field column, String[] values) { return column.in(values); } - default TableOnConditionStep innerJoin( - Table leftPartQueryBase, - QueryStep rightPartQS, - List joinConditions - ) { - return leftPartQueryBase - .innerJoin(DSL.name(rightPartQS.getCteName())) - .on(joinConditions.toArray(Condition[]::new)); + default TableOnConditionStep innerJoin(Table leftPart, Table rightPart, List joinConditions) { + return leftPart.innerJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } - default TableOnConditionStep fullOuterJoin( - Table leftPartQueryBase, - QueryStep rightPartQS, - List joinConditions - ) { - return leftPartQueryBase - .fullOuterJoin(DSL.name(rightPartQS.getCteName())) - .on(joinConditions.toArray(Condition[]::new)); + default TableOnConditionStep fullOuterJoin(Table leftPart, Table rightPart, List joinConditions) { + return leftPart.fullOuterJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } - default TableOnConditionStep leftJoin( - Table leftPartQueryBase, - QueryStep rightPartQS, - List joinConditions - ) { - return leftPartQueryBase - .leftJoin(DSL.name(rightPartQS.getCteName())) - .on(joinConditions.toArray(Condition[]::new)); + default TableOnConditionStep leftJoin(Table leftPart, Table rightPart, List joinConditions) { + return leftPart.leftJoin(rightPart).on(joinConditions.toArray(Condition[]::new)); } default Field toDateField(String dateExpression) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index 404d5a030e..913f7fbad9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -44,7 +44,9 @@ public QueryStep createRelativeStratificationTable(RelativeFormQuery form) { // we want to create the stratification for each distinct validity date range of an entity, // so we first need to unnest the validity date in case it is a multirange - QueryStep withUnnestedValidityDate = functionProvider.unnestValidityDate(baseStep, FormCteStep.UNNEST_DATES.getSuffix()); + Preconditions.checkArgument(baseStep.getSelects().getValidityDate().isPresent(), "Base step must contain a validity date"); + String unnestCteName = FormCteStep.UNNEST_DATES.getSuffix(); + QueryStep withUnnestedValidityDate = functionProvider.unnestDaterange(baseStep.getSelects().getValidityDate().get(), baseStep, unnestCteName); QueryStep indexSelectorStep = createIndexSelectorStep(form, withUnnestedValidityDate); QueryStep indexStartStep = createIndexStartStep(form, indexSelectorStep); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java index af6923cbcf..1bb312e979 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.identifiable.Labeled; +import com.bakdata.conquery.models.identifiable.Named; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -47,7 +47,7 @@ public String cteStepName(CteStep cteStep, String nodeLabel) { return ensureValidLength(cteStep.cteName(nodeLabel)); } - public String selectName(Labeled selectOrFilter) { + public String selectName(Named selectOrFilter) { int selectCount = this.selectCountMap.merge(selectOrFilter.getName(), 1, Integer::sum); String name = lowerAndReplaceWhitespace(selectOrFilter.getName()); return ensureValidLength("%s-%d".formatted(name, selectCount)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index f2f9b80cac..204bdd59e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -41,6 +41,10 @@ public class QueryStep { */ @Builder.Default boolean unionAll = true; + /** + * Determines if the select should be distinct. + */ + boolean selectDistinct; /** * All {@link QueryStep}'s that shall be converted before this {@link QueryStep}. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index 0a542da464..d58e373285 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -108,7 +108,8 @@ public static TableLike constructJoinedTable(List queriesToJo List joinConditions = Stream.concat(joinIdsCondition.stream(), Stream.of(joinDateCondition)).collect(Collectors.toList()); - joinedQuery = joinType.join(joinedQuery, rightPartQS, joinConditions); + Table rightPartTable = DSL.table(DSL.name(rightPartQS.getCteName())); + joinedQuery = joinType.join(joinedQuery, rightPartTable, joinConditions); } return joinedQuery; @@ -170,11 +171,7 @@ private static QueryStep buildStepAndAggregateDates( @FunctionalInterface private interface JoinType { - TableOnConditionStep join( - Table leftPartQueryBase, - QueryStep rightPartQS, - List joinConditions - ); + TableOnConditionStep join(Table leftPart, Table rightPart, List joinConditions); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index 8690587890..04dcf0cd84 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -5,12 +5,11 @@ import org.jooq.CommonTableExpression; import org.jooq.DSLContext; -import org.jooq.Field; import org.jooq.Record; import org.jooq.Select; import org.jooq.SelectConditionStep; import org.jooq.SelectHavingStep; -import org.jooq.SelectSeekStepN; +import org.jooq.SelectSelectStep; import org.jooq.impl.DSL; /** @@ -40,15 +39,11 @@ public Select toSelectQuery(QueryStep queryStep) { grouped = queryBase.groupBy(queryStep.getGroupBy()); } - // ordering - List> orderByFields = queryStep.getSelects().getIds().toFields(); - SelectSeekStepN ordered = grouped.orderBy(orderByFields); - // union if (!queryStep.isUnion()) { - return ordered; + return grouped; } - return union(queryStep, ordered); + return union(queryStep, grouped); } private List> constructPredecessorCteList(QueryStep queryStep) { @@ -76,10 +71,15 @@ private CommonTableExpression toCte(QueryStep queryStep) { private Select toSelectStep(QueryStep queryStep) { - Select selectStep = this.dslContext - .select(queryStep.getSelects().all()) - .from(queryStep.getFromTables()) - .where(queryStep.getConditions()); + SelectSelectStep selectClause; + if (queryStep.isSelectDistinct()) { + selectClause = dslContext.selectDistinct(queryStep.getSelects().all()); + } + else { + selectClause = dslContext.select(queryStep.getSelects().all()); + } + + Select selectStep = selectClause.from(queryStep.getFromTables()).where(queryStep.getConditions()); if (queryStep.isGroupBy()) { selectStep = ((SelectConditionStep) selectStep).groupBy(queryStep.getGroupBy()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 7dd3494050..6c8e6fd6a7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountQuartersFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect; import com.bakdata.conquery.models.events.MajorTypeId; @@ -34,7 +33,7 @@ public class CountQuartersSqlAggregator implements SelectConverter, FilterConverter, SqlAggregator { @Override - public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSelect, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(CountQuartersSelect countQuartersSelect, SelectContext selectContext) { String alias = selectContext.getNameGenerator().selectName(countQuartersSelect); ConnectorSqlTables tables = selectContext.getTables(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index 8d41ce66a7..501340d79f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -2,7 +2,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.filters.specific.CountFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; @@ -36,7 +35,7 @@ public static CountType fromBoolean(boolean value) { } @Override - public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(CountSelect countSelect, SelectContext selectContext) { ConnectorSqlTables tables = selectContext.getTables(); CountType countType = CountType.fromBoolean(countSelect.isDistinct()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index 9545d3b39b..ff7aba4043 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -7,7 +7,6 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.filters.specific.DateDistanceFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; import com.bakdata.conquery.models.events.MajorTypeId; @@ -36,7 +35,7 @@ public class DateDistanceSqlAggregator implements SelectConverter, FilterConverter { @Override - public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(DateDistanceSelect select, SelectContext selectContext) { Column column = select.getColumn(); String alias = selectContext.getNameGenerator().selectName(select); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 5c379e53f5..a1505b3447 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -7,7 +7,6 @@ import java.util.stream.Collectors; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.filters.specific.FlagFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.FlagSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; @@ -73,7 +72,7 @@ public class FlagSqlAggregator implements SelectConverter, FilterCon private static final Param NUMERIC_TRUE_VAL = DSL.val(1); @Override - public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext selectContext) { SqlFunctionProvider functionProvider = selectContext.getConversionContext().getSqlDialect().getFunctionProvider(); SqlTables connectorTables = selectContext.getTables(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 5ea0d55527..95c2d39122 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.filters.specific.SumFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; @@ -96,7 +95,7 @@ private enum SumDistinctCteStep implements CteStep { private static final String SUM_DISTINCT_SUFFIX = "sum_distinct"; @Override - public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(SumSelect sumSelect, SelectContext selectContext) { Column sumColumn = sumSelect.getColumn(); Column subtractColumn = sumSelect.getSubtractColumn(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java index b91da43125..02e770e89d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -7,7 +7,6 @@ import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; @@ -42,8 +41,8 @@ private enum CONCEPT_COLUMN_STEPS implements CteStep { } @Override - public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectContext selectContext) { - Connector connector = selectContext.getSelectHolder(); + public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectContext selectContext) { + Connector connector = selectContext.getTables().getConnector(); if (connector.getColumn() == null) { return ConnectorSqlSelects.none(); } @@ -56,17 +55,17 @@ public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectCon } @Override - public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext selectContext) { + public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext selectContext) { // we will do a union distinct on all Connector tables List connectors; - if (isSingleConnectorConcept(selectContext.getSelectHolder())) { + if (isSingleConnector(selectContext.getTables())) { // we union the Connector table with itself if there is only 1 Connector - Connector connector = selectContext.getSelectHolder().getConcept().getConnectors().get(0); + Connector connector = selectContext.getTables().getConnectorTables().get(0).getConnector(); connectors = List.of(connector, connector); } else { - connectors = selectContext.getSelectHolder().getConnectors(); + connectors = selectContext.getTables().getConnectorTables().stream().map(ConnectorSqlTables::getConnector).toList(); } NameGenerator nameGenerator = selectContext.getNameGenerator(); @@ -97,44 +96,42 @@ public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext .build(); } - private static boolean isSingleConnectorConcept(TreeConcept treeConcept) { - return treeConcept.getConcept().getConnectors().size() == 1; + private static boolean isSingleConnector(ConceptSqlTables tables) { + return tables.getConnectorTables().size() == 1; } private static QueryStep createUnionConnectorConnectorsStep( List connectors, String alias, - SelectContext selectContext + SelectContext selectContext ) { - List unionSteps = selectContext.getTables() - .getConnectorTables() - .stream() - .map(tables -> createConnectorColumnSelectQuery(tables, connectors, alias, selectContext)) - .toList(); - + List unionSteps = connectors.stream().map(connector -> createConnectorColumnSelectQuery(connector, alias, selectContext)).toList(); String unionedColumnsCteName = selectContext.getNameGenerator().cteStepName(CONCEPT_COLUMN_STEPS.UNIONED_COLUMNS, alias); return QueryStep.createUnionStep(unionSteps, unionedColumnsCteName, Collections.emptyList()); } private static QueryStep createConnectorColumnSelectQuery( - ConnectorSqlTables tables, - List connectors, + Connector connector, String alias, - SelectContext selectContext + SelectContext selectContext ) { - Connector matchingConnector = - connectors.stream() - .filter(connector -> isMatchingConnector(tables, connector)) - .findFirst() - .orElseThrow(() -> new IllegalStateException("Could not find matching connector for ConnectorSqlTables %s".formatted(tables))); - - Table connectorTable = DSL.table(DSL.name(tables.cteName(ConceptCteStep.EVENT_FILTER))); - - Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(matchingConnector.getTable(), selectContext.getConversionContext().getConfig()); + // a ConceptColumn select uses all connectors a Concept has, even if they are not part of the CQConcept + // but if they are, we need to make sure we use the event-filtered table instead of the root table + String tableName = selectContext.getTables() + .getConnectorTables() + .stream() + .filter(tables -> Objects.equals(tables.getRootTable(), connector.getTable().getName())) + .findFirst() + .map(tables -> tables.cteName(ConceptCteStep.EVENT_FILTER)) + .orElse(connector.getTable().getName()); + + Table connectorTable = DSL.table(DSL.name(tableName)); + + Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(connector.getTable(), selectContext.getConversionContext().getConfig()); Field qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, connectorTable.getName()).as(SharedAliases.PRIMARY_COLUMN.getAlias()); SqlIdColumns ids = new SqlIdColumns(qualifiedPrimaryColumn); - Field connectorColumn = DSL.field(DSL.name(connectorTable.getName(), matchingConnector.getColumn().getName())); + Field connectorColumn = DSL.field(DSL.name(connectorTable.getName(), connector.getColumn().getName())); Field casted = selectContext.getFunctionProvider().cast(connectorColumn, SQLDataType.VARCHAR).as(alias); FieldWrapper connectorSelect = new FieldWrapper<>(casted); @@ -149,11 +146,7 @@ private static QueryStep createConnectorColumnSelectQuery( .build(); } - private static boolean isMatchingConnector(final ConnectorSqlTables tables, final Connector connector) { - return connector.getColumn() != null && (Objects.equals(tables.getRootTable(), connector.getTable().getName())); - } - - private static FieldWrapper createConnectorColumnStringAgg(SelectContext selectContext, QueryStep unionStep, String alias) { + private static FieldWrapper createConnectorColumnStringAgg(SelectContext selectContext, QueryStep unionStep, String alias) { SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); Field unionedColumn = DSL.field(DSL.name(unionStep.getCteName(), alias), String.class); return new FieldWrapper<>( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java new file mode 100644 index 0000000000..b479b6859b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DateUnionSelectConverter.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateUnionSelect; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; + +public class DateUnionSelectConverter implements SelectConverter { + + @Override + public ConnectorSqlSelects connectorSelect(DateUnionSelect select, SelectContext selectContext) { + return DaterangeSelectUtil.createConnectorSqlSelects( + select, + (daterange, alias, functionProvider) -> new FieldWrapper<>(functionProvider.daterangeStringAggregation(daterange).as(alias)), + selectContext + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java new file mode 100644 index 0000000000..04bbc0b0cf --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DaterangeSelectUtil.java @@ -0,0 +1,126 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.EVENT_FILTER; +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.INTERVAL_PACKING_SELECTS; +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.UNNEST_DATE; +import static com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep.INTERVAL_COMPLETE; + +import java.math.BigDecimal; +import java.sql.Date; +import java.time.temporal.ChronoUnit; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.bakdata.conquery.models.datasets.concepts.DaterangeSelectOrFilter; +import com.bakdata.conquery.models.identifiable.Named; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.conversion.cqelement.concept.IntervalPackingSelectsCte; +import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; +import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.SqlTables; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +class DaterangeSelectUtil { + + @FunctionalInterface + public interface AggregationFunction { + FieldWrapper apply(ColumnDateRange daterange, String alias, SqlFunctionProvider functionProvider); + } + + /** + * Aggregates the daterange of a corresponding {@link DaterangeSelectOrFilter} and applies the respective converted aggregation via + * {@link IntervalPackingSelectsCte}s using additional predecessor tables. + */ + public static > ConnectorSqlSelects createConnectorSqlSelects( + S select, + AggregationFunction aggregationFunction, + SelectContext context + ) { + String alias = context.getNameGenerator().selectName(select); + ConnectorSqlTables tables = context.getTables(); + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); + + ColumnDateRange daterange = functionProvider.forArbitraryDateRange(select).as(alias); + List rootSelects = daterange.toFields().stream() + .map(FieldWrapper::new) + .collect(Collectors.toList()); + + SqlTables daterangeSelectTables = createTables(alias, context); + QueryStep lastIntervalPackingStep = applyIntervalPacking(daterange, daterangeSelectTables, context); + + ColumnDateRange qualified = daterange.qualify(daterangeSelectTables.getPredecessor(INTERVAL_PACKING_SELECTS)); + FieldWrapper aggregationField = aggregationFunction.apply(qualified, alias, functionProvider); + + QueryStep intervalPackingSelectsStep = IntervalPackingSelectsCte.forSelect( + lastIntervalPackingStep, + qualified, + aggregationField, + daterangeSelectTables, + context.getSqlDialect() + ); + + ExtractingSqlSelect finalSelect = aggregationField.qualify(tables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + + return ConnectorSqlSelects.builder() + .preprocessingSelects(rootSelects) + .additionalPredecessor(Optional.of(intervalPackingSelectsStep)) + .finalSelect(finalSelect) + .build(); + } + + public static FieldWrapper createDurationSumSqlSelect(String alias, ColumnDateRange validityDate, SqlFunctionProvider functionProvider) { + Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); + Field durationSum = DSL.sum( + DSL.when(containsInfinityDate(validityDate, functionProvider), DSL.val(null, Integer.class)) + .otherwise(dateDistanceInDays) + ) + .as(alias); + return new FieldWrapper<>(durationSum); + } + + private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlFunctionProvider functionProvider) { + Field negativeInfinity = functionProvider.toDateField(functionProvider.getMinDateExpression()); + Field positiveInfinity = functionProvider.toDateField(functionProvider.getMaxDateExpression()); + return validityDate.getStart().eq(negativeInfinity).or(validityDate.getEnd().eq(positiveInfinity)); + } + + private static SqlTables createTables(String alias, SelectContext context) { + Map predecessorMapping = new HashMap<>(); + String eventFilterCteName = context.getTables().cteName(EVENT_FILTER); + predecessorMapping.putAll(IntervalPackingCteStep.getMappings(context.getSqlDialect())); + if (context.getSqlDialect().supportsSingleColumnRanges()) { + predecessorMapping.put(UNNEST_DATE, INTERVAL_COMPLETE); + predecessorMapping.put(INTERVAL_PACKING_SELECTS, UNNEST_DATE); + } + else { + predecessorMapping.put(INTERVAL_PACKING_SELECTS, INTERVAL_COMPLETE); + } + Map cteNameMap = CteStep.createCteNameMap(predecessorMapping.keySet(), alias, context.getNameGenerator()); + return new SqlTables(eventFilterCteName, cteNameMap, predecessorMapping); + } + + private static QueryStep applyIntervalPacking(ColumnDateRange daterange, SqlTables dateUnionTables, SelectContext context) { + + String eventFilterCteName = context.getTables().cteName(EVENT_FILTER); + IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() + .ids(context.getIds().qualify(eventFilterCteName)) + .daterange(daterange.qualify(eventFilterCteName)) + .tables(dateUnionTables) + .build(); + + return context.getSqlDialect() + .getIntervalPacker() + .aggregateAsArbitrarySelect(intervalPackingContext); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java new file mode 100644 index 0000000000..5cde2abe5b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DistinctSelectConverter.java @@ -0,0 +1,134 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import static org.jooq.impl.DSL.field; +import static org.jooq.impl.DSL.name; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.concepts.select.connector.DistinctSelect; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +/** + * 
+ *  The two additional CTEs this aggregator creates:
+ * 	
    
+ * 	    
  1. 
+ * 	        Select distinct values of a column.
+ *            {@code
+ * 	        	"distinct" as (
+ *     				select distinct "pid", "column"
+ *     				from "event_filter"
+ *  			)
+ *            }
+ * 	    
    2. 
+ * 	    
  2. 
+ * 	        String agg all distinct values of the column.
+ *            {@code
+ * 	        "aggregated" as (
+ *    			 select
+ *    			   "select-1-distinct"."pid",
+ *    			   string_agg(cast("column" as varchar), cast(' ' as varchar) ) as "select-1"
+ *    			 from "distinct"
+ *    			 group by "pid"
+ *   			)
+ *            }
+ * 	    
    3. 
+ * 	

+ *
+ */ +public class DistinctSelectConverter implements SelectConverter { + + @Getter + @RequiredArgsConstructor + private enum DistinctSelectCteStep implements CteStep { + + DISTINCT_SELECT("distinct", null), + STRING_AGG("aggregated", DISTINCT_SELECT); + + private final String suffix; + private final DistinctSelectCteStep predecessor; + } + + @Override + public ConnectorSqlSelects connectorSelect(DistinctSelect distinctSelect, SelectContext selectContext) { + + String alias = selectContext.getNameGenerator().selectName(distinctSelect); + + ConnectorSqlTables tables = selectContext.getTables(); + FieldWrapper preprocessingSelect = new FieldWrapper<>(field(name(tables.getRootTable(), distinctSelect.getColumn().getName())).as(alias)); + + QueryStep distinctSelectCte = createDistinctSelectCte(preprocessingSelect, alias, selectContext); + QueryStep aggregatedCte = createAggregationCte(selectContext, preprocessingSelect, distinctSelectCte, alias); + + ExtractingSqlSelect finalSelect = preprocessingSelect.qualify(tables.cteName(ConceptCteStep.AGGREGATION_FILTER)); + + return ConnectorSqlSelects.builder() + .preprocessingSelect(preprocessingSelect) + .additionalPredecessor(Optional.of(aggregatedCte)) + .finalSelect(finalSelect) + .build(); + } + + private static QueryStep createAggregationCte( + SelectContext selectContext, + FieldWrapper preprocessingSelect, + QueryStep distinctSelectCte, + String alias + ) { + SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); + Field castedColumn = functionProvider.cast(preprocessingSelect.qualify(distinctSelectCte.getCteName()).select(), SQLDataType.VARCHAR); + Field aggregatedColumn = functionProvider.stringAggregation(castedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(castedColumn)) + .as(alias); + + SqlIdColumns ids = distinctSelectCte.getQualifiedSelects().getIds(); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(new FieldWrapper<>(aggregatedColumn)) + .build(); + + return QueryStep.builder() + .cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.STRING_AGG, alias)) + .selects(selects) + .fromTable(QueryStep.toTableLike(distinctSelectCte.getCteName())) + .groupBy(ids.toFields()) + .predecessor(distinctSelectCte) + .build(); + } + + private static QueryStep createDistinctSelectCte( + FieldWrapper preprocessingSelect, + String alias, + SelectContext selectContext + ) { + // values to aggregate must be event-filtered first + String eventFilterTable = selectContext.getTables().cteName(ConceptCteStep.EVENT_FILTER); + ExtractingSqlSelect qualified = preprocessingSelect.qualify(eventFilterTable); + SqlIdColumns ids = selectContext.getIds().qualify(eventFilterTable); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(qualified) + .build(); + + return QueryStep.builder() + .cteName(selectContext.getNameGenerator().cteStepName(DistinctSelectCteStep.DISTINCT_SELECT, alias)) + .selectDistinct(true) + .selects(selects) + .fromTable(QueryStep.toTableLike(eventFilterTable)) + .build(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java new file mode 100644 index 0000000000..de8ec3496f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/DurationSumSelectConverter.java @@ -0,0 +1,21 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DurationSumSelect; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; + +public class DurationSumSelectConverter implements SelectConverter { + + @Override + public ConnectorSqlSelects connectorSelect(DurationSumSelect select, SelectContext selectContext) { + return DaterangeSelectUtil.createConnectorSqlSelects( + select, + (daterange, alias, functionProvider) -> { + ColumnDateRange asDualColumn = functionProvider.toDualColumn(daterange); + return DaterangeSelectUtil.createDurationSumSqlSelect(alias, asDualColumn, functionProvider); + }, + selectContext + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java index 0558140279..852960ead1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDateUnionSelectConverter.java @@ -1,8 +1,6 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDateUnionSelect; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; @@ -13,7 +11,7 @@ public class EventDateUnionSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectContext selectContext) { FieldWrapper stringAggregation = createEventDateUnionAggregation(select, selectContext); ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); @@ -25,7 +23,7 @@ public ConnectorSqlSelects connectorSelect(EventDateUnionSelect select, SelectCo } @Override - public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContext selectContext) { + public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContext selectContext) { FieldWrapper stringAggregation = createEventDateUnionAggregation(select, selectContext); ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); @@ -36,7 +34,7 @@ public ConceptSqlSelects conceptSelect(EventDateUnionSelect select, SelectContex .build(); } - private static FieldWrapper createEventDateUnionAggregation(EventDateUnionSelect select, SelectContext selectContext) { + private static FieldWrapper createEventDateUnionAggregation(EventDateUnionSelect select, SelectContext selectContext) { Preconditions.checkArgument(selectContext.getValidityDate().isPresent(), "Can't convert an EventDateUnionSelect without a validity date being present"); ColumnDateRange validityDate = selectContext.getValidityDate().get(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java index 87ab0900b8..369f4f8489 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/EventDurationSumSelectConverter.java @@ -4,9 +4,7 @@ import java.sql.Date; import java.time.temporal.ChronoUnit; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; @@ -20,7 +18,7 @@ public class EventDurationSumSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, SelectContext selectContext) { FieldWrapper stringAggregation = createEventDurationSumAggregation(select, selectContext); ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); @@ -32,7 +30,7 @@ public ConnectorSqlSelects connectorSelect(EventDurationSumSelect select, Select } @Override - public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectContext selectContext) { + public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectContext selectContext) { FieldWrapper stringAggregation = createEventDurationSumAggregation(select, selectContext); ExtractingSqlSelect finalSelect = stringAggregation.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.UNIVERSAL_SELECTS)); @@ -43,7 +41,7 @@ public ConceptSqlSelects conceptSelect(EventDurationSumSelect select, SelectCont .build(); } - private FieldWrapper createEventDurationSumAggregation(EventDurationSumSelect select, SelectContext selectContext) { + private FieldWrapper createEventDurationSumAggregation(EventDurationSumSelect select, SelectContext selectContext) { Preconditions.checkArgument(selectContext.getValidityDate().isPresent(), "Can't convert an EventDateUnionSelect without a validity date being present"); String predecessorCteName = selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java index b0c8561aa9..b63d1d0422 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSelectConverter.java @@ -1,15 +1,13 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; public class ExistsSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext selectContext) { ExistsSqlSelect existsSqlSelect = createExistsSelect(select, selectContext); return ConnectorSqlSelects.builder() .finalSelect(existsSqlSelect) @@ -17,14 +15,14 @@ public ConnectorSqlSelects connectorSelect(ExistsSelect select, SelectContext selectContext) { + public ConceptSqlSelects conceptSelect(ExistsSelect select, SelectContext selectContext) { ExistsSqlSelect existsSqlSelect = createExistsSelect(select, selectContext); return ConceptSqlSelects.builder() .finalSelect(existsSqlSelect) .build(); } - private static ExistsSqlSelect createExistsSelect(ExistsSelect select, SelectContext selectContext) { + private static ExistsSqlSelect createExistsSelect(ExistsSelect select, SelectContext selectContext) { String alias = selectContext.getNameGenerator().selectName(select); return new ExistsSqlSelect(alias); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java index 5a81632ba2..64b19538bb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FirstValueSelectConverter.java @@ -1,13 +1,12 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; public class FirstValueSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(FirstValueSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(FirstValueSelect select, SelectContext selectContext) { return ValueSelectUtil.createValueSelect( select.getColumn(), selectContext.getNameGenerator().selectName(select), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java index 2ad575be52..06a27c7484 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/LastValueSelectConverter.java @@ -1,13 +1,12 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.connector.LastValueSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; public class LastValueSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(LastValueSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(LastValueSelect select, SelectContext selectContext) { return ValueSelectUtil.createValueSelect( select.getColumn(), selectContext.getNameGenerator().selectName(select), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java index 8e1fbf5eef..208860a993 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/RandomValueSelectConverter.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; @@ -9,7 +8,7 @@ public class RandomValueSelectConverter implements SelectConverter { @Override - public ConnectorSqlSelects connectorSelect(RandomValueSelect select, SelectContext selectContext) { + public ConnectorSqlSelects connectorSelect(RandomValueSelect select, SelectContext selectContext) { ConnectorSqlTables tables = selectContext.getTables(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java index 1f4ff5aee0..d37e743641 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java @@ -4,9 +4,6 @@ import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.datasets.concepts.SelectHolder; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; @@ -21,35 +18,31 @@ @Value @AllArgsConstructor(access = AccessLevel.PRIVATE) -public class SelectContext, T extends SqlTables> implements Context { +public class SelectContext implements Context { - S selectHolder; SqlIdColumns ids; Optional validityDate; T tables; ConversionContext conversionContext; - public static SelectContext create( + public static SelectContext create( CQTable cqTable, SqlIdColumns ids, Optional validityDate, ConnectorSqlTables tables, ConversionContext conversionContext ) { - return new SelectContext<>(cqTable.getConnector(), ids, validityDate, tables, conversionContext); + return new SelectContext<>(ids, validityDate, tables, conversionContext); } - public static SelectContext create( + public static SelectContext create( CQConcept cqConcept, SqlIdColumns ids, Optional validityDate, ConceptSqlTables tables, ConversionContext conversionContext ) { - if (!(cqConcept.getConcept() instanceof TreeConcept treeConcept)) { - throw new IllegalArgumentException("Cannot create a select context for a concept that is not a TreeConcept"); - } - return new SelectContext<>(treeConcept, ids, validityDate, tables, conversionContext); + return new SelectContext<>(ids, validityDate, tables, conversionContext); } public SqlFunctionProvider getFunctionProvider() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java index fc043a84b6..3e8a8bfb29 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectConverter.java @@ -1,18 +1,16 @@ package com.bakdata.conquery.sql.conversion.model.select; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; public interface SelectConverter { - default ConnectorSqlSelects connectorSelect(S select, SelectContext selectContext) { + default ConnectorSqlSelects connectorSelect(S select, SelectContext selectContext) { throw new UnsupportedOperationException("Conversion of Select %s not implemented on Connector-level".formatted(select.getClass())); } - default ConceptSqlSelects conceptSelect(S select, SelectContext selectContext) { + default ConceptSqlSelects conceptSelect(S select, SelectContext selectContext) { throw new UnsupportedOperationException("Conversion of Select %s not implemented or not possible on Concept-level".formatted(select.getClass())); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java index 633a520df1..f6da877a02 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ValueSelectUtil.java @@ -5,7 +5,6 @@ import java.util.function.BiFunction; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; @@ -17,7 +16,7 @@ public static ConnectorSqlSelects createValueSelect( Column column, String alias, BiFunction, List>, Field> aggregationFunction, - SelectContext selectContext + SelectContext selectContext ) { ConnectorSqlTables tables = selectContext.getTables(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java index 808c672a63..457eebe740 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -57,7 +57,8 @@ private static QueryStep overwriteBounds(QueryStep prerequisite, EntityDateQuery SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); // we want to create a stratification for each distinct validity date range of an entity, // so we first need to unnest the validity date in case it is a multirange - QueryStep unnestedEntityDate = functionProvider.unnestValidityDate(prerequisite, FormCteStep.UNNEST_ENTITY_DATE_CTE.getSuffix()); + String unnestCteName = FormCteStep.UNNEST_ENTITY_DATE_CTE.getSuffix(); + QueryStep unnestedEntityDate = functionProvider.unnestDaterange(prerequisite.getSelects().getValidityDate().get(), prerequisite, unnestCteName); Selects unnestedSelects = unnestedEntityDate.getQualifiedSelects(); ColumnDateRange withOverwrittenBounds; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index ce433e53b9..5b2b42d092 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -199,8 +199,9 @@ private static Table joinConnectorTableWithPrerequisite( ) { SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); Table connectorTable = DSL.table(DSL.name(cqTable.getConnector().getTable().getName())); + Table convertedPrerequisiteTable = DSL.table(DSL.name(convertedPrerequisite.getCteName())); List joinOnIds = ids.join(convertedPrerequisite.getQualifiedSelects().getIds()); - return functionProvider.innerJoin(connectorTable, convertedPrerequisite, joinOnIds); + return functionProvider.innerJoin(connectorTable, convertedPrerequisiteTable, joinOnIds); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java index 62b430d9ed..f14fc852b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java @@ -2,19 +2,21 @@ import java.math.BigDecimal; import java.math.RoundingMode; -import java.sql.Date; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; -import java.util.Collections; import java.util.List; +import java.util.function.Function; import java.util.function.Predicate; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.util.DateReader; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor class DefaultResultSetProcessor implements ResultSetProcessor { + private final ConqueryConfig config; private final SqlCDateSetParser sqlCDateSetParser; @Override @@ -48,11 +50,12 @@ public Boolean getBoolean(ResultSet resultSet, int columnIndex) throws SQLExcept @Override public Number getDate(ResultSet resultSet, int columnIndex) throws SQLException { - Date date = resultSet.getDate(columnIndex); - if (date == null) { + String dateString = resultSet.getString(columnIndex); + if (dateString == null) { return null; } - return date.toLocalDate().toEpochDay(); + DateReader dateReader = config.getLocale().getDateReader(); + return dateReader.parseToLocalDate(dateString).toEpochDay(); } @Override @@ -67,12 +70,51 @@ public List> getDateRangeList(ResultSet resultSet, int columnIndex @Override public List getStringList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, (string) -> string); + } + + @Override + public List getBooleanList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, Boolean::valueOf); + } + + @Override + public List getIntegerList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, Integer::valueOf); + } + + @Override + public List getDoubleList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, Double::valueOf); + } + + @Override + public List getMoneyList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString( + resultSet, + columnIndex, + (string) -> BigDecimal.valueOf(Double.parseDouble(string)) + .setScale(2, RoundingMode.HALF_EVEN) + ); + } + + @Override + public List getDateList(ResultSet resultSet, int columnIndex) throws SQLException { + return fromString(resultSet, columnIndex, this::parseWithDateReader); + } + + private Number parseWithDateReader(String string) { + return config.getLocale().getDateReader().parseToLocalDate(string).toEpochDay(); + } + + private List fromString(ResultSet resultSet, int columnIndex, Function parseFunction) throws SQLException { String arrayExpression = resultSet.getString(columnIndex); if (arrayExpression == null) { - return Collections.emptyList(); + return null; } return Arrays.stream(arrayExpression.split(String.valueOf(ResultSetProcessor.UNIT_SEPARATOR))) .filter(Predicate.not(String::isBlank)) + .map(parseFunction) .toList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java index 9f63f58c36..6581ec8b21 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java @@ -26,4 +26,14 @@ public interface ResultSetProcessor { List> getDateRangeList(ResultSet resultSet, int columnIndex) throws SQLException; List getStringList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getBooleanList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getIntegerList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getDoubleList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getMoneyList(ResultSet resultSet, int columnIndex) throws SQLException; + + List getDateList(ResultSet resultSet, int columnIndex) throws SQLException; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessorFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessorFactory.java index 1ab5e99be0..a9165b1ba1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessorFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessorFactory.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.sql.execution; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; public class ResultSetProcessorFactory { - public static ResultSetProcessor create(SqlDialect sqlDialect) { - return new DefaultResultSetProcessor(sqlDialect.getCDateSetParser()); + public static ResultSetProcessor create(ConqueryConfig config, SqlDialect sqlDialect) { + return new DefaultResultSetProcessor(config, sqlDialect.getCDateSetParser()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java b/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java new file mode 100644 index 0000000000..9bae79415c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/commands/NoOpConquery.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.util.commands; + +import com.bakdata.conquery.Conquery; +import com.bakdata.conquery.commands.ManagerNode; +import com.bakdata.conquery.models.config.ConqueryConfig; +import io.dropwizard.core.setup.Environment; + +/** + * {@link io.dropwizard.core.Application} "placeholder" for {@link io.dropwizard.core.cli.ServerCommand}/{@link io.dropwizard.core.cli.EnvironmentCommand}s that should not + * run {@link ManagerNode} related code. + *

+ * The {@link io.dropwizard.core.cli.EnvironmentCommand} calls normally {@link Conquery#run(ConqueryConfig, Environment)} which brings up the manager. + */ +public class NoOpConquery extends Conquery { + + @Override + public void run(ConqueryConfig configuration, Environment environment) throws Exception { + // Do nothing + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java index 3db0a88d50..3f1716a3f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.ExternalId; import com.bakdata.conquery.models.identifiable.mapping.FullIdPrinter; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.worker.Namespace; @@ -23,8 +23,8 @@ public class IdColumnUtil { /** * If a column contains an ID, create a reader for that ID. */ - public static List> getIdReaders(List format, Map idMappers) { - List> out = new ArrayList<>(format.size()); + public static List> getIdReaders(List format, Map idMappers) { + List> out = new ArrayList<>(format.size()); for (int index = 0; index < format.size(); index++) { final ColumnConfig mapper = idMappers.get(format.get(index)); diff --git a/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java b/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java new file mode 100644 index 0000000000..5fbd90e3c2 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/command/DistributedCommandsTest.java @@ -0,0 +1,116 @@ +package com.bakdata.conquery.command; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Map; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.core.GenericType; +import jakarta.ws.rs.core.Response; + +import com.bakdata.conquery.Conquery; +import com.bakdata.conquery.commands.ShardCommand; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.worker.ClusterHealthCheck; +import com.bakdata.conquery.util.NonPersistentStoreFactory; +import com.bakdata.conquery.util.support.ConfigOverride; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import io.dropwizard.core.cli.ServerCommand; +import io.dropwizard.testing.junit5.DropwizardAppExtension; +import io.dropwizard.testing.junit5.DropwizardExtensionsSupport; +import io.dropwizard.util.Duration; +import lombok.Data; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.ExtendWith; + +@ExtendWith(DropwizardExtensionsSupport.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +public class DistributedCommandsTest { + + public static final Duration CONNECT_RETRY_TIMEOUT = Duration.seconds(1); + + private static final ConqueryConfig CONQUERY_CONFIG_MANAGER = new ConqueryConfig() {{ + ConfigOverride.configureRandomPorts(this); + this.setStorage(new NonPersistentStoreFactory()); + }}; + + private static final ConqueryConfig CONQUERY_CONFIG_SHARD = new ConqueryConfig() {{ + ConfigOverride.configureRandomPorts(this); + this.getCluster().setPort(CONQUERY_CONFIG_MANAGER.getCluster().getPort()); + this.getCluster().setConnectRetryTimeout(CONNECT_RETRY_TIMEOUT); + this.setStorage(new NonPersistentStoreFactory()); + }}; + + private static final DropwizardAppExtension MANAGER = new DropwizardAppExtension<>( + Conquery.class, + CONQUERY_CONFIG_MANAGER, + ServerCommand::new + ); + + private static final DropwizardAppExtension SHARD = new DropwizardAppExtension<>( + Conquery.class, + CONQUERY_CONFIG_SHARD, + application -> new ShardCommand() + ); + + @Test + @Order(0) + void checkHttpUpShard() { + Client client = SHARD.client(); + + Response response = client.target( + String.format("http://localhost:%d/ping", SHARD.getAdminPort())) + .request() + .get(); + + assertThat(response.getStatus()).isEqualTo(200); + } + + @Test + @Order(0) + void checkHttpUpManager() { + Client client = MANAGER.client(); + + Response response = client.target( + String.format("http://localhost:%d/ping", MANAGER.getAdminPort())) + .request() + .get(); + + assertThat(response.getStatus()).isEqualTo(200); + } + + @Test + @Order(1) + void clusterEstablished() throws InterruptedException { + Client client = MANAGER.client(); + + // Wait for Shard to be connected + // May use https://github.com/awaitility/awaitility here in the future + Thread.sleep(2 * CONNECT_RETRY_TIMEOUT.toMilliseconds()); + + Response response = client.target( + String.format("http://localhost:%d/healthcheck", MANAGER.getAdminPort())) + .request() + .get(); + + + assertThat(response.getStatus()).isEqualTo(200); + + Map healthCheck = response.readEntity(new GenericType<>() { + }); + + assertThat(healthCheck).containsKey("cluster"); + assertThat(healthCheck.get("cluster").healthy).isTrue(); + assertThat(healthCheck.get("cluster").getMessage()).isEqualTo(String.format(ClusterHealthCheck.HEALTHY_MESSAGE_FMT, 1)); + } + + @Data + @JsonIgnoreProperties(ignoreUnknown = true) + private static class GenericHealthCheckResult { + private boolean healthy; + private String message; + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 29ace90e72..e37943068e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -7,14 +7,7 @@ import java.io.InputStream; import java.net.URI; import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; +import java.util.*; import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -39,17 +32,12 @@ import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.base.Strings; import io.github.classgraph.Resource; -import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.junit.jupiter.api.DynamicContainer; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.DynamicTest; -import org.junit.jupiter.api.extension.BeforeAllCallback; -import org.junit.jupiter.api.extension.Extension; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.RegisterExtension; @Slf4j public class IntegrationTests { @@ -79,15 +67,13 @@ public class IntegrationTests { @Getter private final File workDir; @Getter - @RegisterExtension - public TestConqueryConfig config; + public final ConqueryConfig config = new ConqueryConfig(); @SneakyThrows(IOException.class) public IntegrationTests(String defaultTestRoot, String defaultTestRootPackage) { this.defaultTestRoot = defaultTestRoot; this.defaultTestRootPackage = defaultTestRootPackage; this.workDir = Files.createTempDirectory("conqueryIntegrationTest").toFile(); - this.config = new TestConqueryConfig(); ConfigOverride.configurePathsAndLogging(this.config, this.workDir); } @@ -262,19 +248,6 @@ public synchronized TestConquery getCachedConqueryInstance(File workDir, Conquer return conquery; } - @EqualsAndHashCode(callSuper = true) - public static class TestConqueryConfig extends ConqueryConfig implements Extension, BeforeAllCallback { - - @Override - public void beforeAll(ExtensionContext context) throws Exception { - - context.getTestInstance() - .filter(ConfigOverride.class::isInstance) - .map(ConfigOverride.class::cast) - .ifPresent(co -> co.override(this)); - } - } - private static ResourceTree scanForResources(String testRoot, String pattern) { ResourceTree tree = new ResourceTree(null, null); tree.addAll(CPSTypeIdResolver.SCAN_RESULT.getResourcesMatchingPattern(Pattern.compile("^" + testRoot + pattern))); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java index 07e8aea28a..cefa4caf5c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java @@ -10,9 +10,7 @@ import com.bakdata.conquery.integration.IntegrationTest; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.View; -import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.IdColumnConfig; @@ -21,7 +19,6 @@ import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.ids.Id; import com.bakdata.conquery.models.identifiable.ids.IdUtil; -import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.bakdata.conquery.util.support.StandaloneSupport; import com.bakdata.conquery.util.support.TestSupport; @@ -55,8 +52,8 @@ public abstract class ConqueryTestSpec { @Nullable SqlSpec sqlSpec; - @Nullable - private IdColumnConfig idColumns; + // default IdColumnConfig for SQL mode + private IdColumnConfig idColumns = null; public ConqueryConfig overrideConfig(ConqueryConfig config) { @@ -94,16 +91,12 @@ public static T parseSubTree(TestSupport support, JsonNode node, JavaType ex } public static T parseSubTree(TestSupport support, JsonNode node, JavaType expectedType, Consumer modifierBeforeValidation) throws IOException { - final ObjectMapper om = Jackson.MAPPER.copy(); - ObjectMapper mapper = support.getDataset().injectIntoNew( - new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry(), support.getMetaStorage().getCentralRegistry()) - .injectIntoNew( - om.addHandler(new DatasetPlaceHolderFiller(support)) - ) - ); - final MutableInjectableValues injectableValues = (MutableInjectableValues) mapper.getInjectableValues(); - injectableValues.add(ConqueryConfig.class, support.getConfig()); - injectableValues.add(MetaStorage.class, support.getMetaStorage()); + final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); + support.getDataset().injectInto(mapper); + support.getNamespace().injectInto(mapper); + support.getMetaStorage().injectInto(mapper); + support.getConfig().injectInto(mapper); + mapper.addHandler(new DatasetPlaceHolderFiller(support)); T result = mapper.readerFor(expectedType).readValue(node); @@ -117,13 +110,12 @@ public static T parseSubTree(TestSupport support, JsonNode node, JavaType ex public static List parseSubTreeList(TestSupport support, ArrayNode node, Class expectedType, Consumer modifierBeforeValidation) throws IOException { - final ObjectMapper om = Jackson.MAPPER.copy(); - ObjectMapper mapper = support.getDataset().injectInto( - new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry()).injectIntoNew( - om.addHandler(new DatasetPlaceHolderFiller(support)) - ) - ); - support.getNamespace().getInjectables().forEach(i -> i.injectInto(mapper)); + final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); + support.getDataset().injectInto(mapper); + support.getNamespace().injectInto(mapper); + support.getMetaStorage().injectInto(mapper); + support.getConfig().injectInto(mapper); + mapper.addHandler(new DatasetPlaceHolderFiller(support)); mapper.setConfig(mapper.getDeserializationConfig().withView(View.Api.class)); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java index 21a37e7267..90211c9b65 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java @@ -1,81 +1,54 @@ package com.bakdata.conquery.integration.json; -import java.io.IOException; import java.util.Collection; -import java.util.List; -import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; -import com.bakdata.conquery.integration.common.RequiredSecondaryId; import com.bakdata.conquery.integration.common.RequiredTable; import com.bakdata.conquery.integration.json.filter.FilterTest; import com.bakdata.conquery.integration.sql.CsvTableImporter; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.util.support.StandaloneSupport; -import com.fasterxml.jackson.databind.node.ArrayNode; import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @Slf4j @RequiredArgsConstructor public class SqlTestDataImporter implements TestDataImporter { + private static final RequiredTable ALL_IDS_TABLE = importRequiredTable("/shared/entities.table.json"); + private final CsvTableImporter csvTableImporter; @Override public void importQueryTestData(StandaloneSupport support, QueryTest test) throws Exception { RequiredData content = test.getContent(); + content.getTables().add(ALL_IDS_TABLE); importSecondaryIds(support, content.getSecondaryIds()); importTables(support, content.getTables(), true); importConcepts(support, test.getRawConcepts()); importTableContents(support, content.getTables()); + + importSearchIndexes(support, test.getSearchIndexes()); + importIdMapping(support, content); } @Override public void importFormTestData(StandaloneSupport support, FormTest test) throws Exception { RequiredData content = test.getContent(); + content.getTables().add(ALL_IDS_TABLE); importSecondaryIds(support, content.getSecondaryIds()); importTables(support, content.getTables(), true); importConcepts(support, test.getRawConcepts()); importTableContents(support, content.getTables()); - waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content)); - waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser())); + importIdMapping(support, content); + importPreviousQueries(support, content); } @Override - public void importFilterTestData(StandaloneSupport support, FilterTest filterTest) throws Exception { + public void importFilterTestData(StandaloneSupport support, FilterTest filterTest) { throw new UnsupportedOperationException("Not implemented yet."); } - @Override - public void importSecondaryIds(StandaloneSupport support, List secondaryIds) { - for (RequiredSecondaryId required : secondaryIds) { - final SecondaryIdDescription description = - required.toSecondaryId(support.getDataset(), support.getDatasetRegistry().findRegistry(support.getDataset().getId())); - support.getDatasetsProcessor().addSecondaryId(support.getNamespace(), description); - } - } - - @Override - public void importTables(StandaloneSupport support, List tables, boolean autoConcept) throws JSONException { - for (RequiredTable requiredTable : tables) { - final Table table = requiredTable.toTable(support.getDataset(), support.getNamespaceStorage().getCentralRegistry()); - support.getNamespaceStorage().addTable(table); - } - } - - @Override - public void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException { - List> concepts = - ConqueryTestSpec.parseSubTreeList(support, rawConcepts, Concept.class, concept -> concept.setDataset(support.getDataset())); - for (Concept concept : concepts) { - support.getNamespaceStorage().updateConcept(concept); - } - } - @Override public void importTableContents(StandaloneSupport support, Collection tables) throws Exception { for (RequiredTable table : tables) { @@ -83,4 +56,9 @@ public void importTableContents(StandaloneSupport support, Collection secondaryIds); + void importTableContents(StandaloneSupport support, Collection tables) throws Exception; + + default void importSearchIndexes(StandaloneSupport support, List searchIndices) { + waitUntilDone(support, () -> LoadingUtil.importSearchIndexes(support, searchIndices)); + } - void importTables(StandaloneSupport support, List tables, boolean autoConcept) throws JSONException; + default void importIdMapping(StandaloneSupport support, RequiredData content) { + waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content)); + } - void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException; + default void importSecondaryIds(StandaloneSupport support, List secondaryIds) { + waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, secondaryIds)); + } - void importTableContents(StandaloneSupport support, Collection tables) throws Exception; + default void importTables(StandaloneSupport support, List tables, boolean autoConcept) throws JSONException { + waitUntilDone(support, () -> LoadingUtil.importTables(support, tables, autoConcept)); + } + + default void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException { + waitUntilDone(support, () -> LoadingUtil.importConcepts(support, rawConcepts)); + } + + default void importPreviousQueries(StandaloneSupport support, RequiredData content) { + waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser())); + } default void waitUntilDone(StandaloneSupport support, CheckedRunnable runnable) { runnable.run(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java index 21a537e54b..ba050711b1 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java @@ -2,22 +2,15 @@ import static com.bakdata.conquery.integration.common.LoadingUtil.importInternToExternMappers; -import java.io.IOException; import java.util.Collection; import java.util.Collections; -import java.util.List; import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; -import com.bakdata.conquery.integration.common.RequiredSecondaryId; import com.bakdata.conquery.integration.common.RequiredTable; import com.bakdata.conquery.integration.json.filter.FilterTest; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; -import com.bakdata.conquery.models.exceptions.JSONException; -import com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage; -import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.util.support.StandaloneSupport; -import com.fasterxml.jackson.databind.node.ArrayNode; public class WorkerTestDataImporter implements TestDataImporter { @@ -29,15 +22,13 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw importSecondaryIds(support, content.getSecondaryIds()); importInternToExternMappers(support, test.getInternToExternMappings()); - waitUntilDone(support, () -> LoadingUtil.importSearchIndexes(support, test.getSearchIndexes())); - waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept())); - waitUntilDone(support, () -> LoadingUtil.importConcepts(support, test.getRawConcepts())); + importSearchIndexes(support, test.getSearchIndexes()); + importTables(support, content.getTables(), content.isAutoConcept()); + importConcepts(support, test.getRawConcepts()); waitUntilDone(support, () -> LoadingUtil.importTableContents(support, content.getTables())); - waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content)); - waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser())); + importIdMapping(support, content); + importPreviousQueries(support, content); waitUntilDone(support, () -> LoadingUtil.updateMatchingStats(support)); - - sendUpdateMatchingStatsMessage(support); } @Override @@ -45,12 +36,12 @@ public void importFormTestData(StandaloneSupport support, FormTest test) throws RequiredData content = test.getContent(); - waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, content.getSecondaryIds())); - waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept())); - waitUntilDone(support, () -> LoadingUtil.importConcepts(support, test.getRawConcepts())); + importSecondaryIds(support, content.getSecondaryIds()); + importTables(support, content.getTables(), content.isAutoConcept()); + importConcepts(support, test.getRawConcepts()); waitUntilDone(support, () -> LoadingUtil.importTableContents(support, content.getTables())); - waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content)); - waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser())); + importIdMapping(support, content); + importPreviousQueries(support, content); } @Override @@ -58,17 +49,16 @@ public void importFilterTestData(StandaloneSupport support, FilterTest test) thr RequiredData content = test.getContent(); - LoadingUtil.importInternToExternMappers(support, test.getInternToExternMappings()); - LoadingUtil.importSearchIndexes(support, test.getSearchIndices()); - waitUntilDone(support, () -> LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept())); + importInternToExternMappers(support, test.getInternToExternMappings()); + importSearchIndexes(support, test.getSearchIndices()); + importTables(support, content.getTables(), content.isAutoConcept()); test.setConnector(ConqueryTestSpec.parseSubTree( - support, - test.getRawConnector(), - ConceptTreeConnector.class, - conn -> conn.setConcept(test.getConcept()) - ) - ); + support, + test.getRawConnector(), + ConceptTreeConnector.class, + conn -> conn.setConcept(test.getConcept()) + )); test.getConcept().setConnectors(Collections.singletonList((ConceptTreeConnector) test.getConnector())); waitUntilDone(support, () -> LoadingUtil.uploadConcept(support, support.getDataset(), test.getConcept())); @@ -76,30 +66,10 @@ public void importFilterTestData(StandaloneSupport support, FilterTest test) thr waitUntilDone(support, () -> LoadingUtil.updateMatchingStats(support)); } - @Override - public void importSecondaryIds(StandaloneSupport support, List secondaryIds) { - waitUntilDone(support, () -> LoadingUtil.importSecondaryIds(support, secondaryIds)); - } - - @Override - public void importTables(StandaloneSupport support, List tables, boolean autoConcept) throws JSONException { - waitUntilDone(support, () -> LoadingUtil.importTables(support, tables, autoConcept)); - } - - @Override - public void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException { - waitUntilDone(support, () -> LoadingUtil.importConcepts(support, rawConcepts)); - } @Override public void importTableContents(StandaloneSupport support, Collection tables) throws Exception { waitUntilDone(support, () -> LoadingUtil.importTableContents(support, tables)); } - private static void sendUpdateMatchingStatsMessage(StandaloneSupport support) { - DistributedNamespace namespace = (DistributedNamespace) support.getNamespace(); - namespace.getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(support.getNamespace().getStorage().getAllConcepts())); - support.waitUntilWorkDone(); - } - -} +} \ No newline at end of file diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java index 3a31b73d7c..019e1cb5f7 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; import com.bakdata.conquery.sql.execution.SqlExecutionService; import lombok.Getter; @@ -84,7 +85,8 @@ static void after() throws IOException { public void shouldThrowException() { // This can be removed as soon as we switch to a full integration test including the REST API I18n.init(); - SqlExecutionService executionService = new SqlExecutionService(dslContextWrapper.getDslContext(), ResultSetProcessorFactory.create(testSqlDialect)); + ResultSetProcessor resultSetProcessor = ResultSetProcessorFactory.create(config, testSqlDialect); + SqlExecutionService executionService = new SqlExecutionService(dslContextWrapper.getDslContext(), resultSetProcessor); SqlQuery validQuery = new TestSqlQuery("SELECT 1"); Assertions.assertThatNoException().isThrownBy(() -> executionService.execute(validQuery)); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java index e05a0b4acd..3846fb4286 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.integration.IntegrationTest; import com.bakdata.conquery.integration.json.ConqueryTestSpec; import com.bakdata.conquery.integration.json.JsonIntegrationTest; +import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.config.CSVConfig; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.Connector; @@ -151,7 +152,8 @@ private static SelectFilter setupSearch(StandaloneSupport conquery) throws Ex final CSVConfig csvConf = conquery.getConfig().getCsv(); - final Concept concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next(); + NamespaceStorage namespaceStorage = conquery.getNamespace().getStorage(); + final Concept concept = namespaceStorage.getAllConcepts().stream().filter(c -> c.getName().equals("geschlecht_select")).findFirst().orElseThrow(); final Connector connector = concept.getConnectors().iterator().next(); final SelectFilter filter = (SelectFilter) connector.getFilters().iterator().next(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index 5c88386bde..8d25bc0197 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -3,6 +3,8 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.*; +import jakarta.validation.Validator; + import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.jackson.Jackson; @@ -18,7 +20,6 @@ import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.jersey.validation.Validators; -import jakarta.validation.Validator; import lombok.Getter; import org.junit.jupiter.api.BeforeEach; @@ -37,12 +38,11 @@ public abstract class AbstractSerializationTest { @BeforeEach public void before() { - InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, validator); + metaStorage = new MetaStorage(new NonPersistentStoreFactory()); + InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, metaStorage, validator); final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), "emptyDefaultLabel"); final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler(new ClusterState(), config, creator); - datasetRegistry = new DatasetRegistry<>(0, config, null, clusterNamespaceHandler, indexService); - metaStorage = new MetaStorage(new NonPersistentStoreFactory(), datasetRegistry); - datasetRegistry.setMetaStorage(metaStorage); + datasetRegistry = new DatasetRegistry<>(0, config, creator, clusterNamespaceHandler, indexService); creator.init(datasetRegistry); // Prepare manager node internal mapper @@ -50,7 +50,7 @@ public void before() { when(managerNode.getConfig()).thenReturn(config); when(managerNode.getValidator()).thenReturn(validator); doReturn(datasetRegistry).when(managerNode).getDatasetRegistry(); - when(managerNode.getStorage()).thenReturn(metaStorage); + when(managerNode.getMetaStorage()).thenReturn(metaStorage); when(managerNode.getInternalObjectMapperCreator()).thenReturn(creator); when(managerNode.createInternalObjectMapper(any())).thenCallRealMethod(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java index 69f77b30f9..cc6cbebfde 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java @@ -7,14 +7,11 @@ import java.util.List; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.annotation.JsonCreator; @@ -27,8 +24,7 @@ public class IdRefrenceTest { @Test public void testListReferences() throws IOException { - final ObjectMapper mapper = Jackson.MAPPER.copy(); - mapper.setInjectableValues(new MutableInjectableValues()); + final ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); CentralRegistry registry = new CentralRegistry(); Dataset dataset = new Dataset(); @@ -39,12 +35,9 @@ public void testListReferences() throws IOException { registry.register(dataset); registry.register(table); - final DatasetRegistry datasetRegistry = new DatasetRegistry<>(0, null, null, null, null); - - final MetaStorage metaStorage = new MetaStorage(new NonPersistentStoreFactory(),datasetRegistry); + final MetaStorage metaStorage = new MetaStorage(new NonPersistentStoreFactory()); metaStorage.openStores(null); - datasetRegistry.setMetaStorage(metaStorage); User user = new User("usermail", "userlabel", metaStorage); @@ -61,14 +54,20 @@ public void testListReferences() throws IOException { .contains("\"user.usermail\"") .contains("\"dataset.table\""); - ListHolder holder = new SingletonNamespaceCollection(registry, metaStorage.getCentralRegistry()) - .injectIntoNew(mapper.readerFor(ListHolder.class)) + new SingletonNamespaceCollection(registry) + .injectInto(mapper); + metaStorage.injectInto(mapper); + ListHolder holder = mapper + .readerFor(ListHolder.class) .readValue(json); assertThat(holder.getUsers().get(0)).isSameAs(user); assertThat(holder.getTables().get(0)).isSameAs(table); } + /** + * @implNote this needs to be a class, because jackson ignores NsIdRefCollection on records + */ @Getter @RequiredArgsConstructor(onConstructor_ = @JsonCreator) public static class ListHolder { diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java index 10c92e1004..bec921c38e 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/SerializationTestUtil.java @@ -8,6 +8,7 @@ import java.lang.ref.WeakReference; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.UnaryOperator; +import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; @@ -23,7 +24,6 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import io.dropwizard.jersey.validation.Validators; -import jakarta.validation.Validator; import lombok.NonNull; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -101,11 +101,16 @@ public void test(T value, T expected) throws JSONException, IOException { } for (ObjectMapper objectMapper : objectMappers) { - test( - value, - expected, - objectMapper - ); + try { + test( + value, + expected, + objectMapper + ); + } catch (Exception e) { + Class activeView = objectMapper.getSerializationConfig().getActiveView(); + throw new IllegalStateException("Serdes failed with object mapper using view '" + activeView + "'", e); + } } } @@ -116,7 +121,7 @@ public void test(T value) throws JSONException, IOException { private void test(T value, T expected, ObjectMapper mapper) throws IOException { if (registry != null) { - mapper = new SingletonNamespaceCollection(registry, registry).injectInto(mapper); + mapper = new SingletonNamespaceCollection(registry).injectInto(mapper); } for (Injectable injectable : injectables) { mapper = injectable.injectInto(mapper); diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java index d5549e0723..3afe94bb22 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/tree/GroovyIndexedTest.java @@ -8,12 +8,10 @@ import java.util.Random; import java.util.function.Supplier; import java.util.stream.Stream; - import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Table; @@ -22,7 +20,9 @@ import com.bakdata.conquery.models.exceptions.ConfigurationException; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.identifiable.CentralRegistry; +import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.CalculatedValue; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.node.ObjectNode; import com.github.powerlibraries.io.In; @@ -87,15 +87,13 @@ public static void init() throws IOException, JSONException, ConfigurationExcept // Prepare Serdes injections - final Validator validator = Validators.newValidator(); - final ObjectReader conceptReader = new Injectable(){ - @Override - public MutableInjectableValues inject(MutableInjectableValues values) { - return values.add(Validator.class, validator); - } - }.injectInto(registry.injectIntoNew(dataset.injectIntoNew(Jackson.MAPPER))).readerFor(Concept.class); - - // load tree twice to to avoid references + ObjectMapper mapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); + ((Injectable) values -> values.add(Validator.class, Validators.newValidator())).injectInto(mapper); + new SingletonNamespaceCollection(registry).injectInto(mapper); + dataset.injectInto(mapper); + final ObjectReader conceptReader = mapper.readerFor(Concept.class); + + // load tree twice to avoid references indexedConcept = conceptReader.readValue(node); indexedConcept.setDataset(dataset); diff --git a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java index 42f895727c..ed9e8d76b9 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/identifiable/IdMapSerialisationTest.java @@ -2,13 +2,14 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; +import com.bakdata.conquery.models.identifiable.mapping.ExternalId; public class IdMapSerialisationTest { public static EntityIdMap createTestPersistentMap() { EntityIdMap entityIdMap = new EntityIdMap(); - entityIdMap.addInputMapping("test1", new EntityIdMap.ExternalId("id", "a")); + entityIdMap.addInputMapping("test1", new ExternalId("id", "a")); entityIdMap.addOutputMapping("test2", EntityPrintId.from("c")); diff --git a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java index c9622d73e1..ae13a560f6 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java +++ b/backend/src/test/java/com/bakdata/conquery/util/NonPersistentStoreFactory.java @@ -6,22 +6,14 @@ import java.util.concurrent.ConcurrentHashMap; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.storage.IdentifiableStore; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.io.storage.StoreMappings; -import com.bakdata.conquery.io.storage.WorkerStorage; +import com.bakdata.conquery.io.storage.*; import com.bakdata.conquery.io.storage.xodus.stores.CachedStore; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Role; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.StoreFactory; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.Import; -import com.bakdata.conquery.models.datasets.PreviewConfig; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.*; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.StructureNode; import com.bakdata.conquery.models.events.Bucket; @@ -33,7 +25,6 @@ import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.databind.ObjectMapper; @@ -153,12 +144,12 @@ public SingletonStore createStructureStore(String pathName, Cen } @Override - public IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) { + public IdentifiableStore createExecutionsStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { return StoreMappings.identifiable(executionStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>()), centralRegistry); } @Override - public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, DatasetRegistry datasetRegistry, String pathName, ObjectMapper objectMapper) { + public IdentifiableStore createFormConfigStore(CentralRegistry centralRegistry, String pathName, ObjectMapper objectMapper) { return StoreMappings.identifiable(formConfigStore.computeIfAbsent(pathName, n -> new NonPersistentStore<>()), centralRegistry); } @@ -181,7 +172,7 @@ public IdentifiableStore createGroupStore(CentralRegistry centralRegistry * @implNote intended for Unit-tests */ public MetaStorage createMetaStorage() { - final MetaStorage metaStorage = new MetaStorage(this, null); + final MetaStorage metaStorage = new MetaStorage(this); metaStorage.openStores(null); return metaStorage; } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java b/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java index ec595bb877..3d5dfa73a1 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/ConfigOverride.java @@ -10,18 +10,17 @@ import io.dropwizard.jetty.ConnectorFactory; import io.dropwizard.jetty.HttpConnectorFactory; import lombok.SneakyThrows; +import lombok.experimental.UtilityClass; import org.apache.commons.collections4.CollectionUtils; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.TestInstance.Lifecycle; /** * This interface allows to override the configuration used in tests. */ -@TestInstance(Lifecycle.PER_CLASS) -public interface ConfigOverride { +@UtilityClass +public final class ConfigOverride { @SneakyThrows - static void configurePathsAndLogging(ConqueryConfig config, File tmpDir) { + public static void configurePathsAndLogging(ConqueryConfig config, File tmpDir) { config.setFailOnError(true); @@ -37,7 +36,7 @@ static void configurePathsAndLogging(ConqueryConfig config, File tmpDir) { } @SneakyThrows - static void configureRandomPorts(ConqueryConfig config) { + public static void configureRandomPorts(ConqueryConfig config) { // set random open ports final Collection connectorFactories = CollectionUtils.union( @@ -54,12 +53,4 @@ static void configureRandomPorts(ConqueryConfig config) { config.getCluster().setPort(s.getLocalPort()); } } - - /** - * Is called upon initialization of the test instance of Conquery. - * - * @param config The configuration that is initialized with the defaults. - */ - void override(ConqueryConfig config); - } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java index 1abc9b0fc9..49042dada3 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java @@ -4,6 +4,11 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import jakarta.validation.Validator; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.ClientRequestContext; +import jakarta.ws.rs.client.ClientRequestFilter; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.commands.PreprocessorCommand; import com.bakdata.conquery.commands.ShardNode; @@ -23,11 +28,6 @@ import com.bakdata.conquery.resources.admin.rest.AdminProcessor; import com.google.common.util.concurrent.MoreExecutors; import io.dropwizard.core.setup.Environment; -import jakarta.validation.Validator; -import jakarta.ws.rs.client.Client; -import jakarta.ws.rs.client.ClientRequestContext; -import jakarta.ws.rs.client.ClientRequestFilter; -import jakarta.ws.rs.core.UriBuilder; import lombok.Data; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -94,7 +94,7 @@ public Validator getValidator() { } public MetaStorage getMetaStorage() { - return testConquery.getStandaloneCommand().getManagerNode().getStorage(); + return testConquery.getStandaloneCommand().getManagerNode().getMetaStorage(); } public NamespaceStorage getNamespaceStorage() { diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 4346306bc3..00f9822cd3 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -11,6 +11,8 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import jakarta.validation.Validator; +import jakarta.ws.rs.client.Client; import com.bakdata.conquery.Conquery; import com.bakdata.conquery.commands.DistributedStandaloneCommand; @@ -36,8 +38,6 @@ import io.dropwizard.client.JerseyClientBuilder; import io.dropwizard.core.cli.Command; import io.dropwizard.testing.DropwizardTestSupport; -import jakarta.validation.Validator; -import jakarta.ws.rs.client.Client; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; @@ -149,7 +149,7 @@ public void afterEach() throws Exception { } openSupports.clear(); } - this.getStandaloneCommand().getManagerNode().getStorage().clear(); + this.getStandaloneCommand().getManagerNode().getMetaStorage().clear(); waitUntilWorkDone(); } @@ -190,7 +190,7 @@ public void waitUntilWorkDone() { } public void beforeEach() { - final MetaStorage storage = standaloneCommand.getManagerNode().getStorage(); + final MetaStorage storage = standaloneCommand.getManagerNode().getMetaStorage(); testUser = standaloneCommand.getManagerNode().getConfig().getAuthorizationRealms().getInitialUsers().get(0).createOrOverwriteUser(storage); storage.updateUser(testUser); } @@ -261,7 +261,7 @@ private boolean isBusy() { boolean busy; busy = standaloneCommand.getManagerNode().getJobManager().isSlowWorkerBusy(); busy |= standaloneCommand.getManagerNode() - .getStorage() + .getMetaStorage() .getAllExecutions() .stream() .map(ManagedExecution::getState) diff --git a/backend/src/test/resources/tests/sql/yes/persons.csv b/backend/src/test/resources/shared/entities.csv similarity index 86% rename from backend/src/test/resources/tests/sql/yes/persons.csv rename to backend/src/test/resources/shared/entities.csv index 5499484b3d..514b00398a 100644 --- a/backend/src/test/resources/tests/sql/yes/persons.csv +++ b/backend/src/test/resources/shared/entities.csv @@ -1,7 +1,8 @@ pid,datum -4,2013-11-10 -5,2013-11-10 -6,2013-11-10 1,2012-01-01 2,2013-11-10 3,2013-11-10 +4,2013-11-10 +5,2013-11-10 +6,2013-11-10 +23,2014-01-01 diff --git a/backend/src/test/resources/shared/entities.table.json b/backend/src/test/resources/shared/entities.table.json new file mode 100644 index 0000000000..c6a2561f58 --- /dev/null +++ b/backend/src/test/resources/shared/entities.table.json @@ -0,0 +1,14 @@ +{ + "csv": "shared/entities.csv", + "name": "entities", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + } + ] +} diff --git a/backend/src/test/resources/tests/sql/combined/combined.json b/backend/src/test/resources/tests/sql/combined/combined.json index 3e9b99ce63..119b3c3f58 100644 --- a/backend/src/test/resources/tests/sql/combined/combined.json +++ b/backend/src/test/resources/tests/sql/combined/combined.json @@ -16,10 +16,12 @@ { "id": "concept.connector", "selects": [ + "concept.connector.exists", "concept.connector.event-date", "concept.connector.event_duration_sum", "concept.connector.first_value", - "concept.connector.sum_distinct" + "concept.connector.sum_distinct", + "concept.connector.distinct_select" ], "filters": [ { @@ -83,6 +85,10 @@ } ], "selects": [ + { + "type": "EXISTS", + "name": "exists" + }, { "type": "EVENT_DATE_UNION", "name": "event-date" @@ -96,6 +102,11 @@ "column": "table.value", "type": "FIRST" }, + { + "label": "distinct_select", + "column": "table.value", + "type": "DISTINCT" + }, { "name": "sum_distinct", "type": "SUM", diff --git a/backend/src/test/resources/tests/sql/combined/expected.csv b/backend/src/test/resources/tests/sql/combined/expected.csv index a87c0031ab..f1c7b15856 100644 --- a/backend/src/test/resources/tests/sql/combined/expected.csv +++ b/backend/src/test/resources/tests/sql/combined/expected.csv @@ -1,6 +1,6 @@ -result,dates,concept exists,concept event-date,concept event_duration_sum,concept connector event-date,concept connector event_duration_sum,concept connector first_value,concept connector sum_distinct,concept tree_label first_test_column -1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",367,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",366,1.0,2.0,A1 -2,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",9,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",8,1.01,2.01,B2 -3,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",1,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",4,{2012-01-01/2012-01-03},3,0.5,0.5,A1 -4,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",1,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",5,{2012-01-01/2012-01-04},4,0.5,0.5,B2 -5,{2012-01-01/2012-01-05},1,{2012-01-01/2012-01-05},5,{2012-01-01/2012-01-05},5,1.0,1.5, +result,dates,concept exists,concept event-date,concept event_duration_sum,concept connector exists,concept connector event-date,concept connector event_duration_sum,concept connector first_value,concept connector sum_distinct,concept connector distinct_select,concept tree_label first_test_column +1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31,2015-01-01/2015-01-01}",367,1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",366,1.0,2.0,{1.0},A1 +2,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,"{2010-07-15/2010-07-15,2012-01-01/2012-01-02,2012-01-05/2012-01-10}",9,1,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",8,1.01,2.01,"{1.0,1.01}",B2 +3,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",1,"{2012-01-01/2012-01-03,2013-11-10/2013-11-10}",4,1,{2012-01-01/2012-01-03},3,0.5,0.5,{0.5},A1 +4,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",1,"{2012-01-01/2012-01-04,2012-11-11/2012-11-11}",5,1,{2012-01-01/2012-01-04},4,0.5,0.5,{0.5},B2 +5,{2012-01-01/2012-01-05},1,{2012-01-01/2012-01-05},5,1,{2012-01-01/2012-01-05},5,1.0,1.5,"{0.5,1.0}", diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/content.csv b/backend/src/test/resources/tests/sql/selects/concept_values/content.csv new file mode 100644 index 0000000000..88a503b22b --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/concept_values/content.csv @@ -0,0 +1,8 @@ +pid,datum,test_column +1,2012-01-01,"A1" +1,2012-01-01,"A1" +1,2012-01-02,"B2" +1,2012-01-02,"B2" +2,2010-07-15,"B2" +3,2013-11-10,"A1" +4,2012-11-11,"B2" diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/content2.csv b/backend/src/test/resources/tests/sql/selects/concept_values/content2.csv new file mode 100644 index 0000000000..44d8d7121e --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/concept_values/content2.csv @@ -0,0 +1,6 @@ +pid,datum,test_column +1,2012-01-01,"B2" +1,2012-01-02,"A1" +2,2010-07-15,"B2" +3,2013-11-10,"A1" +4,2012-11-11,"B2" diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/expected.csv b/backend/src/test/resources/tests/sql/selects/concept_values/expected.csv new file mode 100644 index 0000000000..320a84c64c --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/concept_values/expected.csv @@ -0,0 +1,5 @@ +result,dates,tree select +1,{2012-01-01/2012-01-02},"{A1,B2}" +2,{2010-07-15/2010-07-15},{B2} +3,{2013-11-10/2013-11-10},{A1} +4,{2012-11-11/2012-11-11},{B2} \ No newline at end of file diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json b/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json new file mode 100644 index 0000000000..dd9193c671 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/concept_values/single_connector.json @@ -0,0 +1,112 @@ +{ + "type": "QUERY_TEST", + "label": "CONCEPT_VALUES Test with a single connector", + "expectedCsv": "tests/sql/selects/concept_values/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "CONCEPT", + "selects": [ + "tree.select" + ], + "ids": [ + "tree" + ], + "tables": [ + { + "id": "tree.test_column" + } + ] + } + }, + "concepts": [ + { + "label": "tree", + "type": "TREE", + "selects": [ + { + "type": "CONCEPT_VALUES", + "name": "select", + "asIds": false + } + ], + "connectors": [ + { + "name": "test_column", + "column": "table.test_column", + "validityDates": { + "label": "datum", + "column": "table.datum" + } + }, + { + "label": "tree_label2", + "name": "test_column2", + "column": "table2.test_column", + "validityDates": { + "label": "datum", + "column": "table2.datum" + } + } + ], + "children": [ + { + "label": "test_child1", + "condition": { + "type": "PREFIX_LIST", + "prefixes": "A" + }, + "children": [] + }, + { + "label": "test_child2", + "condition": { + "type": "PREFIX_LIST", + "prefixes": "B" + }, + "children": [] + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/concept_values/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "test_column", + "type": "STRING" + } + ] + }, + { + "csv": "tests/sql/selects/concept_values/content2.csv", + "name": "table2", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "test_column", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json b/backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json similarity index 89% rename from backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json rename to backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json index c98cc4559f..746ca1563f 100644 --- a/backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json +++ b/backend/src/test/resources/tests/sql/selects/concept_values/two_connectors.json @@ -1,7 +1,7 @@ { "type": "QUERY_TEST", - "label": "CONCEPT_VALUES Test", - "expectedCsv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/expected_raw.csv", + "label": "CONCEPT_VALUES Test with 2 connectors", + "expectedCsv": "tests/sql/selects/concept_values/expected.csv", "query": { "type": "CONCEPT_QUERY", "root": { @@ -75,7 +75,7 @@ "content": { "tables": [ { - "csv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/content.csv", + "csv": "tests/sql/selects/concept_values/content.csv", "name": "table", "primaryColumn": { "name": "pid", @@ -93,7 +93,7 @@ ] }, { - "csv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/content2.csv", + "csv": "tests/sql/selects/concept_values/content2.csv", "name": "table2", "primaryColumn": { "name": "pid", diff --git a/backend/src/test/resources/tests/sql/selects/date_union/content.csv b/backend/src/test/resources/tests/sql/selects/date_union/content.csv new file mode 100644 index 0000000000..edb160e384 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_union/content.csv @@ -0,0 +1,11 @@ +pid,indexdatum_start,indexdatum_end,geburtsdatum +1,2010-01-01,2010-03-31,2010-01-31 +2,2010-01-01,2010-03-31,1998-01-01 +3,2010-01-01,2010-03-31,1997-12-31 +4,2010-01-01,2010-03-31, +5,2010-01-01,2010-03-31,2011-01-01 +6,2010-01-01,2010-03-31,1998-03-01 +7,2010-01-01,2010-03-31,2010-01-31 +8,2010-01-01,2010-03-31,1998-03-01 +9,2010-01-01,2010-03-31,1998-04-01 +10,2010-01-01,2010-03-31,2012-01-01 diff --git a/backend/src/test/resources/tests/sql/selects/date_union/date_union.json b/backend/src/test/resources/tests/sql/selects/date_union/date_union.json new file mode 100644 index 0000000000..d2ffff5374 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_union/date_union.json @@ -0,0 +1,67 @@ +{ + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, + "label": "DATE_UNION select test", + "expectedCsv": "tests/sql/selects/date_union/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": "concept.connector.date-union" + } + ] + } + }, + "concepts": [ + { + "name": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector", + "table": "table", + "selects": { + "type": "DATE_UNION", + "name": "date-union", + "startColumn": "table.indexdatum_start", + "endColumn": "table.indexdatum_end" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/date_union/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "indexdatum_start", + "type": "DATE" + }, + { + "name": "indexdatum_end", + "type": "DATE" + }, + { + "name": "geburtsdatum", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/date_union/expected.csv b/backend/src/test/resources/tests/sql/selects/date_union/expected.csv new file mode 100644 index 0000000000..8fd8203bcf --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_union/expected.csv @@ -0,0 +1,11 @@ +result,dates,concept date-union +1,{},{2010-01-01/2010-03-31} +10,{},{2010-01-01/2010-03-31} +2,{},{2010-01-01/2010-03-31} +3,{},{2010-01-01/2010-03-31} +4,{},{2010-01-01/2010-03-31} +5,{},{2010-01-01/2010-03-31} +6,{},{2010-01-01/2010-03-31} +7,{},{2010-01-01/2010-03-31} +8,{},{2010-01-01/2010-03-31} +9,{},{2010-01-01/2010-03-31} diff --git a/backend/src/test/resources/tests/sql/selects/distinct/content.csv b/backend/src/test/resources/tests/sql/selects/distinct/content.csv new file mode 100644 index 0000000000..03265056db --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/distinct/content.csv @@ -0,0 +1,14 @@ +pid,datum,value +1,2012-01-01,"f" +1,2012-01-02,"f" +2,2010-07-15, +3,2012-01-01,"f" +3,2012-01-02,"m" +4,2012-01-01,"f" +4,2012-01-02,"m" +4,2012-01-03,"f" +4,2012-01-04,"m" +5,2012-01-01,"f " +5,2012-01-02,"m " +5,2012-01-03,"f" +5,2012-01-04,"m" diff --git a/backend/src/test/resources/tests/sql/selects/distinct/distinct.json b/backend/src/test/resources/tests/sql/selects/distinct/distinct.json new file mode 100644 index 0000000000..4ff17cd780 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/distinct/distinct.json @@ -0,0 +1,65 @@ +{ + "type": "QUERY_TEST", + "label": "DISTINCT select", + "expectedCsv": "tests/sql/selects/distinct/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": [ + "concept.connector.select" + ] + } + ] + } + }, + "concepts": [ + { + "label": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector", + "table": "table", + "validityDates": { + "label": "datum", + "column": "table.datum" + }, + "selects": { + "name": "select", + "type": "DISTINCT", + "column": "table.value" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/distinct/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "value", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/distinct/expected.csv b/backend/src/test/resources/tests/sql/selects/distinct/expected.csv new file mode 100644 index 0000000000..6b7033299b --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/distinct/expected.csv @@ -0,0 +1,6 @@ +result,dates,concept select +1,{2012-01-01/2012-01-02},{f} +2,{2010-07-15/2010-07-15}, +3,{2012-01-01/2012-01-02},"{f,m}" +4,{2012-01-01/2012-01-04},"{f,m}" +5,{2012-01-01/2012-01-04},"{f,f ,m,m }" diff --git a/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/content.csv b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/content.csv new file mode 100644 index 0000000000..f3fe352607 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/content.csv @@ -0,0 +1,8 @@ +pid,datum_start,datum_end +1,2014-06-30,2015-06-30 +2,2014-06-30,2015-06-30 +2,2014-06-30,2015-06-30 +3,2014-06-30,2015-06-30 +4,2014-06-30,2015-06-30 +5,2014-06-30,2015-06-30 +5,2014-06-30,2015-06-30 diff --git a/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/exists.spec.json b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/exists.spec.json new file mode 100644 index 0000000000..c3592c4365 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/exists.spec.json @@ -0,0 +1,61 @@ +{ + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, + "label": "EXISTS select on connector", + "expectedCsv": "tests/sql/selects/exists/exists_on_connector/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": "concept.connector.exists" + } + ] + } + }, + "concepts": [ + { + "name": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector", + "table": "exists_table", + "selects": { + "type": "EXISTS", + "name": "exists" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/exists/exists_on_connector/content.csv", + "name": "exists_table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/expected.csv b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/expected.csv new file mode 100644 index 0000000000..2760828d6b --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/exists/exists_on_connector/expected.csv @@ -0,0 +1,6 @@ +result,dates,concept exists +2,{},1 +4,{},1 +3,{},1 +5,{},1 +1,{},1 diff --git a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json index 088565c6c6..b6862fc9e0 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json +++ b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/duration_sum.json @@ -45,13 +45,10 @@ "connectors": { "name": "connector", "column": "table.column", - "validityDates": { - "name": "datum", - "column": "table.datum" - }, "selects": { - "type": "EVENT_DURATION_SUM", - "name": "event_duration_sum" + "type": "DURATION_SUM", + "name": "event_duration_sum", + "column": "table.datum" } }, "children": [ diff --git a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv index 9d58b233f6..50cc9ee49a 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv +++ b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv @@ -1,6 +1,6 @@ result,dates,tree a event_duration_sum -2,{2012-01-01/2012-01-01},1 -3,{2012-01-01/2012-01-02},1 -4,{2012-01-01/2012-01-02},2 -5,{2012-01-01/2012-01-05},4 -6,{-∞/+∞}, +2,{},1 +3,{},1 +4,{},2 +5,{},4 +6,{}, diff --git a/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/content.csv b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/content.csv new file mode 100644 index 0000000000..f70ad54470 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/content.csv @@ -0,0 +1,25 @@ +pid,datum,column +1,2012-01-01,"A" +1,2012-01-01, + +2,2012-01-01,"A" +2,2012-01-01,"B" + +3,2012-01-01,"A" +3,2012-01-02,"B" + +4,2012-01-01,"A" +4,2012-01-02,"A" +4,2012-01-02,"B" + +5,2012-01-01,"A" +5,2012-01-02,"A" +5,2012-01-03,"A" +5,2012-01-04,"A" +5,2012-01-02,"B" +5,2012-01-03,"B" +5,2012-01-04,"B" +5,2012-01-05,"B" + +6,,"A" +6,,"B" diff --git a/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json new file mode 100644 index 0000000000..8e9a14690d --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/duration_sum.json @@ -0,0 +1,99 @@ +{ + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, + "label": "AND DURATION SUM on event date Test", + "expectedCsv": "tests/sql/selects/sum/event_duration_sum/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "CONCEPT", + "ids": [ + "tree.a" + ], + "tables": [ + { + "id": "tree.connector", + "selects": [ + "tree.connector.event_duration_sum" + ] + } + ] + }, + { + "type": "CONCEPT", + "ids": [ + "tree.b" + ], + "tables": [ + { + "id": "tree.connector" + } + ] + } + ] + } + }, + "concepts": [ + { + "name": "tree", + "type": "TREE", + "connectors": { + "name": "connector", + "column": "table.column", + "validityDates": { + "name": "datum", + "column": "table.datum" + }, + "selects": { + "type": "EVENT_DURATION_SUM", + "name": "event_duration_sum" + } + }, + "children": [ + { + "name": "a", + "condition": { + "type": "PREFIX_LIST", + "prefixes": "A" + }, + "children": [] + }, + { + "name": "b", + "condition": { + "type": "PREFIX_LIST", + "prefixes": "B" + }, + "children": [] + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/sum/event_duration_sum/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "column", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/expected.csv b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/expected.csv new file mode 100644 index 0000000000..9d58b233f6 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/event_duration_sum/expected.csv @@ -0,0 +1,6 @@ +result,dates,tree a event_duration_sum +2,{2012-01-01/2012-01-01},1 +3,{2012-01-01/2012-01-02},1 +4,{2012-01-01/2012-01-02},2 +5,{2012-01-01/2012-01-05},4 +6,{-∞/+∞}, diff --git a/backend/src/test/resources/tests/sql/yes/CQYES.test.json b/backend/src/test/resources/tests/sql/yes/cqyes.json similarity index 54% rename from backend/src/test/resources/tests/sql/yes/CQYES.test.json rename to backend/src/test/resources/tests/sql/yes/cqyes.json index e5b9e5fe3b..e5d28e42d0 100644 --- a/backend/src/test/resources/tests/sql/yes/CQYES.test.json +++ b/backend/src/test/resources/tests/sql/yes/cqyes.json @@ -10,28 +10,11 @@ }, "concepts": [ ], - "idColumns": { - "table": "persons", - "ids": [ - { - "name": "PID", - "field": "pid", - "primaryId": true, - "label": { - "de": "PID" - }, - "description": { - "en": "Pid of an Entity.", - "de": "Pid einer Entität." - } - } - ] - }, "content": { "tables": [ { - "csv": "tests/sql/yes/persons.csv", - "name": "persons", + "csv": "tests/sql/yes/no-op.csv", + "name": "no-op", "primaryColumn": { "name": "pid", "type": "STRING" diff --git a/backend/src/test/resources/tests/sql/yes/expected.csv b/backend/src/test/resources/tests/sql/yes/expected.csv index edf163f57b..0a872351ef 100644 --- a/backend/src/test/resources/tests/sql/yes/expected.csv +++ b/backend/src/test/resources/tests/sql/yes/expected.csv @@ -1,7 +1,8 @@ -PID,dates -4,{} -5,{} -6,{} +result,dates 1,{} 2,{} +23,{} 3,{} +4,{} +5,{} +6,{} diff --git a/backend/src/test/resources/tests/sql/yes/no-op.csv b/backend/src/test/resources/tests/sql/yes/no-op.csv new file mode 100644 index 0000000000..d9a4329883 --- /dev/null +++ b/backend/src/test/resources/tests/sql/yes/no-op.csv @@ -0,0 +1,2 @@ +pid,datum +irrelevant_id_for_cqyes,2013-11-10 diff --git a/pom.xml b/pom.xml index 28d426aa1b..8c7c5ae5d4 100644 --- a/pom.xml +++ b/pom.xml @@ -18,7 +18,7 @@ 17 - 1.18.22 + 1.18.24 UTF-8 ${java.required} ${java.required}