From f490ce110cf7686c5a49b410cf9708b5e99fcd8c Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 31 Jul 2024 12:55:03 +0200 Subject: [PATCH 01/40] puts mina related method in separate class --- .../bakdata/conquery/commands/ShardNode.java | 275 +---------------- .../mode/cluster/ClusterConnectionShard.java | 286 ++++++++++++++++++ .../network/NetworkMessageContext.java | 4 +- 3 files changed, 303 insertions(+), 262 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index 26e41c8945..93bd96584d 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -1,38 +1,21 @@ package com.bakdata.conquery.commands; -import java.net.InetSocketAddress; import java.util.Collection; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.View; -import com.bakdata.conquery.io.mina.BinaryJacksonCoder; -import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; -import com.bakdata.conquery.io.mina.ChunkReader; -import com.bakdata.conquery.io.mina.ChunkWriter; -import com.bakdata.conquery.io.mina.NetworkSession; import com.bakdata.conquery.io.storage.WorkerStorage; +import com.bakdata.conquery.mode.cluster.ClusterConnectionShard; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; -import com.bakdata.conquery.models.jobs.JobManagerStatus; -import com.bakdata.conquery.models.jobs.ReactingJob; import com.bakdata.conquery.models.jobs.SimpleJob; -import com.bakdata.conquery.models.messages.SlowMessage; -import com.bakdata.conquery.models.messages.network.MessageToShardNode; -import com.bakdata.conquery.models.messages.network.NetworkMessageContext; -import com.bakdata.conquery.models.messages.network.NetworkMessageContext.ShardNodeNetworkContext; -import com.bakdata.conquery.models.messages.network.specific.AddShardNode; -import com.bakdata.conquery.models.messages.network.specific.RegisterWorker; -import com.bakdata.conquery.models.messages.network.specific.UpdateJobManagerStatus; -import com.bakdata.conquery.models.worker.IdResolveContext; import com.bakdata.conquery.models.worker.Worker; -import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.Workers; import com.bakdata.conquery.util.io.ConqueryMDC; import com.fasterxml.jackson.databind.DeserializationConfig; @@ -41,18 +24,9 @@ import io.dropwizard.core.ConfiguredBundle; import io.dropwizard.core.setup.Environment; import io.dropwizard.lifecycle.Managed; -import io.dropwizard.util.Duration; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; -import org.apache.mina.core.RuntimeIoException; -import org.apache.mina.core.future.ConnectFuture; -import org.apache.mina.core.service.IoHandler; -import org.apache.mina.core.session.IdleStatus; -import org.apache.mina.core.session.IoSession; -import org.apache.mina.filter.FilterEvent; -import org.apache.mina.transport.socket.nio.NioSocketConnector; -import org.jetbrains.annotations.NotNull; /** * This node holds a shard of data (in so called {@link Worker}s for the different datasets in conquery. @@ -61,23 +35,15 @@ */ @Slf4j @Getter -public class ShardNode implements ConfiguredBundle, IoHandler, Managed { +public class ShardNode implements ConfiguredBundle, Managed { public static final String DEFAULT_NAME = "shard-node"; private final String name; - - private NioSocketConnector connector; - private ConnectFuture future; private JobManager jobManager; - private Validator validator; - private ConqueryConfig config; - private ShardNodeNetworkContext context; @Setter private Workers workers; - @Setter - private ScheduledExecutorService scheduler; - private Environment environment; + private ClusterConnectionShard clusterConnection; public ShardNode() { this(DEFAULT_NAME); @@ -90,27 +56,24 @@ public ShardNode(String name) { @Override public void run(ConqueryConfig config, Environment environment) throws Exception { - this.environment = environment; - this.config = config; - jobManager = new JobManager(getName(), config.isFailOnError()); environment.lifecycle().manage(this); - validator = environment.getValidator(); - - scheduler = environment.lifecycle().scheduledExecutorService("Scheduled Messages").build(); - - scheduler.scheduleAtFixedRate(this::reportJobManagerStatus, 30, 1, TimeUnit.SECONDS); workers = new Workers( - getConfig().getQueries().getExecutionPool(), - () -> createInternalObjectMapper(View.Persistence.Shard.class), - () -> createInternalObjectMapper(View.InternalCommunication.class), - getConfig().getCluster().getEntityBucketSize(), - getConfig().getQueries().getSecondaryIdSubPlanRetention() + config.getQueries().getExecutionPool(), + () -> createInternalObjectMapper(View.Persistence.Shard.class, config, environment.getValidator()), + () -> createInternalObjectMapper(View.InternalCommunication.class, config, environment.getValidator()), + config.getCluster().getEntityBucketSize(), + config.getQueries().getSecondaryIdSubPlanRetention() ); + clusterConnection = + new ClusterConnectionShard(config, environment, workers, jobManager, () -> createInternalObjectMapper(View.InternalCommunication.class, config, environment.getValidator())); + + environment.lifecycle().manage(clusterConnection); + final Collection workerStorages = config.getStorage().discoverWorkerStorages(); @@ -142,35 +105,6 @@ public void run(ConqueryConfig config, Environment environment) throws Exception log.info("All Worker loaded: {}", workers.getWorkers().size()); } - private void reportJobManagerStatus() { - if (context == null || !context.isConnected()) { - return; - } - - - // Collect the ShardNode and all its workers jobs into a single queue - - for (Worker worker : workers.getWorkers().values()) { - final JobManagerStatus jobManagerStatus = new JobManagerStatus( - null, worker.getInfo().getDataset(), - worker.getJobManager().getJobStatus() - ); - - try { - context.trySend(new UpdateJobManagerStatus(jobManagerStatus)); - } - catch (Exception e) { - log.warn("Failed to report job manager status", e); - - if (config.isFailOnError()) { - System.exit(1); - } - } - } - - - } - /** * Pendant to {@link ManagerNode#createInternalObjectMapper(Class)}. *

@@ -178,12 +112,12 @@ private void reportJobManagerStatus() { * * @return a preconfigured binary object mapper */ - public ObjectMapper createInternalObjectMapper(Class viewClass) { - final ObjectMapper objectMapper = getConfig().configureObjectMapper(Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER)); + private static ObjectMapper createInternalObjectMapper(Class viewClass, ConqueryConfig config, Validator validator) { + final ObjectMapper objectMapper = config.configureObjectMapper(Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER)); final MutableInjectableValues injectableValues = new MutableInjectableValues(); objectMapper.setInjectableValues(injectableValues); - injectableValues.add(Validator.class, getValidator()); + injectableValues.add(Validator.class, validator); // Set serialization config @@ -203,197 +137,20 @@ public ObjectMapper createInternalObjectMapper(Class viewClass) return objectMapper; } - @Override - public void messageReceived(IoSession session, Object message) { - setLocation(session); - if (!(message instanceof MessageToShardNode)) { - log.error("Unknown message type {} in {}", message.getClass(), message); - return; - } - - log.trace("{} received {} from {}", getName(), message.getClass().getSimpleName(), session.getRemoteAddress()); - ReactingJob job = new ReactingJob<>((MessageToShardNode) message, context); - - if (message instanceof SlowMessage slowMessage) { - slowMessage.setProgressReporter(job.getProgressReporter()); - jobManager.addSlowJob(job); - } - else { - jobManager.addFastJob(job); - } - } - - private static void setLocation(IoSession session) { - final String loc = session.getLocalAddress().toString(); - ConqueryMDC.setLocation(loc); - } - - @Override - public void exceptionCaught(IoSession session, Throwable cause) { - setLocation(session); - log.error("Exception caught", cause); - } - - @Override - public void sessionOpened(IoSession session) { - setLocation(session); - NetworkSession networkSession = new NetworkSession(session); - - context = new NetworkMessageContext.ShardNodeNetworkContext(this, networkSession, workers, config, validator); - log.info("Connected to ManagerNode @ `{}`", session.getRemoteAddress()); - - // Authenticate with ManagerNode - context.send(new AddShardNode()); - - for (Worker w : workers.getWorkers().values()) { - w.setSession(new NetworkSession(session)); - WorkerInformation info = w.getInfo(); - log.info("Sending worker identity '{}'", info.getName()); - networkSession.send(new RegisterWorker(info)); - } - - scheduleIdleLogger(scheduler, session, config.getCluster().getIdleTimeOut()); - } - - private static void scheduleIdleLogger(ScheduledExecutorService scheduler, IoSession session, Duration timeout) { - scheduler.scheduleAtFixedRate( - () -> { - setLocation(session); - - final Duration elapsed = Duration.milliseconds(System.currentTimeMillis() - session.getLastIoTime()); - if (elapsed.compareTo(timeout) > 0) { - log.warn("No message sent or received since {}", elapsed); - } - }, - timeout.toSeconds(), timeout.toSeconds() / 2, TimeUnit.SECONDS - ); - } - - @Override - public void sessionClosed(IoSession session) { - setLocation(session); - log.info("Disconnected from ManagerNode."); - - scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS); - } - - @Override - public void sessionCreated(IoSession session) { - setLocation(session); - log.debug("Session created."); - } - - @Override - public void sessionIdle(IoSession session, IdleStatus status) { - setLocation(session); - log.warn("Session idle {}.", status); - } - - @Override - public void messageSent(IoSession session, Object message) { - setLocation(session); - log.trace("Message sent: {}", message); - } - - @Override - public void inputClosed(IoSession session) { - setLocation(session); - log.info("Input closed."); - session.closeNow(); - scheduler.schedule(this::disconnectFromCluster, 0, TimeUnit.SECONDS); - } - - @Override - public void event(IoSession session, FilterEvent event) throws Exception { - setLocation(session); - log.trace("Event handled: {}", event); - } - @Override public void start() throws Exception { for (Worker value : workers.getWorkers().values()) { value.getJobManager().addSlowJob(new SimpleJob("Update Bucket Manager", value.getBucketManager()::fullUpdate)); } - scheduler.schedule(this::connectToCluster, 0, TimeUnit.MINUTES); - } - private void connectToCluster() { - InetSocketAddress address = new InetSocketAddress( - config.getCluster().getManagerURL().getHostAddress(), - config.getCluster().getPort() - ); - - disconnectFromCluster(); - - connector = getClusterConnector(workers); - - while (true) { - try { - log.info("Trying to connect to {}", address); - - // Try opening a connection (Note: This fails immediately instead of waiting a minute to try and connect) - future = connector.connect(address); - - future.awaitUninterruptibly(); - - if (future.isConnected()) { - break; - } - - future.cancel(); - // Sleep thirty seconds then retry. - TimeUnit.SECONDS.sleep(config.getCluster().getConnectRetryTimeout().toSeconds()); - - } - catch (RuntimeIoException e) { - log.warn("Failed to connect to {}", address, e); - } - catch (InterruptedException e) { - log.warn("Interrupted while trying to connector to cluster, giving up.", e); - break; - } - } - } - - @NotNull - private NioSocketConnector getClusterConnector(IdResolveContext workers) { - ObjectMapper om = createInternalObjectMapper(View.InternalCommunication.class); - - NioSocketConnector connector = new NioSocketConnector(); - - BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, validator, om); - connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om))); - connector.setHandler(this); - connector.getSessionConfig().setAll(config.getCluster().getMina()); - return connector; - } - @Override public void stop() throws Exception { getJobManager().close(); workers.stop(); - - disconnectFromCluster(); - } - - private void disconnectFromCluster() { - if (future != null) { - future.cancel(); - } - - //after the close command was send - if (context != null) { - context.awaitClose(); - } - - if (connector != null) { - log.info("Connection was closed by ManagerNode"); - connector.dispose(); - } } public boolean isBusy() { diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java new file mode 100644 index 0000000000..b4e696b6d6 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -0,0 +1,286 @@ +package com.bakdata.conquery.mode.cluster; + +import java.net.InetSocketAddress; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; + +import com.bakdata.conquery.io.mina.BinaryJacksonCoder; +import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; +import com.bakdata.conquery.io.mina.ChunkReader; +import com.bakdata.conquery.io.mina.ChunkWriter; +import com.bakdata.conquery.io.mina.NetworkSession; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.jobs.JobManagerStatus; +import com.bakdata.conquery.models.jobs.ReactingJob; +import com.bakdata.conquery.models.messages.SlowMessage; +import com.bakdata.conquery.models.messages.network.MessageToShardNode; +import com.bakdata.conquery.models.messages.network.NetworkMessageContext; +import com.bakdata.conquery.models.messages.network.specific.AddShardNode; +import com.bakdata.conquery.models.messages.network.specific.RegisterWorker; +import com.bakdata.conquery.models.messages.network.specific.UpdateJobManagerStatus; +import com.bakdata.conquery.models.worker.IdResolveContext; +import com.bakdata.conquery.models.worker.Worker; +import com.bakdata.conquery.models.worker.WorkerInformation; +import com.bakdata.conquery.models.worker.Workers; +import com.bakdata.conquery.util.io.ConqueryMDC; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.dropwizard.core.setup.Environment; +import io.dropwizard.lifecycle.Managed; +import io.dropwizard.util.Duration; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.mina.core.RuntimeIoException; +import org.apache.mina.core.future.ConnectFuture; +import org.apache.mina.core.service.IoHandler; +import org.apache.mina.core.session.IdleStatus; +import org.apache.mina.core.session.IoSession; +import org.apache.mina.filter.FilterEvent; +import org.apache.mina.transport.socket.nio.NioSocketConnector; +import org.jetbrains.annotations.NotNull; + +/** + * Pendant to {@link ClusterConnectionManager} + */ +@RequiredArgsConstructor +@Slf4j +public class ClusterConnectionShard implements Managed, IoHandler { + + private final ConqueryConfig config; + private final Environment environment; + private final Workers workers; + private final JobManager jobManager; + private final Supplier communicationMapperSupplier; + + private final ScheduledExecutorService scheduler = environment.lifecycle().scheduledExecutorService("cluster-connection-shard").build(); + + private NioSocketConnector connector; + private ConnectFuture future; + private NetworkMessageContext.ShardNodeNetworkContext context; + + @Override + public void sessionCreated(IoSession session) { + setLocation(session); + log.debug("Session created."); + } + + + @Override + public void sessionOpened(IoSession session) { + setLocation(session); + NetworkSession networkSession = new NetworkSession(session); + + context = new NetworkMessageContext.ShardNodeNetworkContext(networkSession, workers, config, environment.getValidator()); + log.info("Connected to ManagerNode @ `{}`", session.getRemoteAddress()); + + // Authenticate with ManagerNode + context.send(new AddShardNode()); + + for (Worker w : workers.getWorkers().values()) { + w.setSession(new NetworkSession(session)); + WorkerInformation info = w.getInfo(); + log.info("Sending worker identity '{}'", info.getName()); + networkSession.send(new RegisterWorker(info)); + } + + scheduleIdleLogger(scheduler, session, config.getCluster().getIdleTimeOut()); + } + + @Override + public void sessionClosed(IoSession session) { + setLocation(session); + log.info("Disconnected from ManagerNode."); + + scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS); + } + + @Override + public void sessionIdle(IoSession session, IdleStatus status) { + setLocation(session); + log.warn("Session idle {}.", status); + } + + @Override + public void exceptionCaught(IoSession session, Throwable cause) { + setLocation(session); + log.error("Exception caught", cause); + } + + + @Override + public void messageReceived(IoSession session, Object message) { + setLocation(session); + if (!(message instanceof MessageToShardNode)) { + log.error("Unknown message type {} in {}", message.getClass(), message); + return; + } + + log.trace("{} received {} from {}", environment.getName(), message.getClass().getSimpleName(), session.getRemoteAddress()); + ReactingJob job = new ReactingJob<>((MessageToShardNode) message, context); + + if (message instanceof SlowMessage slowMessage) { + slowMessage.setProgressReporter(job.getProgressReporter()); + jobManager.addSlowJob(job); + } + else { + jobManager.addFastJob(job); + } + } + + + @Override + public void messageSent(IoSession session, Object message) { + setLocation(session); + log.trace("Message sent: {}", message); + } + + + @Override + public void inputClosed(IoSession session) { + setLocation(session); + log.info("Input closed."); + session.closeNow(); + scheduler.schedule(this::disconnectFromCluster, 0, TimeUnit.SECONDS); + } + + + @Override + public void event(IoSession session, FilterEvent event) throws Exception { + setLocation(session); + log.trace("Event handled: {}", event); + } + + + private static void setLocation(IoSession session) { + final String loc = session.getLocalAddress().toString(); + ConqueryMDC.setLocation(loc); + } + + + private void connectToCluster() { + InetSocketAddress address = new InetSocketAddress( + config.getCluster().getManagerURL().getHostAddress(), + config.getCluster().getPort() + ); + + disconnectFromCluster(); + + connector = getClusterConnector(workers); + + while (true) { + try { + log.info("Trying to connect to {}", address); + + // Try opening a connection (Note: This fails immediately instead of waiting a minute to try and connect) + future = connector.connect(address); + + future.awaitUninterruptibly(); + + if (future.isConnected()) { + break; + } + + future.cancel(); + // Sleep thirty seconds then retry. + TimeUnit.SECONDS.sleep(config.getCluster().getConnectRetryTimeout().toSeconds()); + + } + catch (RuntimeIoException e) { + log.warn("Failed to connect to {}", address, e); + } + catch (InterruptedException e) { + log.warn("Interrupted while trying to connector to cluster, giving up.", e); + break; + } + } + } + + + private void disconnectFromCluster() { + if (future != null) { + future.cancel(); + } + + //after the close command was send + if (context != null) { + context.awaitClose(); + } + + if (connector != null) { + log.info("Connection was closed by ManagerNode"); + connector.dispose(); + } + } + + + @NotNull + private NioSocketConnector getClusterConnector(IdResolveContext workers) { + ObjectMapper om = communicationMapperSupplier.get(); + + NioSocketConnector connector = new NioSocketConnector(); + + BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, environment.getValidator(), om); + connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om))); + connector.setHandler(this); + connector.getSessionConfig().setAll(config.getCluster().getMina()); + return connector; + } + + + private static void scheduleIdleLogger(ScheduledExecutorService scheduler, IoSession session, Duration timeout) { + scheduler.scheduleAtFixedRate( + () -> { + setLocation(session); + + final Duration elapsed = Duration.milliseconds(System.currentTimeMillis() - session.getLastIoTime()); + if (elapsed.compareTo(timeout) > 0) { + log.warn("No message sent or received since {}", elapsed); + } + }, + timeout.toSeconds(), timeout.toSeconds() / 2, TimeUnit.SECONDS + ); + } + + + private void reportJobManagerStatus() { + if (context == null || !context.isConnected()) { + return; + } + + + // Collect the ShardNode and all its workers jobs into a single queue + + for (Worker worker : workers.getWorkers().values()) { + final JobManagerStatus jobManagerStatus = new JobManagerStatus( + null, worker.getInfo().getDataset(), + worker.getJobManager().getJobStatus() + ); + + try { + context.trySend(new UpdateJobManagerStatus(jobManagerStatus)); + } + catch (Exception e) { + log.warn("Failed to report job manager status", e); + + if (config.isFailOnError()) { + System.exit(1); + } + } + } + } + + @Override + public void start() throws Exception { + // Connect async as the manager might not be up jet or is started by a test in succession + scheduler.schedule(this::connectToCluster, 0, TimeUnit.MINUTES); + + scheduler.scheduleAtFixedRate(this::reportJobManagerStatus, 30, 1, TimeUnit.SECONDS); + + } + + @Override + public void stop() throws Exception { + disconnectFromCluster(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java index 499de2d11b..f975d9a3b8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java @@ -32,15 +32,13 @@ public boolean isConnected() { @Getter public static class ShardNodeNetworkContext extends NetworkMessageContext { - private final ShardNode shardNode; private final Workers workers; private final ConqueryConfig config; private final Validator validator; private final NetworkSession rawSession; - public ShardNodeNetworkContext(ShardNode shardNode, NetworkSession session, Workers workers, ConqueryConfig config, Validator validator) { + public ShardNodeNetworkContext(NetworkSession session, Workers workers, ConqueryConfig config, Validator validator) { super(session, config.getCluster().getBackpressure()); - this.shardNode = shardNode; this.workers = workers; this.config = config; this.validator = validator; From bcc3e1b623f2d6761e8beab4e0fc4a29848f6510 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 31 Jul 2024 13:30:29 +0200 Subject: [PATCH 02/40] include scheduler into ClusterConnectionShard --- .../conquery/mode/cluster/ClusterConnectionShard.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java index b4e696b6d6..960b5f0ba4 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -53,8 +53,7 @@ public class ClusterConnectionShard implements Managed, IoHandler { private final JobManager jobManager; private final Supplier communicationMapperSupplier; - private final ScheduledExecutorService scheduler = environment.lifecycle().scheduledExecutorService("cluster-connection-shard").build(); - + private ScheduledExecutorService scheduler; private NioSocketConnector connector; private ConnectFuture future; private NetworkMessageContext.ShardNodeNetworkContext context; @@ -272,6 +271,7 @@ private void reportJobManagerStatus() { @Override public void start() throws Exception { + scheduler = environment.lifecycle().scheduledExecutorService("cluster-connection-shard").build(); // Connect async as the manager might not be up jet or is started by a test in succession scheduler.schedule(this::connectToCluster, 0, TimeUnit.MINUTES); @@ -282,5 +282,6 @@ public void start() throws Exception { @Override public void stop() throws Exception { disconnectFromCluster(); + scheduler.shutdown(); } } From 6bb6066487962210fdd7c69c242572fd35c542c4 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 31 Jul 2024 14:49:46 +0200 Subject: [PATCH 03/40] use MdcFilter for Mina filter chain --- .../bakdata/conquery/io/mina/MdcFilter.java | 47 +++++++++++++++++++ .../cluster/ClusterConnectionManager.java | 14 +++--- .../mode/cluster/ClusterConnectionShard.java | 21 +-------- 3 files changed, 57 insertions(+), 25 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/io/mina/MdcFilter.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/MdcFilter.java b/backend/src/main/java/com/bakdata/conquery/io/mina/MdcFilter.java new file mode 100644 index 0000000000..f79dc4d820 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/MdcFilter.java @@ -0,0 +1,47 @@ +package com.bakdata.conquery.io.mina; + +import com.bakdata.conquery.util.io.ConqueryMDC; +import lombok.RequiredArgsConstructor; +import org.apache.mina.core.filterchain.IoFilterEvent; +import org.apache.mina.filter.util.CommonEventFilter; + +@RequiredArgsConstructor +public class MdcFilter extends CommonEventFilter { + + private final ThreadLocal callDepth = ThreadLocal.withInitial(() -> 0); + + private final String locationFmt; + + /** + * Adapted from {@link org.apache.mina.filter.logging.MdcInjectionFilter} + */ + @Override + protected void filter(IoFilterEvent event) throws Exception { + + // since this method can potentially call into itself + // we need to check the call depth before clearing the MDC + int currentCallDepth = callDepth.get(); + callDepth.set(currentCallDepth + 1); + + if (currentCallDepth == 0) { + /* copy context to the MDC when necessary. */ + ConqueryMDC.setLocation(String.format(locationFmt, event.getSession().getLocalAddress().toString())); + } + + try { + /* propagate event down the filter chain */ + event.fire(); + } + finally { + if (currentCallDepth == 0) { + /* remove context from the MDC */ + ConqueryMDC.clearLocation(); + + callDepth.remove(); + } + else { + callDepth.set(currentCallDepth); + } + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java index 04f622708c..1d1ed236b4 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java @@ -5,7 +5,13 @@ import jakarta.validation.Validator; import com.bakdata.conquery.io.jackson.View; -import com.bakdata.conquery.io.mina.*; +import com.bakdata.conquery.io.mina.BinaryJacksonCoder; +import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; +import com.bakdata.conquery.io.mina.ChunkReader; +import com.bakdata.conquery.io.mina.ChunkWriter; +import com.bakdata.conquery.io.mina.MdcFilter; +import com.bakdata.conquery.io.mina.MinaAttributes; +import com.bakdata.conquery.io.mina.NetworkSession; import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.Job; @@ -16,7 +22,6 @@ import com.bakdata.conquery.models.messages.network.NetworkMessageContext; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.DistributedNamespace; -import com.bakdata.conquery.util.io.ConqueryMDC; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -44,25 +49,21 @@ public class ClusterConnectionManager extends IoHandlerAdapter { @Override public void sessionOpened(IoSession session) { - ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); log.info("New client {} connected, waiting for identity", session.getRemoteAddress()); } @Override public void sessionClosed(IoSession session) { - ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); log.info("Client '{}' disconnected ", session.getAttribute(MinaAttributes.IDENTIFIER)); } @Override public void exceptionCaught(IoSession session, Throwable cause) { - ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); log.error("caught exception", cause); } @Override public void messageReceived(IoSession session, Object message) { - ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); if (message instanceof MessageToManagerNode toManagerNode) { log.trace("ManagerNode received {} from {}", message.getClass().getSimpleName(), session.getRemoteAddress()); @@ -90,6 +91,7 @@ public void messageReceived(IoSession session, Object message) { public void start() throws IOException { acceptor = new NioSocketAcceptor(); + acceptor.getFilterChain().addFirst("mdc", new MdcFilter("Manager[%s]")); ObjectMapper om = internalObjectMapperCreator.createInternalObjectMapper(View.InternalCommunication.class); config.configureObjectMapper(om); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java index 960b5f0ba4..b61d8df93c 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; import com.bakdata.conquery.io.mina.ChunkReader; import com.bakdata.conquery.io.mina.ChunkWriter; +import com.bakdata.conquery.io.mina.MdcFilter; import com.bakdata.conquery.io.mina.NetworkSession; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; @@ -24,7 +25,6 @@ import com.bakdata.conquery.models.worker.Worker; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.Workers; -import com.bakdata.conquery.util.io.ConqueryMDC; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.core.setup.Environment; import io.dropwizard.lifecycle.Managed; @@ -60,14 +60,12 @@ public class ClusterConnectionShard implements Managed, IoHandler { @Override public void sessionCreated(IoSession session) { - setLocation(session); log.debug("Session created."); } @Override public void sessionOpened(IoSession session) { - setLocation(session); NetworkSession networkSession = new NetworkSession(session); context = new NetworkMessageContext.ShardNodeNetworkContext(networkSession, workers, config, environment.getValidator()); @@ -88,7 +86,6 @@ public void sessionOpened(IoSession session) { @Override public void sessionClosed(IoSession session) { - setLocation(session); log.info("Disconnected from ManagerNode."); scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS); @@ -96,20 +93,17 @@ public void sessionClosed(IoSession session) { @Override public void sessionIdle(IoSession session, IdleStatus status) { - setLocation(session); log.warn("Session idle {}.", status); } @Override public void exceptionCaught(IoSession session, Throwable cause) { - setLocation(session); log.error("Exception caught", cause); } @Override public void messageReceived(IoSession session, Object message) { - setLocation(session); if (!(message instanceof MessageToShardNode)) { log.error("Unknown message type {} in {}", message.getClass(), message); return; @@ -130,14 +124,12 @@ public void messageReceived(IoSession session, Object message) { @Override public void messageSent(IoSession session, Object message) { - setLocation(session); log.trace("Message sent: {}", message); } @Override public void inputClosed(IoSession session) { - setLocation(session); log.info("Input closed."); session.closeNow(); scheduler.schedule(this::disconnectFromCluster, 0, TimeUnit.SECONDS); @@ -146,17 +138,9 @@ public void inputClosed(IoSession session) { @Override public void event(IoSession session, FilterEvent event) throws Exception { - setLocation(session); log.trace("Event handled: {}", event); } - - private static void setLocation(IoSession session) { - final String loc = session.getLocalAddress().toString(); - ConqueryMDC.setLocation(loc); - } - - private void connectToCluster() { InetSocketAddress address = new InetSocketAddress( config.getCluster().getManagerURL().getHostAddress(), @@ -220,6 +204,7 @@ private NioSocketConnector getClusterConnector(IdResolveContext workers) { NioSocketConnector connector = new NioSocketConnector(); BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, environment.getValidator(), om); + connector.getFilterChain().addFirst("mdc", new MdcFilter("Shard[%s]")); connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om))); connector.setHandler(this); connector.getSessionConfig().setAll(config.getCluster().getMina()); @@ -230,8 +215,6 @@ private NioSocketConnector getClusterConnector(IdResolveContext workers) { private static void scheduleIdleLogger(ScheduledExecutorService scheduler, IoSession session, Duration timeout) { scheduler.scheduleAtFixedRate( () -> { - setLocation(session); - final Duration elapsed = Duration.milliseconds(System.currentTimeMillis() - session.getLastIoTime()); if (elapsed.compareTo(timeout) > 0) { log.warn("No message sent or received since {}", elapsed); From 67377ba3cd657f18e17a6815b7779a0ac49a57f9 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 31 Jul 2024 14:56:01 +0200 Subject: [PATCH 04/40] sets verbose logs to trace --- .../bakdata/conquery/mode/cluster/ClusterConnectionShard.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java index b61d8df93c..949b367aa2 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -93,7 +93,7 @@ public void sessionClosed(IoSession session) { @Override public void sessionIdle(IoSession session, IdleStatus status) { - log.warn("Session idle {}.", status); + log.trace("Session idle {}.", status); } @Override @@ -217,7 +217,7 @@ private static void scheduleIdleLogger(ScheduledExecutorService scheduler, IoSes () -> { final Duration elapsed = Duration.milliseconds(System.currentTimeMillis() - session.getLastIoTime()); if (elapsed.compareTo(timeout) > 0) { - log.warn("No message sent or received since {}", elapsed); + log.trace("No message sent or received since {}", elapsed); } }, timeout.toSeconds(), timeout.toSeconds() / 2, TimeUnit.SECONDS From 0a083043a179f5a7297e22852020eb2d41ca5755 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 1 Aug 2024 09:29:45 +0200 Subject: [PATCH 05/40] fix build error --- .../java/com/bakdata/conquery/commands/ShardNode.java | 2 +- .../bakdata/conquery/io/AbstractSerializationTest.java | 7 +------ .../stores/types/ColumnStoreSerializationTests.java | 9 +-------- 3 files changed, 3 insertions(+), 15 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index 93bd96584d..28c0092ee0 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -112,7 +112,7 @@ public void run(ConqueryConfig config, Environment environment) throws Exception * * @return a preconfigured binary object mapper */ - private static ObjectMapper createInternalObjectMapper(Class viewClass, ConqueryConfig config, Validator validator) { + public static ObjectMapper createInternalObjectMapper(Class viewClass, ConqueryConfig config, Validator validator) { final ObjectMapper objectMapper = config.configureObjectMapper(Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER)); final MutableInjectableValues injectableValues = new MutableInjectableValues(); diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index 8d25bc0197..da50cecfe4 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -60,12 +60,7 @@ public void before() { metaStorage.loadData(); // Prepare shard node internal mapper - final ShardNode shardNode = mock(ShardNode.class); - when(shardNode.getConfig()).thenReturn(config); - when(shardNode.getValidator()).thenReturn(validator); - - when(shardNode.createInternalObjectMapper(any())).thenCallRealMethod(); - shardInternalMapper = shardNode.createInternalObjectMapper(View.Persistence.Shard.class); + shardInternalMapper = ShardNode.createInternalObjectMapper(View.Persistence.Shard.class, config, validator); // Prepare api response mapper doCallRealMethod().when(managerNode).customizeApiObjectMapper(any(ObjectMapper.class)); diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java index caa533c839..cb0d7b484a 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java @@ -1,9 +1,6 @@ package com.bakdata.conquery.models.events.stores.types; import static org.assertj.core.api.Assertions.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; import java.io.IOException; import java.util.Arrays; @@ -63,12 +60,8 @@ public static void setupRegistry() { // Prepare shard node internal mapper - final ShardNode shardNode = mock(ShardNode.class); - when(shardNode.getConfig()).thenReturn(new ConqueryConfig()); - when(shardNode.getValidator()).thenReturn(Validators.newValidator()); - when(shardNode.createInternalObjectMapper(any())).thenCallRealMethod(); - shardInternalMapper = shardNode.createInternalObjectMapper(View.Persistence.Shard.class); + shardInternalMapper = ShardNode.createInternalObjectMapper(View.Persistence.Shard.class, new ConqueryConfig(), Validators.newValidator()); } @Test From 572e16c0ddc7370a907b3df187cc57057f3dbca5 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 1 Aug 2024 10:36:08 +0200 Subject: [PATCH 06/40] close scheduler before disconnect to avoid scheduled reconnects --- .../conquery/mode/cluster/ClusterConnectionShard.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java index 949b367aa2..49fc13481a 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.mode.cluster; import java.net.InetSocketAddress; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.function.Supplier; @@ -88,7 +89,12 @@ public void sessionOpened(IoSession session) { public void sessionClosed(IoSession session) { log.info("Disconnected from ManagerNode."); - scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS); + try { + scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS); + } + catch (RejectedExecutionException e) { + log.trace("Scheduler rejected execution (probably in shutdown). Skipping reconnect attempt", e); + } } @Override @@ -264,7 +270,8 @@ public void start() throws Exception { @Override public void stop() throws Exception { - disconnectFromCluster(); + // close scheduler before disconnect to avoid scheduled reconnects scheduler.shutdown(); + disconnectFromCluster(); } } From 8e52c6012c2455064f0a2d062b74b671337a107b Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:21:33 +0200 Subject: [PATCH 07/40] use awaitility to check for cluster up --- backend/pom.xml | 6 +++ .../network/specific/RegisterWorker.java | 9 ----- .../java/com/bakdata/conquery/util/Wait.java | 32 ---------------- .../conquery/util/support/TestConquery.java | 38 ++++++++----------- 4 files changed, 21 insertions(+), 64 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/util/Wait.java diff --git a/backend/pom.xml b/backend/pom.xml index d2987fa93b..f4769dfe4e 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -409,5 +409,11 @@ prometheus-metrics-core 1.2.1 + + org.awaitility + awaitility + 4.2.1 + test + diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java index ef29e18919..ae1c25c95f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.messages.network.specific; -import java.time.Duration; - import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.messages.namespaces.specific.RequestConsistency; import com.bakdata.conquery.models.messages.network.MessageToManagerNode; @@ -9,7 +7,6 @@ import com.bakdata.conquery.models.messages.network.NetworkMessageContext.ManagerNodeNetworkContext; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.models.worker.WorkerInformation; -import com.bakdata.conquery.util.Wait; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -24,12 +21,6 @@ public class RegisterWorker extends MessageToManagerNode { @Override public void react(ManagerNodeNetworkContext context) throws Exception { ShardNodeInformation node = getShardNode(context); - Wait - .builder() - .stepTime(Duration.ofMillis(5)) - .total(Duration.ofSeconds(10)) - .build() - .until(()->getShardNode(context) != null); if(node == null) { throw new IllegalStateException("Could not find the slave "+context.getRemoteAddress()+" to register worker "+info.getId()); diff --git a/backend/src/main/java/com/bakdata/conquery/util/Wait.java b/backend/src/main/java/com/bakdata/conquery/util/Wait.java deleted file mode 100644 index d6ac483eca..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/util/Wait.java +++ /dev/null @@ -1,32 +0,0 @@ -package com.bakdata.conquery.util; - -import java.time.Duration; -import java.util.function.BooleanSupplier; - -import com.google.common.util.concurrent.Uninterruptibles; -import jakarta.validation.constraints.NotNull; -import lombok.Builder; - -@Builder -public class Wait { - - @NotNull - private final Duration stepTime; - - @NotNull - private final Duration total; - - public void until(BooleanSupplier condition) { - final long attempts = total.dividedBy(stepTime); - - long attempt = 0; - - while (!condition.getAsBoolean()) { - if (attempt >= attempts) { - throw new RuntimeException("Failed while waiting after " + attempt + " attempts"); - } - Uninterruptibles.sleepUninterruptibly(stepTime); - attempt++; - } - } -} diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 00f9822cd3..444b35769e 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.util.support; -import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; import static org.junit.jupiter.api.Assertions.fail; import java.io.File; @@ -10,7 +10,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import jakarta.validation.Validator; import jakarta.ws.rs.client.Client; @@ -32,7 +31,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; -import com.bakdata.conquery.util.Wait; import com.bakdata.conquery.util.io.Cloner; import com.google.common.util.concurrent.Uninterruptibles; import io.dropwizard.client.JerseyClientBuilder; @@ -44,7 +42,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.glassfish.jersey.client.ClientProperties; -import org.junit.jupiter.api.extension.ExtensionContext; /** * Represents the test instance of Conquery. @@ -57,23 +54,14 @@ public class TestConquery { private final File tmpDir; private final ConqueryConfig config; private final TestDataImporter testDataImporter; + private final Set openSupports = new HashSet<>(); @Getter private StandaloneCommand standaloneCommand; @Getter private DropwizardTestSupport dropwizard; - private Set openSupports = new HashSet<>(); @Getter private Client client; - private AtomicBoolean started = new AtomicBoolean(false); - - /** - * Returns the extension context used by the beforeAll-callback. - * - * @return The context. - */ - @Getter - private ExtensionContext beforeAllContext; // Initial user which is set before each test from the config. private User testUser; @@ -117,7 +105,7 @@ public void beforeAll() throws Exception { // define server - dropwizard = new DropwizardTestSupport(TestBootstrappingConquery.class, config, app -> { + dropwizard = new DropwizardTestSupport<>(TestBootstrappingConquery.class, config, app -> { if (config.getSqlConnectorConfig().isEnabled()) { standaloneCommand = new SqlStandaloneCommand((Conquery) app); } @@ -129,6 +117,14 @@ public void beforeAll() throws Exception { // start server dropwizard.before(); + + if (!config.getSqlConnectorConfig().isEnabled()) { + // Wait for shards to be connected + ClusterManager manager = (ClusterManager) standaloneCommand.getManager(); + ClusterState clusterState = manager.getConnectionManager().getClusterState(); + await().atMost(10, TimeUnit.SECONDS).until(() -> clusterState.getShardNodes().size() == 2); + } + // create HTTP client for api tests client = new JerseyClientBuilder(this.getDropwizard().getEnvironment()) .withProperty(ClientProperties.CONNECT_TIMEOUT, 10000) @@ -136,13 +132,13 @@ public void beforeAll() throws Exception { .build("test client"); } - public void afterAll() throws Exception { + public void afterAll() { client.close(); dropwizard.after(); FileUtils.deleteQuietly(tmpDir); } - public void afterEach() throws Exception { + public void afterEach() { synchronized (openSupports) { for (StandaloneSupport openSupport : openSupports) { removeSupportDataset(openSupport); @@ -206,13 +202,9 @@ private synchronized StandaloneSupport buildDistributedSupport(DatasetId dataset ClusterManager manager = (ClusterManager) standaloneCommand.getManager(); ClusterState clusterState = manager.getConnectionManager().getClusterState(); - assertThat(clusterState.getShardNodes()).hasSize(2); - Wait.builder() - .total(Duration.ofSeconds(5)) - .stepTime(Duration.ofMillis(5)) - .build() - .until(() -> clusterState.getWorkerHandlers().get(datasetId).getWorkers().size() == clusterState.getShardNodes().size()); + await().atMost(10, TimeUnit.SECONDS) + .until(() -> clusterState.getWorkerHandlers().get(datasetId).getWorkers().size() == clusterState.getShardNodes().size()); return buildSupport(datasetId, name, StandaloneSupport.Mode.WORKER); } From 7bfc42552233c394341a1feff30fde01ff13d3a8 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 1 Aug 2024 11:52:04 +0200 Subject: [PATCH 08/40] don't spin up shard nodes async in standalone mode, because adding managed objects to the environments lifecycle is not threadsafe --- .../DistributedStandaloneCommand.java | 59 ++++--------------- .../bakdata/conquery/commands/ShardNode.java | 35 +++-------- .../mode/cluster/ClusterConnectionShard.java | 11 +++- .../conquery/models/worker/Workers.java | 13 +++- 4 files changed, 41 insertions(+), 77 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java index 8e368fc189..23f867b39c 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java @@ -1,10 +1,8 @@ package com.bakdata.conquery.commands; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; import java.util.Vector; -import java.util.concurrent.*; import com.bakdata.conquery.Conquery; import com.bakdata.conquery.mode.cluster.ClusterManager; @@ -12,7 +10,6 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.XodusStoreFactory; import com.bakdata.conquery.util.io.ConqueryMDC; -import com.google.common.util.concurrent.ThreadFactoryBuilder; import io.dropwizard.core.cli.ServerCommand; import io.dropwizard.core.setup.Bootstrap; import io.dropwizard.core.setup.Environment; @@ -73,60 +70,26 @@ public void startStandalone(Environment environment, Namespace namespace, Conque conquery.setManagerNode(managerNode); conquery.run(manager); - //create thread pool to start multiple ShardNodes at the same time - ExecutorService starterPool = Executors.newFixedThreadPool( - config.getStandalone().getNumberOfShardNodes(), - new ThreadFactoryBuilder() - .setNameFormat("ShardNode Storage Loader %d") - .setUncaughtExceptionHandler((t, e) -> { - ConqueryMDC.setLocation(t.getName()); - log.error("{} failed to init storage of ShardNode", t.getName(), e); - }) - .build() - ); - - List> tasks = new ArrayList<>(); - for (int i = 0; i < config.getStandalone().getNumberOfShardNodes(); i++) { - - final int id = i; + for (int id = 0; id < config.getStandalone().getNumberOfShardNodes(); id++) { - tasks.add(starterPool.submit(() -> { - ShardNode sc = new ShardNode(ShardNode.DEFAULT_NAME + id); + ShardNode sc = new ShardNode(ShardNode.DEFAULT_NAME + id); - shardNodes.add(sc); + shardNodes.add(sc); - ConqueryMDC.setLocation(sc.getName()); + ConqueryMDC.setLocation(sc.getName()); - ConqueryConfig clone = config; + ConqueryConfig clone = config; - if (config.getStorage() instanceof XodusStoreFactory) { - final Path managerDir = ((XodusStoreFactory) config.getStorage()).getDirectory().resolve("shard-node" + id); - clone = config.withStorage(((XodusStoreFactory) config.getStorage()).withDirectory(managerDir)); - } + if (config.getStorage() instanceof XodusStoreFactory) { + final Path managerDir = ((XodusStoreFactory) config.getStorage()).getDirectory().resolve("shard-node" + id); + clone = config.withStorage(((XodusStoreFactory) config.getStorage()).withDirectory(managerDir)); + } - sc.run(clone, environment); - return sc; - })); + sc.run(clone, environment); } + ConqueryMDC.setLocation("ManagerNode"); log.debug("Waiting for ShardNodes to start"); - starterPool.shutdown(); - starterPool.awaitTermination(1, TimeUnit.HOURS); - //catch exceptions on tasks - boolean failed = false; - for (Future f : tasks) { - try { - f.get(); - - } - catch (ExecutionException e) { - log.error("during ShardNodes creation", e); - failed = true; - } - } - if (failed) { - System.exit(-1); - } // starts the Jersey Server log.debug("Starting REST Server"); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index 28c0092ee0..3f9b65ba82 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -13,8 +13,6 @@ import com.bakdata.conquery.io.storage.WorkerStorage; import com.bakdata.conquery.mode.cluster.ClusterConnectionShard; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.jobs.JobManager; -import com.bakdata.conquery.models.jobs.SimpleJob; import com.bakdata.conquery.models.worker.Worker; import com.bakdata.conquery.models.worker.Workers; import com.bakdata.conquery.util.io.ConqueryMDC; @@ -23,7 +21,7 @@ import com.fasterxml.jackson.databind.SerializationConfig; import io.dropwizard.core.ConfiguredBundle; import io.dropwizard.core.setup.Environment; -import io.dropwizard.lifecycle.Managed; +import io.dropwizard.lifecycle.setup.LifecycleEnvironment; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; @@ -35,12 +33,11 @@ */ @Slf4j @Getter -public class ShardNode implements ConfiguredBundle, Managed { +public class ShardNode implements ConfiguredBundle { public static final String DEFAULT_NAME = "shard-node"; private final String name; - private JobManager jobManager; @Setter private Workers workers; private ClusterConnectionShard clusterConnection; @@ -56,9 +53,7 @@ public ShardNode(String name) { @Override public void run(ConqueryConfig config, Environment environment) throws Exception { - - jobManager = new JobManager(getName(), config.isFailOnError()); - environment.lifecycle().manage(this); + LifecycleEnvironment lifecycle = environment.lifecycle(); workers = new Workers( config.getQueries().getExecutionPool(), @@ -68,11 +63,13 @@ public void run(ConqueryConfig config, Environment environment) throws Exception config.getQueries().getSecondaryIdSubPlanRetention() ); + lifecycle.manage(workers); + clusterConnection = - new ClusterConnectionShard(config, environment, workers, jobManager, () -> createInternalObjectMapper(View.InternalCommunication.class, config, environment.getValidator())); + new ClusterConnectionShard(config, environment, workers, () -> createInternalObjectMapper(View.InternalCommunication.class, config, environment.getValidator())); - environment.lifecycle().manage(clusterConnection); + lifecycle.manage(clusterConnection); final Collection workerStorages = config.getStorage().discoverWorkerStorages(); @@ -137,23 +134,7 @@ public static ObjectMapper createInternalObjectMapper(Class view return objectMapper; } - @Override - public void start() throws Exception { - for (Worker value : workers.getWorkers().values()) { - value.getJobManager().addSlowJob(new SimpleJob("Update Bucket Manager", value.getBucketManager()::fullUpdate)); - } - - - } - - @Override - public void stop() throws Exception { - getJobManager().close(); - - workers.stop(); - } - public boolean isBusy() { - return getJobManager().isSlowWorkerBusy() || workers.isBusy(); + return clusterConnection.isBusy() || workers.isBusy(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java index 49fc13481a..fc7c28b8b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -51,9 +51,9 @@ public class ClusterConnectionShard implements Managed, IoHandler { private final ConqueryConfig config; private final Environment environment; private final Workers workers; - private final JobManager jobManager; private final Supplier communicationMapperSupplier; + private JobManager jobManager; private ScheduledExecutorService scheduler; private NioSocketConnector connector; private ConnectFuture future; @@ -260,6 +260,10 @@ private void reportJobManagerStatus() { @Override public void start() throws Exception { + + + jobManager = new JobManager(environment.getName(), config.isFailOnError()); + scheduler = environment.lifecycle().scheduledExecutorService("cluster-connection-shard").build(); // Connect async as the manager might not be up jet or is started by a test in succession scheduler.schedule(this::connectToCluster, 0, TimeUnit.MINUTES); @@ -268,10 +272,15 @@ public void start() throws Exception { } + public boolean isBusy() { + return jobManager.isSlowWorkerBusy(); + } + @Override public void stop() throws Exception { // close scheduler before disconnect to avoid scheduled reconnects scheduler.shutdown(); disconnectFromCluster(); + jobManager.close(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java index cb7f603021..0f4981d03b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java @@ -18,8 +18,10 @@ import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; +import com.bakdata.conquery.models.jobs.SimpleJob; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; +import io.dropwizard.lifecycle.Managed; import lombok.Getter; import lombok.NonNull; import lombok.Setter; @@ -31,7 +33,7 @@ * Each Shard contains one {@link Worker} per {@link Dataset}. */ @Slf4j -public class Workers extends IdResolveContext { +public class Workers extends IdResolveContext implements Managed { @Getter @Setter private AtomicInteger nextWorker = new AtomicInteger(0); @Getter @@ -152,6 +154,15 @@ public boolean isBusy() { return false; } + @Override + public void start() throws Exception { + + for (Worker value : getWorkers().values()) { + value.getJobManager().addSlowJob(new SimpleJob("Update Bucket Manager", value.getBucketManager()::fullUpdate)); + } + } + + @Override public void stop() { jobsThreadPool.shutdown(); for (Worker w : workers.values()) { From 3ae3619c485fe136919ba46fe5a01befd407632d Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:58:04 +0200 Subject: [PATCH 09/40] extract sync object from persistable execution class --- .../conquery/apiv1/QueryProcessor.java | 66 +++++---- .../conquery/apiv1/forms/FormConfigAPI.java | 7 +- .../conquery/io/result/ExternalResult.java | 24 ++- .../external/ExternalResultProcessor.java | 19 ++- .../auth/permissions/DatasetPermission.java | 3 +- .../auth/permissions/ExecutionPermission.java | 3 +- .../permissions/FormConfigPermission.java | 3 +- .../auth/permissions/FormPermission.java | 7 +- .../models/config/ExternalResultProvider.java | 7 +- .../models/execution/ManagedExecution.java | 138 +++++++----------- .../conquery/models/execution/Owned.java | 2 +- .../models/forms/configs/FormConfig.java | 14 +- .../FormConfigProcessor.java | 7 +- .../forms/managed/ExternalExecution.java | 127 +++++++--------- .../models/forms/managed/ManagedForm.java | 13 +- .../forms/managed/ManagedInternalForm.java | 18 +-- .../query/DistributedExecutionManager.java | 59 ++++---- .../models/query/ExecutionManager.java | 135 +++++++++++++---- .../models/query/ExternalResultImpl.java | 57 ++++++++ .../conquery/models/query/ManagedQuery.java | 29 ++-- .../query/preview/EntityPreviewExecution.java | 20 ++- .../models/query/results/FormShardResult.java | 16 +- .../resources/admin/rest/AdminResource.java | 28 ++-- .../conquery/resources/api/QueryResource.java | 30 ++-- .../resources/api/ResultExternalResource.java | 19 ++- .../sql/conquery/SqlExecutionManager.java | 50 +++++-- .../sql/execution/SqlExecutionResult.java | 14 +- .../sql/execution/SqlExecutionService.java | 3 +- .../api/StoredQueriesProcessorTest.java | 10 +- .../api/form/config/FormConfigTest.java | 40 +++-- .../integration/common/LoadingUtil.java | 14 +- .../conquery/integration/json/FormTest.java | 31 +--- .../tests/ExternalFormBackendTest.java | 20 +-- .../conquery/io/result/ResultTestUtil.java | 7 +- .../result/excel/ExcelResultRenderTest.java | 12 +- .../stores/SerializingStoreDumpTest.java | 6 +- .../conquery/models/SerializationTests.java | 11 +- .../models/execution/DefaultLabelTest.java | 11 +- .../tasks/PermissionCleanupTaskTest.java | 3 +- .../conquery/tasks/QueryCleanupTaskTest.java | 4 +- .../conquery/util/support/TestConquery.java | 13 +- 41 files changed, 624 insertions(+), 476 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 92d2b9b463..ac9cdabcbc 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -21,6 +21,12 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; +import jakarta.inject.Inject; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.validation.Validator; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -61,6 +67,7 @@ import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.i18n.I18n; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; @@ -84,12 +91,6 @@ import com.bakdata.conquery.util.io.IdColumnUtil; import com.google.common.collect.ClassToInstanceMap; import com.google.common.collect.MutableClassToInstanceMap; -import jakarta.inject.Inject; -import jakarta.servlet.http.HttpServletRequest; -import jakarta.validation.Validator; -import jakarta.ws.rs.BadRequestException; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.UriBuilder; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -112,22 +113,23 @@ public class QueryProcessor { public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { final Collection allQueries = storage.getAllExecutions(); - return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders); + return getQueriesFiltered(dataset.getId(), RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders); } - public Stream getQueriesFiltered(Dataset datasetId, UriBuilder uriBuilder, Subject subject, Collection allQueries, boolean allProviders) { + public Stream getQueriesFiltered(DatasetId datasetId, UriBuilder uriBuilder, Subject subject, Collection allQueries, boolean allProviders) { return allQueries.stream() // The following only checks the dataset, under which the query was submitted, but a query can target more that // one dataset. - .filter(q -> q.getDataset().equals(datasetId)) + .filter(q -> q.getDataset().getId().equals(datasetId)) // to exclude subtypes from somewhere else .filter(QueryProcessor::canFrontendRender) .filter(Predicate.not(ManagedExecution::isSystem)) .filter(q -> q.getState().equals(ExecutionState.DONE) || q.getState().equals(ExecutionState.NEW)) .filter(q -> subject.isPermitted(q, Ability.READ)) .map(mq -> { - final OverviewExecutionStatus status = mq.buildStatusOverview(uriBuilder.clone(), subject); + Namespace namespace = datasetRegistry.get(mq.getDataset().getId()); + final OverviewExecutionStatus status = mq.buildStatusOverview(uriBuilder.clone(), subject, namespace); if (mq.isReadyToDownload()) { status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders)); } @@ -206,8 +208,8 @@ public void cancel(Subject subject, Dataset dataset, ManagedExecution query) { log.info("User[{}] cancelled Query[{}]", subject.getId(), query.getId()); - final ExecutionManager executionManager = datasetRegistry.get(dataset.getId()).getExecutionManager(); - executionManager.cancelQuery(dataset, query); + final ExecutionManager executionManager = datasetRegistry.get(dataset.getId()).getExecutionManager(); + executionManager.cancelQuery(query); } public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatch patch) { @@ -275,14 +277,19 @@ public void deleteQuery(Subject subject, ManagedExecution execution) { storage.removeExecution(execution.getId()); } + public ExecutionState awaitDone(ManagedExecution query, int time, TimeUnit unit) { + final Namespace namespace = datasetRegistry.get(query.getDataset().getId()); + return namespace.getExecutionManager().awaitDone(query, time, unit); + } + public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { final Namespace namespace = datasetRegistry.get(query.getDataset().getId()); query.initExecutable(namespace, config); - final FullExecutionStatus status = query.buildStatusFull(subject); + final FullExecutionStatus status = query.buildStatusFull(subject, namespace); - if (query.isReadyToDownload() && subject.isPermitted(query.getDataset(), Ability.DOWNLOAD)) { + if (query.isReadyToDownload() && subject.isPermitted(namespace.getDataset(), Ability.DOWNLOAD)) { status.setResultUrls(getResultAssets(config.getResultProviders(), query, url, allProviders)); } return status; @@ -317,7 +324,7 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext execution = ((ManagedQuery) namespace .getExecutionManager() - .createExecution(query, subject.getUser(), dataset, false)); + .createExecution(query, subject.getUser(), namespace, false)); execution.setLastResultCount((long) statistic.getResolved().size()); @@ -338,7 +345,8 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri subject.authorize(dataset, Ability.ENTITY_PREVIEW); subject.authorize(dataset, Ability.PRESERVE_ID); - final PreviewConfig previewConfig = datasetRegistry.get(dataset.getId()).getPreviewConfig(); + Namespace namespace = datasetRegistry.get(dataset.getId()); + final PreviewConfig previewConfig = namespace.getPreviewConfig(); final EntityPreviewForm form = EntityPreviewForm.create(entity, idKind, dateRange, sources, previewConfig.getSelects(), previewConfig.getTimeStratifiedSelects(), datasetRegistry); @@ -350,7 +358,7 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri final EntityPreviewExecution execution = (EntityPreviewExecution) postQuery(dataset, form, subject, true); - if (execution.awaitDone(10, TimeUnit.SECONDS) == ExecutionState.RUNNING) { + if (namespace.getExecutionManager().awaitDone(execution, 10, TimeUnit.SECONDS) == ExecutionState.RUNNING) { log.warn("Still waiting for {} after 10 Seconds.", execution.getId()); throw new ConqueryError.ExecutionProcessingTimeoutError(); } @@ -361,7 +369,7 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri } - final FullExecutionStatus status = execution.buildStatusFull(subject); + final FullExecutionStatus status = execution.buildStatusFull(subject, namespace); status.setResultUrls(getResultAssets(config.getResultProviders(), execution, uriBuilder, false)); return status; } @@ -402,17 +410,25 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Subje query.authorize(subject, dataset, visitors, storage); // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the Userish in methods - ExecutionMetrics.reportNamespacedIds(visitors.getInstance(NamespacedIdentifiableCollector.class).getIdentifiables(), primaryGroupName); + NamespacedIdentifiableCollector namespaceIdCollector = visitors.getInstance(NamespacedIdentifiableCollector.class); + if (namespaceIdCollector == null) { + throw new IllegalStateException("NamespacedIdentifiableCollector was not registered properly"); + } + ExecutionMetrics.reportNamespacedIds(namespaceIdCollector.getIdentifiables(), primaryGroupName); ExecutionMetrics.reportQueryClassUsage(query.getClass(), primaryGroupName); final Namespace namespace = datasetRegistry.get(dataset.getId()); - final ExecutionManager executionManager = namespace.getExecutionManager(); + final ExecutionManager executionManager = namespace.getExecutionManager(); // If this is only a re-executing query, try to execute the underlying query instead. { - final Optional executionId = visitors.getInstance(QueryUtils.OnlyReusingChecker.class).getOnlyReused(); + QueryUtils.OnlyReusingChecker reusedChecker = visitors.getInstance(QueryUtils.OnlyReusingChecker.class); + if (reusedChecker == null) { + throw new IllegalStateException("OnlyReusingChecker was not registered properly"); + } + final Optional executionId = reusedChecker.getOnlyReused(); final Optional execution = @@ -424,13 +440,13 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Subje } // Execute the query - return executionManager.runQuery(namespace, query, subject.getUser(), dataset, config, system); + return executionManager.runQuery(namespace, query, subject.getUser(), config, system); } /** * Determine if the submitted query does reuse ONLY another query and restart that instead of creating another one. */ - private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ConqueryConfig config, ExecutionManager executionManager, User user) { + private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId executionId, Namespace namespace, ConqueryConfig config, ExecutionManager executionManager, User user) { ManagedExecution execution = storage.getExecution(executionId); @@ -458,7 +474,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId exe if (!user.isOwner(execution)) { final ManagedExecution newExecution = - executionManager.createExecution(execution.getSubmitted(), user, execution.getDataset(), false); + executionManager.createExecution(execution.getSubmitted(), user, namespace, false); newExecution.setLabel(execution.getLabel()); newExecution.setTags(execution.getTags().clone()); storage.updateExecution(newExecution); @@ -511,7 +527,7 @@ public Stream> resolveEntities(Subject subject, List> assetMap); /** - * Implementations should use the {@link com.bakdata.eva.resources.ExternalResultResource#getDownloadURL(UriBuilder, ManagedExecution, String)} - * to build the url for the asset. + * Returns assert builders for all results registered by an {@link com.bakdata.conquery.models.forms.managed.ExternalExecution} (see {@link AssetBuilder}). * The provided assetId is the one that is used by {@link ExternalResult#fetchExternalResult(String)} to retrieve the download. */ @JsonIgnore @@ -37,6 +48,7 @@ public interface ExternalResult { */ @FunctionalInterface interface AssetBuilder extends Function { - } + + User getServiceUser(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/external/ExternalResultProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/external/ExternalResultProcessor.java index 50eceee008..e4a9a290e8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/external/ExternalResultProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/external/ExternalResultProcessor.java @@ -1,29 +1,28 @@ package com.bakdata.conquery.io.result.external; import jakarta.inject.Inject; -import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.Response; import com.bakdata.conquery.io.result.ExternalResult; import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.forms.managed.ExternalExecution; +import com.bakdata.conquery.models.query.ExecutionManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor(onConstructor_ = @Inject) public class ExternalResultProcessor { - public Response getResult(Subject subject, ManagedExecution execution, String fileName) { + private final DatasetRegistry datasetRegistry; - ResultUtil.authorizeExecutable(subject, execution); - - if (!(execution instanceof ExternalResult)) { - throw new WebApplicationException("The execution exists, but produces not a zipped result", Response.Status.CONFLICT); + public Response getResult(Subject subject, ExternalExecution execution, String fileName) { - } + ResultUtil.authorizeExecutable(subject, execution); - T externalExecution = (T) execution; + ExecutionManager executionManager = datasetRegistry.get(execution.getDataset().getId()).getExecutionManager(); + ExternalResult externalResult = executionManager.getExternalResult(execution.getId()); - return externalExecution.fetchExternalResult(fileName); + return externalResult.fetchExternalResult(fileName); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/DatasetPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/DatasetPermission.java index db68212eae..d3e9d3719b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/DatasetPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/DatasetPermission.java @@ -4,6 +4,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import org.jetbrains.annotations.TestOnly; @CPSType(id = "DATASET", base = StringPermissionBuilder.class) public class DatasetPermission extends StringPermissionBuilder { @@ -38,7 +39,7 @@ public static ConqueryPermission onInstance(Set abilities, DatasetId in return INSTANCE.instancePermission(abilities, instance); } - @Deprecated + @TestOnly public static ConqueryPermission onInstance(Ability ability, DatasetId instance) { return INSTANCE.instancePermission(ability, instance); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ExecutionPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ExecutionPermission.java index 83cfda3f87..4c0b55bb54 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ExecutionPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/ExecutionPermission.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import org.jetbrains.annotations.TestOnly; @CPSType(id = "EXECUTION", base = StringPermissionBuilder.class) public class ExecutionPermission extends StringPermissionBuilder { @@ -43,7 +44,7 @@ public Set getAllowedAbilities() { } //// Helper functions - @Deprecated + @TestOnly public static ConqueryPermission onInstance(Ability ability, ManagedExecutionId instance) { return INSTANCE.instancePermission(ability, instance); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormConfigPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormConfigPermission.java index bbb683a939..519c0c0a2c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormConfigPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormConfigPermission.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; +import org.jetbrains.annotations.TestOnly; @CPSType(id = "FORM_CONFIG", base = StringPermissionBuilder.class) public class FormConfigPermission extends StringPermissionBuilder { @@ -36,7 +37,7 @@ public Set getAllowedAbilities() { } //// Helper functions - @Deprecated + @TestOnly public static ConqueryPermission onInstance(Set abilities, FormConfigId instance) { return INSTANCE.instancePermission(abilities, instance); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormPermission.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormPermission.java index 242b17d3bf..91daa31c9c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormPermission.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/FormPermission.java @@ -1,9 +1,10 @@ package com.bakdata.conquery.models.auth.permissions; -import com.bakdata.conquery.io.cps.CPSType; - import java.util.Set; +import com.bakdata.conquery.io.cps.CPSType; +import org.jetbrains.annotations.TestOnly; + /** * Permission to restrict the usage of a specific form type. * The forms are programmatically distinguished from their CPSType. @@ -34,7 +35,7 @@ public static ConqueryPermission onInstance(Set abilities, String insta return INSTANCE.instancePermission(abilities, instance); } - @Deprecated + @TestOnly public static ConqueryPermission onInstance(Ability ability, String instance) { return INSTANCE.instancePermission(ability, instance); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExternalResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExternalResultProvider.java index ffef2f45ec..fa364c7fd6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExternalResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExternalResultProvider.java @@ -2,16 +2,15 @@ import java.util.Collection; import java.util.Collections; - import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.io.result.ExternalResult; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; import com.bakdata.conquery.io.result.external.ExternalResultProcessor; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.forms.managed.ExternalExecution; import com.bakdata.conquery.resources.api.ResultExternalResource; import io.dropwizard.jersey.DropwizardResourceConfig; import lombok.Getter; @@ -28,7 +27,7 @@ public class ExternalResultProvider implements ResultRendererProvider { @Override public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { - if (!(exec instanceof ExternalResult)) { + if (!(exec instanceof ExternalExecution)) { return Collections.emptyList(); } @@ -36,7 +35,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } - return ((ExternalResult) exec).getResultAssets().map(assetBuilder -> assetBuilder.apply(uriBuilder.clone())).toList(); + return ((ExternalExecution) exec).getResultAssets().map(assetBuilder -> assetBuilder.apply(uriBuilder.clone())).toList(); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index 03b9a0d1fd..e3d8489e13 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -8,11 +8,9 @@ import java.util.List; import java.util.Set; import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; - -import javax.annotation.Nullable; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; @@ -39,6 +37,7 @@ import com.bakdata.conquery.models.identifiable.IdentifiableImpl; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.worker.Namespace; @@ -51,12 +50,10 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.annotation.OptBoolean; import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.Uninterruptibles; -import jakarta.validation.constraints.NotNull; -import jakarta.ws.rs.core.UriBuilder; import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.Setter; import lombok.ToString; @@ -71,6 +68,7 @@ @CPSBase @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @EqualsAndHashCode(callSuper = false) +@NoArgsConstructor(access = AccessLevel.PROTECTED) public abstract class ManagedExecution extends IdentifiableImpl implements Taggable, Shareable, Labelable, Owned, Visitable { /** @@ -85,7 +83,6 @@ public abstract class ManagedExecution extends IdentifiableImpl executionManager) { if (this.error != null && !this.error.equalsRegardingCodeAndMessage(error)) { // Warn only again if the error is different (failed might by called per collected result) log.warn("The execution [{}] failed again with:\n\t{}\n\tThe previous error was: {}", getId(), this.error, error); @@ -208,7 +199,7 @@ public void fail(ConqueryErrorInfo error) { log.warn("The execution [{}] failed with:\n\t{}", getId(), getError()); } - finish(ExecutionState.FAILED); + finish(ExecutionState.FAILED, executionManager); } public void start() { @@ -219,20 +210,11 @@ public void start() { startTime = LocalDateTime.now(); setState(ExecutionState.RUNNING); - - resetLock(); + getMetaStorage().updateExecution(this); } } - private void resetLock() { - executingLock = new CountDownLatch(1); - } - - private void clearLock() { - executingLock.countDown(); - } - - public void finish(ExecutionState executionState) { + public void finish(ExecutionState executionState, ExecutionManager executionManager) { if (getState() == ExecutionState.NEW) { log.error("Query[{}] was never run.", getId(), new Exception()); } @@ -244,12 +226,8 @@ public void finish(ExecutionState executionState) { // Set execution state before acting on the latch to prevent a race condition // Not sure if also the storage needs an update first setState(executionState); - clearLock(); + getMetaStorage().updateExecution(this); - // No need to persist failed queries. (As they are most likely invalid) - if (getState() == ExecutionState.DONE) { - getStorage().updateExecution(this); - } } log.info("{} {} {} within {}", getState(), queryId, getClass().getSimpleName(), getExecutionTime()); @@ -262,19 +240,8 @@ public Duration getExecutionTime() { return (startTime != null && finishTime != null) ? Duration.between(startTime, finishTime) : null; } - /** - * Blocks until a execution finished of the specified timeout is reached. Return immediately if the execution is not running - */ - public ExecutionState awaitDone(int time, TimeUnit unit) { - if (getState() != ExecutionState.RUNNING) { - return getState(); - } - Uninterruptibles.awaitUninterruptibly(executingLock, time, unit); - - return getState(); - } - public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status) { + public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status, Namespace namespace) { status.setLabel(label == null ? queryId.toString() : getLabelWithoutAutoLabelSuffix()); status.setPristineLabel(label == null || queryId.toString().equals(label) || isAutoLabeled()); status.setId(getId()); @@ -289,17 +256,18 @@ public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus sta status.setContainsDates(containsDates); if (owner != null) { - status.setOwner(owner.getId()); - status.setOwnerName(owner.getLabel()); + User user = owner; + status.setOwner(user.getId()); + status.setOwnerName(user.getLabel()); } } /** * Renders a lightweight status with meta information about this query. Computation an size should be small for this. */ - public OverviewExecutionStatus buildStatusOverview(UriBuilder url, Subject subject) { + public OverviewExecutionStatus buildStatusOverview(UriBuilder url, Subject subject, Namespace namespace) { OverviewExecutionStatus status = new OverviewExecutionStatus(); - setStatusBase(subject, status); + setStatusBase(subject, status, namespace); return status; } @@ -308,21 +276,19 @@ public OverviewExecutionStatus buildStatusOverview(UriBuilder url, Subject subje * Renders an extensive status of this query (see {@link FullExecutionStatus}. The rendering can be computation intensive and can produce a large * object. The use of the full status is only intended if a client requested specific information about this execution. */ - public FullExecutionStatus buildStatusFull(Subject subject) { - - initExecutable(namespace, config); + public FullExecutionStatus buildStatusFull(Subject subject, Namespace namespace) { FullExecutionStatus status = new FullExecutionStatus(); - setStatusFull(status, subject); + setStatusFull(status, subject, namespace); return status; } - public void setStatusFull(FullExecutionStatus status, Subject subject) { - setStatusBase(subject, status); + public void setStatusFull(FullExecutionStatus status, Subject subject, Namespace namespace) { + setStatusBase(subject, status, namespace); - setAdditionalFieldsForStatusWithColumnDescription(subject, status); - setAdditionalFieldsForStatusWithSource(subject, status); + setAdditionalFieldsForStatusWithColumnDescription(subject, status, namespace); + setAdditionalFieldsForStatusWithSource(subject, status, namespace); setAdditionalFieldsForStatusWithGroups(status); setAvailableSecondaryIds(status); status.setProgress(progress); @@ -338,7 +304,7 @@ private void setAvailableSecondaryIds(FullExecutionStatus status) { final QueryUtils.AvailableSecondaryIdCollector secondaryIdCollector = new QueryUtils.AvailableSecondaryIdCollector(); visit(secondaryIdCollector); - + // TODO may work with ids directly here instead of resolving status.setAvailableSecondaryIds(secondaryIdCollector.getIds()); } @@ -347,7 +313,7 @@ private void setAdditionalFieldsForStatusWithGroups(FullExecutionStatus status) * This is usually not done very often and should be reasonable fast, so don't cache this. */ List permittedGroups = new ArrayList<>(); - for (Group group : storage.getAllGroups()) { + for (Group group : getMetaStorage().getAllGroups()) { for (Permission perm : group.getPermissions()) { if (perm.implies(createPermission(Ability.READ.asSet()))) { permittedGroups.add(group.getId()); @@ -358,14 +324,14 @@ private void setAdditionalFieldsForStatusWithGroups(FullExecutionStatus status) status.setGroups(permittedGroups); } - protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { + protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status, Namespace namespace) { // Implementation specific } /** * Sets additional fields of an {@link ExecutionStatus} when a more specific status is requested. */ - protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status) { + protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status, Namespace namespace) { QueryDescription query = getSubmitted(); status.setCanExpand(canSubjectExpand(subject, query)); @@ -442,7 +408,7 @@ public ConqueryPermission createPermission(Set abilities) { return ExecutionPermission.onInstance(abilities, getId()); } - public void reset() { + public void reset(ExecutionManager executionManager) { // This avoids endless loops with already reset queries if(getState().equals(ExecutionState.NEW)){ return; diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/Owned.java b/backend/src/main/java/com/bakdata/conquery/models/execution/Owned.java index cc319cf49f..1c77dd5911 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/Owned.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/Owned.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.models.execution; -import com.bakdata.conquery.models.auth.permissions.Authorized; import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.permissions.Authorized; public interface Owned extends Authorized { User getOwner(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java b/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java index 12299bd15c..baa4706bf9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/configs/FormConfig.java @@ -10,13 +10,11 @@ import java.util.Set; import java.util.UUID; import java.util.function.Consumer; - import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.FormConfigPatch; import com.bakdata.conquery.io.jackson.serializer.MetaIdRef; -import com.bakdata.conquery.io.jackson.serializer.NsIdRef; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.Subject; @@ -24,12 +22,12 @@ import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.auth.permissions.FormConfigPermission; -import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.Labelable; import com.bakdata.conquery.models.execution.Owned; import com.bakdata.conquery.models.execution.Shareable; import com.bakdata.conquery.models.execution.Taggable; import com.bakdata.conquery.models.identifiable.IdentifiableImpl; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.FormConfigId; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.util.VariableDefaultValue; @@ -56,8 +54,7 @@ @FieldNameConstants public class FormConfig extends IdentifiableImpl implements Shareable, Labelable, Taggable, Owned { - @NsIdRef - protected Dataset dataset; + protected DatasetId dataset; @NotEmpty private String formType; @VariableDefaultValue @NonNull @@ -86,14 +83,14 @@ public FormConfig(String formType, JsonNode values) { @Override public FormConfigId createId() { - return new FormConfigId(dataset.getId(), formType, formId); + return new FormConfigId(dataset, formType, formId); } /** * Provides an overview (meta data) of this form configuration without the * actual form field values. */ - public FormConfigOverviewRepresentation overview(Subject subject) { + public FormConfigOverviewRepresentation overview(MetaStorage storage, Subject subject) { String ownerName = Optional.ofNullable(owner).map(User::getLabel).orElse(null); return FormConfigOverviewRepresentation.builder() @@ -124,7 +121,6 @@ public FormConfigFullRepresentation fullRepresentation(MetaStorage storage, Subj for(Permission perm : group.getPermissions()) { if(perm.implies(createPermission(Ability.READ.asSet()))) { permittedGroups.add(group.getId()); - continue; } } } @@ -192,7 +188,7 @@ public static class FormConfigFullRepresentation extends FormConfigOverviewRepre } public Consumer valueSetter() { - return (patch) -> {setValues(patch.getValues());}; + return (patch) -> setValues(patch.getValues()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java index 4ade864697..6000c90819 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java @@ -5,7 +5,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Stream; - import jakarta.inject.Inject; import jakarta.validation.Validator; @@ -87,12 +86,12 @@ public Stream getConfigsByFormType(@NonNull Su final Set formTypesFinal = requestedFormType; final Stream stream = storage.getAllFormConfigs().stream() - .filter(c -> dataset.equals(c.getDataset())) + .filter(c -> dataset.getId().equals(c.getDataset())) .filter(c -> formTypesFinal.contains(c.getFormType())) .filter(c -> subject.isPermitted(c, Ability.READ)); - return stream.map(c -> c.overview(subject)); + return stream.map(c -> c.overview(storage, subject)); } /** @@ -117,7 +116,7 @@ public FormConfig addConfig(Subject subject, Dataset targetDataset, FormConfigAP subject.authorize(namespace.getDataset(), Ability.READ); - final FormConfig internalConfig = config.intern(storage.getUser(subject.getId()), targetDataset); + final FormConfig internalConfig = config.intern(storage.getUser(subject.getId()), targetDataset.getId()); // Add the config immediately to the submitted dataset addConfigToDataset(internalConfig); diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java index f110eeeb87..ffa07cbe6d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java @@ -3,13 +3,11 @@ import java.net.URI; import java.net.URISyntaxException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.UUID; +import java.util.concurrent.CountDownLatch; import java.util.stream.Stream; -import jakarta.ws.rs.core.Response; - import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.apiv1.forms.ExternalForm; @@ -25,16 +23,19 @@ import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.query.ExecutionManager; +import com.bakdata.conquery.models.query.ExternalResultImpl; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.api.ResultExternalResource; import com.bakdata.conquery.util.AuthUtil; -import com.fasterxml.jackson.annotation.JacksonInject; -import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.OptBoolean; import com.google.common.base.Preconditions; import com.google.common.collect.MoreCollectors; import it.unimi.dsi.fastutil.Pair; +import lombok.AccessLevel; import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; @@ -46,47 +47,25 @@ @Slf4j @CPSType(id = "EXTERNAL_EXECUTION", base = ManagedExecution.class) @EqualsAndHashCode(callSuper = true, doNotUseGetters = true) -public class ExternalExecution extends ManagedForm implements ExternalResult { +@NoArgsConstructor(access = AccessLevel.PROTECTED) +public class ExternalExecution extends ManagedForm { + @Getter private UUID externalTaskId; @JsonIgnore @EqualsAndHashCode.Exclude - private ExternalFormBackendApi api; - - @JsonIgnore - @EqualsAndHashCode.Exclude - private FormBackendConfig formBackendConfig; + private ExecutionManager executionManager; - @JsonIgnore - @EqualsAndHashCode.Exclude - private User serviceUser; - - /** - * Pairs of external result assets (internal url) and their internal asset builder. - * The internal asset builder generates the asset url with the context of a user request. - */ - @JsonIgnore - private List> resultsAssetMap = Collections.emptyList(); - - @JsonCreator - protected ExternalExecution(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) { - super(storage); - } - - public ExternalExecution(ExternalForm form, User user, Dataset dataset, MetaStorage storage) { - super(form, user, dataset, storage); + public ExternalExecution(ExternalForm form, User user, Dataset dataset, MetaStorage metaStorage) { + super(form, user, dataset, metaStorage); } @Override - protected void doInitExecutable() { - formBackendConfig = getConfig().getPluginConfigs(FormBackendConfig.class) - .filter(c -> c.supportsFormType(getSubmittedForm().getFormType())) - .collect(MoreCollectors.onlyElement()); - - api = formBackendConfig.createApi(); + protected void doInitExecutable(Namespace namespace) { + executionManager = namespace.getExecutionManager(); } @@ -97,50 +76,60 @@ public void start() { synchronized (this) { if (externalTaskId != null) { - syncExternalState(); + syncExternalState(executionManager); } if (getState() == ExecutionState.RUNNING) { throw new ConqueryError.ExecutionProcessingError(); } - super.start(); // Create service user - serviceUser = formBackendConfig.createServiceUser(getOwner(), getDataset()); + Dataset dataset = getNamespace().getDataset(); + User originalUser = getOwner(); + FormBackendConfig formBackendConfig = getConfig().getPluginConfigs(FormBackendConfig.class) + .filter(c -> c.supportsFormType(getSubmittedForm().getFormType())) + .collect(MoreCollectors.onlyElement()); + User serviceUser = formBackendConfig.createServiceUser(originalUser, dataset); + ExternalFormBackendApi api = formBackendConfig.createApi(); - final ExternalTaskState externalTaskState = api.postForm(getSubmitted(), getOwner(), serviceUser, getDataset()); + final ExternalTaskState externalTaskState = api.postForm(getSubmitted(), originalUser, serviceUser, dataset); + + executionManager.addResult(this, new ExternalResultImpl(new CountDownLatch(0), api, formBackendConfig, serviceUser)); externalTaskId = externalTaskState.getId(); + + super.start(); } } - private synchronized void syncExternalState() { + private synchronized void syncExternalState(ExecutionManager executionManager) { Preconditions.checkNotNull(externalTaskId, "Cannot check external task, because no Id is present"); - final ExternalTaskState formState = api.getFormState(externalTaskId); + final ExternalTaskState formState = this.executionManager.getExternalResult(this.getId()).getApi().getFormState(externalTaskId); - updateStatus(formState); + updateStatus(formState, executionManager); } - private void updateStatus(ExternalTaskState formState) { + private void updateStatus(ExternalTaskState formState, ExecutionManager executionManager) { switch (formState.getStatus()) { case RUNNING -> { setState(ExecutionState.RUNNING); setProgress(formState.getProgress().floatValue()); } - case FAILURE -> fail(formState.getError()); + case FAILURE -> fail(formState.getError(), executionManager); case SUCCESS -> { - resultsAssetMap = registerResultAssets(formState); - finish(ExecutionState.DONE); + List> resultsAssetMap = registerResultAssets(formState); + this.executionManager.getExternalResult(this.getId()).setResultsAssetMap(resultsAssetMap); + finish(ExecutionState.DONE, executionManager); } - case CANCELLED -> reset(); + case CANCELLED -> reset(executionManager); } } - private List> registerResultAssets(ExternalTaskState response) { - final List> assetMap = new ArrayList<>(); + private List> registerResultAssets(ExternalTaskState response) { + final List> assetMap = new ArrayList<>(); response.getResults().forEach(asset -> assetMap.add(Pair.of(asset, createResultAssetBuilder(asset)))); return assetMap; } @@ -148,7 +137,7 @@ private List> registerResultAssets(ExternalTaskS /** * The {@link ResultAsset} is request-dependent, so we can prepare only builder here which takes an url builder. */ - private AssetBuilder createResultAssetBuilder(ResultAsset asset) { + private ExternalResult.AssetBuilder createResultAssetBuilder(ResultAsset asset) { return (uriBuilder) -> { try { final URI externalDownloadURL = ResultExternalResource.getDownloadURL(uriBuilder.clone(), this, asset.getAssetId()); @@ -161,10 +150,10 @@ private AssetBuilder createResultAssetBuilder(ResultAsset asset) { } @Override - public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status) { - syncExternalState(); + public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status, Namespace namespace) { + syncExternalState(namespace.getExecutionManager()); - super.setStatusBase(subject, status); + super.setStatusBase(subject, status, namespace); } @Override @@ -172,33 +161,25 @@ public void cancel() { //TODO this is no longer called as the ExecutionManager used to call this. Preconditions.checkNotNull(externalTaskId, "Cannot check external task, because no Id is present"); - updateStatus(api.cancelTask(externalTaskId)); - } - - @Override - public Stream getResultAssets() { - return resultsAssetMap.stream().map(Pair::value); - } - - @Override - public Response fetchExternalResult(String assetId) { - final ResultAsset resultRef = resultsAssetMap.stream() - .map(Pair::key).filter(a -> a.getAssetId().equals(assetId)) - .collect(MoreCollectors.onlyElement()); - - return api.getResult(resultRef.url()); + updateStatus(executionManager.getExternalResult(this.getId()).getApi().cancelTask(externalTaskId), executionManager); } @Override - public void finish(ExecutionState executionState) { + public void finish(ExecutionState executionState, ExecutionManager executionManager) { if (getState().equals(executionState)) { return; } + User serviceUser = executionManager.getExternalResult(this.getId()).getServiceUser(); + + super.finish(executionState, executionManager); - super.finish(executionState); synchronized (this) { - AuthUtil.cleanUpUserAndBelongings(serviceUser, getStorage()); - serviceUser = null; + AuthUtil.cleanUpUserAndBelongings(serviceUser, getMetaStorage()); } } + + @JsonIgnore + public Stream getResultAssets() { + return executionManager.getExternalResult(this.getId()).getResultAssets(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java index 410460a5ac..cf04d068e5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java @@ -12,12 +12,12 @@ import com.bakdata.conquery.models.forms.configs.FormConfig; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.Visitable; -import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.OptBoolean; import com.fasterxml.jackson.databind.DatabindContext; +import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.ToString; import lombok.extern.slf4j.Slf4j; @@ -28,6 +28,7 @@ @Slf4j @EqualsAndHashCode(callSuper = true) @CPSType(id = "MANAGED_FORM", base = ManagedExecution.class) +@NoArgsConstructor(access = AccessLevel.PROTECTED) public abstract class ManagedForm extends ManagedExecution { /** @@ -44,10 +45,6 @@ public abstract class ManagedForm extends ManagedExecution { @Getter private Form submittedForm; - protected ManagedForm(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) { - super(storage); - } - protected ManagedForm(F submittedForm, User owner, Dataset submittedDataset, MetaStorage storage) { super(owner, submittedDataset, storage); this.submittedForm = submittedForm; @@ -66,9 +63,9 @@ public void start() { .tags(this.getTags()) .values(getSubmittedForm().getValues()).build(); - final FormConfig formConfig = build.intern(getOwner(), getDataset()); + final FormConfig formConfig = build.intern(getOwner(), getDataset().getId()); - getStorage().addFormConfig(formConfig); + getMetaStorage().addFormConfig(formConfig); } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java index 237a3d4c2e..98944728a0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java @@ -29,11 +29,12 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.FormShardResult; -import com.fasterxml.jackson.annotation.JacksonInject; +import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.OptBoolean; +import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -45,6 +46,7 @@ @CPSType(base = ManagedExecution.class, id = "INTERNAL_FORM") @Getter @EqualsAndHashCode(callSuper = true) +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class ManagedInternalForm extends ManagedForm implements SingleTableResult, InternalExecution { @@ -63,10 +65,6 @@ public class ManagedInternalForm extends ManagedF @EqualsAndHashCode.Exclude private final IdMap flatSubQueries = new IdMap<>(); - public ManagedInternalForm(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) { - super(storage); - } - public ManagedInternalForm(F form, User user, Dataset submittedDataset, MetaStorage storage) { super(form, user, submittedDataset, storage); } @@ -77,9 +75,9 @@ public ManagedQuery getSubQuery(ManagedExecutionId subQueryId) { } @Override - public void doInitExecutable() { + public void doInitExecutable(Namespace namespace) { // Convert sub queries to sub executions - getSubmitted().resolve(new QueryResolveContext(getNamespace(), getConfig(), getStorage(), null)); + getSubmitted().resolve(new QueryResolveContext(getNamespace(), getConfig(), getMetaStorage(), null)); subQueries = createSubExecutions(); // Initialize sub executions @@ -92,7 +90,7 @@ private Map createSubExecutions() { .entrySet() .stream().collect(Collectors.toMap( Map.Entry::getKey, - e -> e.getValue().toManagedExecution(getOwner(), getDataset(), getStorage()) + e -> e.getValue().toManagedExecution(getOwner(), getDataset(), getMetaStorage()) )); } @@ -101,7 +99,7 @@ private Map createSubExecutions() { @Override public void start() { synchronized (this) { - subQueries.values().stream().forEach(flatSubQueries::add); + subQueries.values().forEach(flatSubQueries::add); } flatSubQueries.values().forEach(ManagedQuery::start); super.start(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java index 7db39b2938..e967778005 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java @@ -5,6 +5,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.stream.Stream; import com.bakdata.conquery.io.storage.MetaStorage; @@ -12,15 +13,14 @@ import com.bakdata.conquery.mode.cluster.ClusterState; import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.Group; -import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.InternalExecution; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.forms.managed.ManagedInternalForm; import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.bakdata.conquery.models.messages.namespaces.specific.CancelQuery; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.ShardResult; -import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.models.worker.WorkerHandler; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -28,16 +28,17 @@ @Slf4j public class DistributedExecutionManager extends ExecutionManager { - public record DistributedResult(Map> results) implements Result { - - public DistributedResult() { - this(new ConcurrentHashMap<>()); - } + public record DistributedResult(Map> results, CountDownLatch executingLock) implements ExecutionManager.InternalResult { @Override public Stream streamQueryResults() { return results.values().stream().flatMap(Collection::stream); } + + @Override + public CountDownLatch getExecutingLock() { + return executingLock; + } } private final ClusterState clusterState; @@ -50,14 +51,19 @@ public DistributedExecutionManager(MetaStorage storage, ClusterState state) { @Override - protected void doExecute(Namespace namespace, InternalExecution internalExecution) { - ManagedExecution execution = (ManagedExecution & InternalExecution) internalExecution; + protected > void doExecute(E execution) { - log.info("Executing Query[{}] in Dataset[{}]", execution.getQueryId(), namespace.getDataset().getId()); + log.info("Executing Query[{}] in Dataset[{}]", execution.getQueryId(), execution.getDataset()); + + addResult(execution.getId(), new DistributedResult(new ConcurrentHashMap<>(), new CountDownLatch(1))); + + if (execution instanceof ManagedInternalForm form) { + form.getSubQueries().values().forEach((query) -> addResult(query.getId(), new DistributedResult(new ConcurrentHashMap<>(), new CountDownLatch(1)))); + } final WorkerHandler workerHandler = getWorkerHandler(execution); - workerHandler.sendToAll(internalExecution.createExecutionMessage()); + workerHandler.sendToAll(execution.createExecutionMessage()); } private WorkerHandler getWorkerHandler(ManagedExecution execution) { @@ -71,41 +77,44 @@ private WorkerHandler getWorkerHandler(ManagedExecution execution) { * @implNote subQueries of Forms are managed by the form itself, so need to be passed from outside. */ @SneakyThrows - public > void handleQueryResult(R result, E query) { + public > void handleQueryResult(R result, E execution) { log.debug("Received Result[size={}] for Query[{}]", result.getResults().size(), result.getQueryId()); log.trace("Received Result\n{}", result.getResults()); - if (query.getState() != ExecutionState.RUNNING) { - log.warn("Received result for Query[{}] that is not RUNNING but {}", query.getId(), query.getState()); + if (execution.getState() != ExecutionState.RUNNING) { + log.warn("Received result for Query[{}] that is not RUNNING but {}", execution.getId(), execution.getState()); return; } if (result.getError().isPresent()) { - query.fail(result.getError().get()); + execution.fail(result.getError().get(), this); + clearLock(execution.getId()); } else { // We don't collect all results together into a fat list as that would cause lots of huge re-allocations for little gain. - final DistributedResult results = getResult(query, DistributedResult::new); + final DistributedResult results = getResult(execution.getId()); results.results.put(result.getWorkerId(), result.getResults()); final Set finishedWorkers = results.results.keySet(); // If all known workers have returned a result, the query is DONE. - if (finishedWorkers.equals(getWorkerHandler(query).getAllWorkerIds())) { - query.finish(ExecutionState.DONE); + if (finishedWorkers.equals(getWorkerHandler(execution).getAllWorkerIds())) { + execution.finish(ExecutionState.DONE, this); + clearLock(execution.getId()); + } } // State changed to DONE or FAILED - if (query.getState() != ExecutionState.RUNNING) { - final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(query.getOwner(), getStorage()).map(Group::getName).orElse("none"); + if (execution.getState() != ExecutionState.RUNNING) { + final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner(), getStorage()).map(Group::getName).orElse("none"); ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).dec(); - ExecutionMetrics.getQueryStateCounter(query.getState(), primaryGroupName).inc(); - ExecutionMetrics.getQueriesTimeHistogram(primaryGroupName).update(query.getExecutionTime().toMillis()); + ExecutionMetrics.getQueryStateCounter(execution.getState(), primaryGroupName).inc(); + ExecutionMetrics.getQueriesTimeHistogram(primaryGroupName).update(execution.getExecutionTime().toMillis()); /* This log is here to prevent an NPE which could occur when no strong reference to result.getResults() existed anymore after the query finished and immediately was reset */ @@ -115,11 +124,11 @@ public } @Override - public void cancelQuery(Dataset dataset, ManagedExecution query) { + public void doCancelQuery(ManagedExecution execution) { log.debug("Sending cancel message to all workers."); - query.cancel(); - getWorkerHandler(query).sendToAll(new CancelQuery(query.getId())); + execution.cancel(); + getWorkerHandler(execution).sendToAll(new CancelQuery(execution.getId())); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index 4f560a0cbf..03be13e3c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -1,35 +1,49 @@ package com.bakdata.conquery.models.query; +import java.util.Objects; import java.util.UUID; -import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.io.result.ExternalResult; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.metrics.ExecutionMetrics; import com.bakdata.conquery.models.auth.AuthorizationHelper; import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.error.ConqueryError; +import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.InternalExecution; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.forms.managed.ExternalExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.Namespace; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalNotification; +import com.google.common.util.concurrent.Uninterruptibles; import lombok.Data; import lombok.extern.slf4j.Slf4j; @Data @Slf4j -public abstract class ExecutionManager { +public abstract class ExecutionManager { + /** + * TODO Result might not be the appropriate name any more, because we will put everything in this instance + * belonging to an execution, which cannot/should not be serialized/cached in a store. + */ public interface Result { + + CountDownLatch getExecutingLock(); + } + + public interface InternalResult extends Result{ Stream streamQueryResults(); } @@ -41,10 +55,16 @@ public interface Result { .removalListener(this::executionRemoved) .build(); + private final Cache externalExecutionResults = + CacheBuilder.newBuilder() + .softValues() + .removalListener(this::executionRemoved) + .build(); + /** * Manage state of evicted Queries, setting them to NEW. */ - private void executionRemoved(RemovalNotification removalNotification) { + private void executionRemoved(RemovalNotification removalNotification) { // If removal was done manually we assume it was also handled properly if (!removalNotification.wasEvicted()) { return; @@ -58,7 +78,7 @@ private void executionRemoved(RemovalNotification removal // The query might already be deleted if (execution != null) { - execution.reset(); + execution.reset(this); } } @@ -67,23 +87,35 @@ public ManagedExecution getExecution(ManagedExecutionId execution) { return storage.getExecution(execution); } - protected R getResult(ManagedExecution execution, Callable defaultProvider) throws ExecutionException { - return executionResults.get(execution.getId(), defaultProvider); + protected R getResult(ManagedExecutionId id) throws ExecutionException { + return executionResults.getIfPresent(id); + } + + public ExternalResult getExternalResult(ManagedExecutionId id) { + return externalExecutionResults.getIfPresent(id); + } + + protected void addResult(ManagedExecutionId id, R result) { + executionResults.put(id, result); } - protected void addResult(ManagedExecution execution, R result) { - executionResults.put(execution.getId(), result); + /** + * Is called upon start by the external execution + */ + public void addResult(ExternalExecution execution, ExternalResult result) { + externalExecutionResults.put(execution.getId(), result); } - public final ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system) { - final ManagedExecution execution = createExecution(query, user, submittedDataset, system); + public final ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, ConqueryConfig config, boolean system) { + final ManagedExecution execution = createExecution(query, user, namespace, system); + execute(namespace, execution, config); return execution; } - public final void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config) { + public final void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config) { clearQueryResults(execution); @@ -103,30 +135,37 @@ public final void execute(Namespace namespace, ManagedExecution execution, Conq throw e; } - log.info("Starting execution[{}]", execution.getQueryId()); - - execution.start(); + ManagedExecutionId executionId = execution.getId(); + log.info("Starting execution[{}]", executionId); + try { + execution.start(); - final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner(), storage).map(Group::getName).orElse("none"); - ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).inc(); + final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner(), storage).map(Group::getName).orElse("none"); + ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).inc(); - if (execution instanceof InternalExecution internalExecution) { - doExecute(namespace, internalExecution); + if (execution instanceof InternalExecution internalExecution) { + doExecute((ManagedExecution & InternalExecution) internalExecution); + } + } + catch (Exception e) { + log.warn("Failed to execute '{}'", executionId); + execution.fail(ConqueryError.asConqueryError(e), this); } } - protected abstract void doExecute(Namespace namespace, InternalExecution execution); + protected abstract > void doExecute(E execution); // Visible for testing - public final ManagedExecution createExecution(QueryDescription query, User user, Dataset submittedDataset, boolean system) { - return createQuery(query, UUID.randomUUID(), user, submittedDataset, system); + public final ManagedExecution createExecution(QueryDescription query, User user, Namespace namespace, boolean system) { + return createExecution(query, UUID.randomUUID(), user, namespace, system); } - public final ManagedExecution createQuery(QueryDescription query, UUID queryId, User user, Dataset submittedDataset, boolean system) { + public final ManagedExecution createExecution(QueryDescription query, UUID queryId, User user, Namespace namespace, boolean system) { // Transform the submitted query into an initialized execution - ManagedExecution managed = query.toManagedExecution(user, submittedDataset, storage); + ManagedExecution managed = query.toManagedExecution(user, namespace.getDataset(), storage); managed.setSystem(system); managed.setQueryId(queryId); + managed.setMetaStorage(storage); // Store the execution storage.addExecution(managed); @@ -134,10 +173,23 @@ public final ManagedExecution createQuery(QueryDescription query, UUID queryId, return managed; } - public abstract void cancelQuery(final Dataset dataset, final ManagedExecution query); + public final void cancelQuery(final ManagedExecution execution) { + if (execution instanceof ExternalExecution externalExecution) { + externalExecution.cancel(); + externalExecutionResults.invalidate(execution.getId()); + return; + } + + doCancelQuery(execution); + } + + + public abstract void doCancelQuery(final ManagedExecution execution); public void clearQueryResults(ManagedExecution execution) { executionResults.invalidate(execution.getId()); + + //TODO external execution } public Stream streamQueryResults(ManagedExecution execution) { @@ -148,4 +200,37 @@ public Stream streamQueryResults(ManagedExecution execution) { : resultParts.streamQueryResults(); } + + public void clearLock(ManagedExecutionId id) { + R result = Objects.requireNonNull(executionResults.getIfPresent(id), "Cannot clear lock on absent execution result"); + + result.getExecutingLock().countDown(); + //TODO external execution + } + + /** + * Blocks until an execution finished of the specified timeout is reached. Return immediately if the execution is not running + */ + public ExecutionState awaitDone(ManagedExecution execution, int time, TimeUnit unit) { + ManagedExecutionId id = execution.getId(); + ExecutionState state = execution.getState(); + if (state != ExecutionState.RUNNING) { + return state; + } + + Result result; + if (execution instanceof InternalExecution) { + result = executionResults.getIfPresent(id); + } + else { + result = externalExecutionResults.getIfPresent(id); + } + + if (result == null) { + throw new IllegalStateException("Execution is running, but no result is registered"); + } + Uninterruptibles.awaitUninterruptibly(result.getExecutingLock(), time, unit); + + return execution.getState(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java new file mode 100644 index 0000000000..79936f5a45 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java @@ -0,0 +1,57 @@ +package com.bakdata.conquery.models.query; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.stream.Stream; +import jakarta.ws.rs.core.Response; + +import com.bakdata.conquery.apiv1.execution.ResultAsset; +import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; +import com.bakdata.conquery.io.result.ExternalResult; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.config.FormBackendConfig; +import com.google.common.collect.MoreCollectors; +import it.unimi.dsi.fastutil.Pair; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; + +@RequiredArgsConstructor +public class ExternalResultImpl implements ExternalResult { + private final CountDownLatch latch; + + @Getter + private final ExternalFormBackendApi api; + + private final FormBackendConfig formBackendConfig; + + @Getter + private final User serviceUser; + + /** + * Pairs of external result assets (internal url) and their internal asset builder. + * The internal asset builder generates the asset url with the context of a user request. + */ + @Setter + private List> resultsAssetMap = Collections.emptyList(); + + @Override + public CountDownLatch getExecutingLock() { + return latch; + } + + @Override + public Stream getResultAssets() { + return resultsAssetMap.stream().map(Pair::value); + } + + @Override + public Response fetchExternalResult(String assetId) { + final ResultAsset resultRef = resultsAssetMap.stream() + .map(Pair::key).filter(a -> a.getAssetId().equals(assetId)) + .collect(MoreCollectors.onlyElement()); + + return api.getResult(resultRef.url()); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 83d6984b3a..437e24e4fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -27,11 +27,12 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.ShardResult; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.QueryUtils; -import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.OptBoolean; +import lombok.AccessLevel; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.Setter; import lombok.ToString; @@ -42,6 +43,7 @@ @ToString(callSuper = true) @Slf4j @CPSType(base = ManagedExecution.class, id = "MANAGED_QUERY") +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class ManagedQuery extends ManagedExecution implements SingleTableResult, InternalExecution { // Needs to be resolved externally before being executed @@ -55,27 +57,23 @@ public class ManagedQuery extends ManagedExecution implements SingleTableResult, private transient List columnDescriptions; - protected ManagedQuery(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) { - super(storage); - } - public ManagedQuery(Query query, User owner, Dataset submittedDataset, MetaStorage storage) { super(owner, submittedDataset, storage); this.query = query; } @Override - protected void doInitExecutable() { - query.resolve(new QueryResolveContext(getNamespace(), getConfig(), getStorage(), null)); + protected void doInitExecutable(Namespace namespace) { + query.resolve(new QueryResolveContext(getNamespace(), getConfig(), getMetaStorage(), null)); } @Override - public void finish(ExecutionState executionState) { + public void finish(ExecutionState executionState, ExecutionManager executionManager) { //TODO this is not optimal with SQLExecutionService as this might fully evaluate the query. lastResultCount = query.countResults(streamResults(OptionalLong.empty())); - super.finish(executionState); + super.finish(executionState, executionManager); } @@ -101,9 +99,9 @@ public long resultRowCount() { } @Override - public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status) { + public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status, Namespace namespace) { - super.setStatusBase(subject, status); + super.setStatusBase(subject, status, namespace); status.setNumberOfResults(getLastResultCount()); Query query = getQuery(); @@ -114,7 +112,8 @@ public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus sta } } - protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { + @Override + protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status, Namespace namespace) { if (columnDescriptions == null) { columnDescriptions = generateColumnDescriptions(isInitialized(), getConfig()); } @@ -127,8 +126,8 @@ public List getResultInfos() { } @Override - public void reset() { - super.reset(); + public void reset(ExecutionManager executionManager) { + super.reset(executionManager); getNamespace().getExecutionManager().clearQueryResults(this); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index df62608dfb..bf9c27122a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -42,14 +42,15 @@ import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; -import com.fasterxml.jackson.annotation.JacksonInject; +import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.OptBoolean; import com.fasterxml.jackson.databind.node.BooleanNode; import com.fasterxml.jackson.databind.node.DecimalNode; import com.fasterxml.jackson.databind.node.IntNode; import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.MoreCollectors; +import lombok.AccessLevel; +import lombok.NoArgsConstructor; import lombok.ToString; import org.jetbrains.annotations.NotNull; @@ -59,15 +60,12 @@ */ @CPSType(id = "ENTITY_PREVIEW_EXECUTION", base = ManagedExecution.class) @ToString +@NoArgsConstructor(access = AccessLevel.PROTECTED) public class EntityPreviewExecution extends ManagedInternalForm { @ToString.Exclude private PreviewConfig previewConfig; - protected EntityPreviewExecution(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) { - super(storage); - } - public EntityPreviewExecution(EntityPreviewForm entityPreviewQuery, User user, Dataset submittedDataset, MetaStorage storage) { super(entityPreviewQuery, user, submittedDataset, storage); } @@ -257,8 +255,8 @@ public boolean isSystem() { } @Override - public void doInitExecutable() { - super.doInitExecutable(); + public void doInitExecutable(Namespace namespace) { + super.doInitExecutable(namespace); previewConfig = getNamespace().getPreviewConfig(); } @@ -268,12 +266,12 @@ public void doInitExecutable() { * Most importantly to {@link EntityPreviewStatus#setInfos(List)} to for infos of entity. */ @Override - public FullExecutionStatus buildStatusFull(Subject subject) { + public FullExecutionStatus buildStatusFull(Subject subject, Namespace namespace) { initExecutable(getNamespace(), getConfig()); final EntityPreviewStatus status = new EntityPreviewStatus(); - setStatusFull(status, subject); + setStatusFull(status, subject, namespace); status.setQuery(getValuesQuery().getQuery()); @@ -430,7 +428,7 @@ private ManagedQuery getValuesQuery() { } @Override - protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status) { + protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status, Namespace namespace) { status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getConfig())); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java index a07989bb0b..c72692fb2a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java @@ -33,6 +33,10 @@ public void addResult(DistributedExecutionManager executionManager) { final ManagedInternalForm managedInternalForm = (ManagedInternalForm) executionManager.getExecution(getFormId()); final ManagedQuery subQuery = managedInternalForm.getSubQuery(getQueryId()); + if (subQuery == null) { + throw new IllegalStateException("Subquery %s did not belong to form %s. Known subqueries: %s".formatted(getQueryId(), formId, managedInternalForm.getSubQueries())); + } + executionManager.handleQueryResult(this, subQuery); @@ -41,12 +45,20 @@ public void addResult(DistributedExecutionManager executionManager) { managedInternalForm.fail( getError().orElseThrow( () -> new IllegalStateException(String.format("Query[%s] failed but no error was set.", subQuery.getId())) - ) + ), + executionManager ); + + // Signal to waiting threads that the form finished + executionManager.clearLock(formId); } if (managedInternalForm.allSubQueriesDone()) { - managedInternalForm.finish(ExecutionState.DONE); + + managedInternalForm.finish(ExecutionState.DONE, executionManager); + + // Signal to waiting threads that the form finished + executionManager.clearLock(formId); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java index db5cbbc540..322af14f23 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java @@ -9,6 +9,17 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.UUID; +import jakarta.inject.Inject; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.PathParam; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; @@ -18,20 +29,10 @@ import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.jobs.JobManagerStatus; import com.bakdata.conquery.models.messages.network.specific.CancelJobMessage; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.resources.admin.ui.AdminUIResource; import io.dropwizard.auth.Auth; -import jakarta.inject.Inject; -import jakarta.ws.rs.Consumes; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.POST; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.PathParam; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.QueryParam; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; -import jakarta.ws.rs.core.UriBuilder; import lombok.RequiredArgsConstructor; @Consumes({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING}) @@ -108,13 +109,14 @@ public FullExecutionStatus[] getQueries(@Auth Subject currentUser, @QueryParam(" .filter(t -> t.getCreationTime().toLocalDate().isAfter(since) || t.getCreationTime().toLocalDate().isEqual(since)) .limit(limit) .map(t -> { + Namespace namespace = processor.getDatasetRegistry().get(t.getDataset().getId()); try { - return t.buildStatusFull(currentUser); + return t.buildStatusFull(currentUser, namespace); } catch (ConqueryError e) { // Initialization of execution probably failed, so we construct a status based on the overview status final FullExecutionStatus fullExecutionStatus = new FullExecutionStatus(); - t.setStatusBase(currentUser, fullExecutionStatus); + t.setStatusBase(currentUser, fullExecutionStatus, namespace); fullExecutionStatus.setStatus(ExecutionState.FAILED); fullExecutionStatus.setError(e); return fullExecutionStatus; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index 599c06500b..facc8705ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -4,19 +4,6 @@ import static com.bakdata.conquery.resources.ResourceConstants.QUERY; import java.util.concurrent.TimeUnit; - -import com.bakdata.conquery.apiv1.AdditionalMediaTypes; -import com.bakdata.conquery.apiv1.MetaDataPatch; -import com.bakdata.conquery.apiv1.QueryProcessor; -import com.bakdata.conquery.apiv1.RequestAwareUriBuilder; -import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.execution.ExecutionState; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.query.SingleTableResult; -import io.dropwizard.auth.Auth; -import io.dropwizard.jersey.PATCH; import jakarta.inject.Inject; import jakarta.servlet.http.HttpServletRequest; import jakarta.ws.rs.BadRequestException; @@ -31,6 +18,19 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.Response; + +import com.bakdata.conquery.apiv1.AdditionalMediaTypes; +import com.bakdata.conquery.apiv1.MetaDataPatch; +import com.bakdata.conquery.apiv1.QueryProcessor; +import com.bakdata.conquery.apiv1.RequestAwareUriBuilder; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.execution.ExecutionState; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.query.SingleTableResult; +import io.dropwizard.auth.Auth; +import io.dropwizard.jersey.PATCH; import lombok.RequiredArgsConstructor; @Path("queries") @@ -52,7 +52,7 @@ public FullExecutionStatus getStatus(@Auth Subject subject, @PathParam(QUERY) Ma subject.authorize(query.getDataset(), Ability.READ); subject.authorize(query, Ability.READ); - query.awaitDone(1, TimeUnit.SECONDS); + processor.awaitDone(query, 1, TimeUnit.SECONDS); return processor.getQueryFullStatus(query, subject, RequestAwareUriBuilder.fromRequest(servletRequest), allProviders); } @@ -68,7 +68,7 @@ public Response getDescription(@Auth Subject subject, @PathParam(QUERY) ManagedE subject.authorize(query.getDataset(), Ability.READ); subject.authorize(query, Ability.READ); - if(query.awaitDone(1, TimeUnit.SECONDS) != ExecutionState.DONE){ + if (processor.awaitDone(query, 1, TimeUnit.SECONDS) != ExecutionState.DONE) { return Response.status(Response.Status.CONFLICT.getStatusCode(), "Query is still running.").build(); // Request was submitted too early. } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExternalResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExternalResource.java index 93732d3e1b..7736ee2ffb 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExternalResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExternalResource.java @@ -6,13 +6,6 @@ import java.net.URI; import java.net.URISyntaxException; - -import com.bakdata.conquery.io.result.ExternalResult; -import com.bakdata.conquery.io.result.external.ExternalResultProcessor; -import com.bakdata.conquery.models.auth.entities.Subject; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.resources.ResourceConstants; -import io.dropwizard.auth.Auth; import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.HeaderParam; @@ -21,6 +14,12 @@ import jakarta.ws.rs.QueryParam; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.UriBuilder; + +import com.bakdata.conquery.io.result.external.ExternalResultProcessor; +import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.forms.managed.ExternalExecution; +import com.bakdata.conquery.resources.ResourceConstants; +import io.dropwizard.auth.Auth; import lombok.extern.slf4j.Slf4j; @Path("result/external") @@ -32,7 +31,7 @@ public class ResultExternalResource { private ExternalResultProcessor processor; - public static URI getDownloadURL(UriBuilder uriBuilder, E exec, String filename) + public static URI getDownloadURL(UriBuilder uriBuilder, ExternalExecution exec, String filename) throws URISyntaxException { return uriBuilder .path(ResultExternalResource.class) @@ -57,12 +56,12 @@ public static URI getDownloadURL(U @Path("{" + QUERY + "}/{" + FILENAME + "}") public Response download( @Auth Subject subject, - @PathParam(QUERY) ManagedExecution execution, + @PathParam(QUERY) ExternalExecution execution, @PathParam(FILENAME) String fileName, @HeaderParam("user-agent") String userAgent, @QueryParam("charset") String queryCharset ) { - log.info("Result download for {} on dataset {} by user {} ({}).", execution, execution.getDataset().getId(), subject.getId(), subject.getName()); + log.info("Result download for {} on dataset {} by user {} ({}).", execution, execution.getDataset(), subject.getId(), subject.getName()); return processor.getResult(subject, execution, fileName); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index 7d9c88f6d9..29cbef83c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -4,9 +4,9 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.InternalExecution; @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.ManagedQuery; -import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import com.bakdata.conquery.sql.execution.SqlExecutionResult; @@ -37,17 +36,26 @@ public SqlExecutionManager(SqlConverter sqlConverter, SqlExecutionService sqlExe } @Override - protected void doExecute(Namespace namespace, InternalExecution execution) { + protected > void doExecute(E execution) { + + addResult(execution.getId(), new SqlExecutionResult()); if (execution instanceof ManagedQuery managedQuery) { - CompletableFuture sqlQueryExecution = executeAsync(managedQuery); + CompletableFuture sqlQueryExecution = executeAsync(managedQuery, this); runningExecutions.put(managedQuery.getId(), sqlQueryExecution); return; } if (execution instanceof ManagedInternalForm managedForm) { - CompletableFuture.allOf(managedForm.getSubQueries().values().stream().map(this::executeAsync).toArray(CompletableFuture[]::new)) - .thenRun(() -> managedForm.finish(ExecutionState.DONE)); + CompletableFuture.allOf(managedForm.getSubQueries().values().stream().map(managedQuery -> { + addResult(managedQuery.getId(), new SqlExecutionResult()); + return executeAsync(managedQuery, this); + + }).toArray(CompletableFuture[]::new)) + .thenRun(() -> { + managedForm.finish(ExecutionState.DONE, this); + clearLock(managedForm.getId()); + }); return; } @@ -55,9 +63,9 @@ protected void doExecute(Namespace namespace, InternalExecution execution) { } @Override - public void cancelQuery(Dataset dataset, ManagedExecution query) { + public void doCancelQuery(ManagedExecution execution) { - CompletableFuture sqlQueryExecution = runningExecutions.remove(query.getId()); + CompletableFuture sqlQueryExecution = runningExecutions.remove(execution.getId()); // already finished/canceled if (sqlQueryExecution == null) { @@ -68,20 +76,34 @@ public void cancelQuery(Dataset dataset, ManagedExecution query) { sqlQueryExecution.cancel(true); } - query.cancel(); + execution.cancel(); } - private CompletableFuture executeAsync(ManagedQuery managedQuery) { + private CompletableFuture executeAsync(ManagedQuery managedQuery, SqlExecutionManager executionManager) { SqlQuery sqlQuery = converter.convert(managedQuery.getQuery()); return CompletableFuture.supplyAsync(() -> executionService.execute(sqlQuery)) .thenAccept(result -> { - addResult(managedQuery, result); + ManagedExecutionId id = managedQuery.getId(); + try { + // We need to transfer the columns and data from the query result together with the execution lock to a new result + SqlExecutionResult startResult = getResult(id); + SqlExecutionResult + finishResult = + new SqlExecutionResult(result.getColumnNames(), result.getTable(), startResult.getExecutingLock()); + addResult(id, finishResult); + } + catch (ExecutionException e) { + throw new RuntimeException(e); + } managedQuery.setLastResultCount(((long) result.getRowCount())); - managedQuery.finish(ExecutionState.DONE); - runningExecutions.remove(managedQuery.getId()); + managedQuery.finish(ExecutionState.DONE, executionManager); + runningExecutions.remove(id); + + // Unlock waiting requests + clearLock(id); }) .exceptionally(e -> { - managedQuery.fail(new ConqueryError.SqlError(e)); + managedQuery.fail(ConqueryError.asConqueryError(e), this); runningExecutions.remove(managedQuery.getId()); return null; }); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java index fa9a313918..63b4bfa516 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.execution; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.stream.Stream; import com.bakdata.conquery.models.query.ExecutionManager; @@ -8,15 +9,24 @@ import lombok.Value; @Value -public class SqlExecutionResult implements ExecutionManager.Result { +public class SqlExecutionResult implements ExecutionManager.InternalResult { List columnNames; List table; int rowCount; + CountDownLatch executingLock; - public SqlExecutionResult(List columnNames, List table) { + public SqlExecutionResult() { + this.columnNames = null; + this.table = null; + this.executingLock = new CountDownLatch(1); + rowCount = 0; + } + + public SqlExecutionResult(List columnNames, List table, CountDownLatch executingLock) { this.columnNames = columnNames; this.table = table; + this.executingLock = executingLock; rowCount = table.size(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java index a2a4ff7050..1bef1e2f96 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java @@ -7,6 +7,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.stream.Collectors; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -58,7 +59,7 @@ private SqlExecutionResult createStatementAndExecute(SqlQuery sqlQuery, Connecti final List columnNames = getColumnNames(resultSet, columnCount); final List resultTable = createResultTable(resultSet, resultTypes, columnCount); - return new SqlExecutionResult(columnNames, resultTable); + return new SqlExecutionResult(columnNames, resultTable, new CountDownLatch(1)); } // not all DB vendors throw SQLExceptions catch (SQLException | RuntimeException e) { diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index f1b1488925..01bd9a04e9 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -10,6 +10,8 @@ import java.util.List; import java.util.UUID; import java.util.stream.Collectors; +import jakarta.validation.Validator; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.QueryProcessor; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; @@ -50,8 +52,6 @@ import com.google.common.collect.ImmutableList; import io.dropwizard.core.setup.Environment; import io.dropwizard.jersey.validation.Validators; -import jakarta.validation.Validator; -import jakarta.ws.rs.core.UriBuilder; import lombok.SneakyThrows; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; @@ -128,7 +128,7 @@ public static void beforeAll() { @Test public void getQueriesFiltered() { - List infos = processor.getQueriesFiltered(DATASET_0, URI_BUILDER, USERS[0], queries, true) + List infos = processor.getQueriesFiltered(DATASET_0.getId(), URI_BUILDER, USERS[0], queries, true) .collect(Collectors.toList()); assertThat(infos) @@ -155,8 +155,8 @@ private static User mockUser(int id, List allowedQueryIds) { } - private static ManagedForm mockManagedForm(User user, ManagedExecutionId id, ExecutionState execState, final Dataset dataset){ - return new ManagedInternalForm(new ExportForm(), user, dataset, STORAGE) { + private static ManagedForm mockManagedForm(User user, ManagedExecutionId id, ExecutionState execState, final Dataset dataset) { + return new ManagedInternalForm<>(new ExportForm(), user, dataset, STORAGE) { { setState(execState); setCreationTime(LocalDateTime.MIN); diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index f20effb7f5..83d96953ad 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -2,8 +2,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import java.net.URL; import java.time.ZoneId; @@ -12,7 +11,6 @@ import java.util.Map; import java.util.UUID; import java.util.stream.Stream; - import jakarta.validation.Validator; import com.bakdata.conquery.apiv1.FormConfigPatch; @@ -20,6 +18,7 @@ import com.bakdata.conquery.apiv1.forms.export_form.AbsoluteMode; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.forms.export_form.RelativeMode; +import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.MutableInjectableValues; @@ -52,15 +51,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; import com.fasterxml.jackson.databind.node.TextNode; -import io.dropwizard.jersey.validation.Validators; import io.dropwizard.core.setup.Environment; +import io.dropwizard.jersey.validation.Validators; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.TestInstance.Lifecycle; -import org.mockito.Mockito; /** @@ -70,17 +68,15 @@ @TestInstance(Lifecycle.PER_CLASS) public class FormConfigTest { - private ConqueryConfig config = new ConqueryConfig(); + private final ConqueryConfig config = new ConqueryConfig(); private MetaStorage storage; - private DatasetRegistry namespacesMock; private FormConfigProcessor processor; - private AuthorizationController controller; private Validator validator = Validators.newValidatorFactory().getValidator(); - private Dataset dataset = new Dataset("test"); - private Dataset dataset1 = new Dataset("test1"); + private final Dataset dataset = new Dataset("test"); + private final Dataset dataset1 = new Dataset("test1"); private DatasetId datasetId; private DatasetId datasetId1; private ExportForm form; @@ -94,7 +90,7 @@ public void setupTestClass() throws Exception { datasetId1 = dataset1.getId(); // Mock DatasetRegistry for translation - namespacesMock = Mockito.mock(DatasetRegistry.class); + DatasetRegistry namespacesMock = mock(DatasetRegistry.class); doAnswer(invocation -> { throw new UnsupportedOperationException("Not yet implemented"); @@ -102,7 +98,7 @@ public void setupTestClass() throws Exception { doAnswer(invocation -> { final DatasetId id = invocation.getArgument(0); - Namespace namespaceMock = Mockito.mock(LocalNamespace.class); + Namespace namespaceMock = mock(LocalNamespace.class); if (id.equals(datasetId)) { when(namespaceMock.getDataset()).thenReturn(dataset); } @@ -123,14 +119,18 @@ else if (id.equals(datasetId1)) { ((MutableInjectableValues) FormConfigProcessor.getMAPPER().getInjectableValues()) .add(IdResolveContext.class, namespacesMock); processor = new FormConfigProcessor(validator, storage, namespacesMock); - controller = new AuthorizationController(storage, config, new Environment(this.getClass().getSimpleName()), null); + AuthorizationController controller = new AuthorizationController(storage, config, new Environment(this.getClass().getSimpleName()), null); controller.start(); } @BeforeEach public void setupTest() { - final ManagedQuery managedQuery = new ManagedQuery(null, null, dataset, null); + + user = new User("test", "test", storage); + storage.addUser(user); + + final ManagedQuery managedQuery = new ManagedQuery(mock(Query.class), user, dataset, null); managedQuery.setQueryId(UUID.randomUUID()); form = new ExportForm(); @@ -138,10 +138,6 @@ public void setupTest() { form.setTimeMode(mode); form.setQueryGroupId(managedQuery.getId()); mode.setForm(form); - - - user = new User("test", "test", storage); - storage.addUser(user); } @AfterEach @@ -161,7 +157,7 @@ public void addConfigWithoutTranslation() { processor.addConfig(user, dataset, formConfig); - assertThat(storage.getAllFormConfigs()).containsExactly(formConfig.intern(user, dataset)); + assertThat(storage.getAllFormConfigs()).containsExactly(formConfig.intern(user, dataset.getId())); } @Test @@ -171,7 +167,7 @@ public void deleteConfig() { ObjectMapper mapper = FormConfigProcessor.getMAPPER(); FormConfig formConfig = new FormConfig(form.getClass().getAnnotation(CPSType.class).id(), mapper.valueToTree(form)); - formConfig.setDataset(dataset); + formConfig.setDataset(dataset.getId()); user.addPermission(formConfig.createPermission(AbilitySets.FORM_CONFIG_CREATOR)); storage.addFormConfig(formConfig); @@ -193,7 +189,7 @@ public void getConfig() { ObjectMapper mapper = FormConfigProcessor.getMAPPER(); JsonNode values = mapper.valueToTree(form); FormConfig formConfig = new FormConfig(form.getClass().getAnnotation(CPSType.class).id(), values); - formConfig.setDataset(dataset); + formConfig.setDataset(dataset.getId()); formConfig.setOwner(user); user.addPermission(formConfig.createPermission(Ability.READ.asSet())); storage.addFormConfig(formConfig); @@ -337,7 +333,7 @@ public void patchConfig() { // CHECK PART 1 FormConfig patchedFormExpected = new FormConfig(form.getClass().getAnnotation(CPSType.class).id(), values); - patchedFormExpected.setDataset(dataset); + patchedFormExpected.setDataset(dataset.getId()); patchedFormExpected.setFormId(config.getFormId()); patchedFormExpected.setLabel("newTestLabel"); patchedFormExpected.setShared(true); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index ef12549871..a5e7bf4fd3 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -16,6 +16,10 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.apiv1.query.ConceptQuery; @@ -48,10 +52,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; import com.univocity.parsers.csv.CsvParser; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; import lombok.NonNull; import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; @@ -74,7 +74,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData ConceptQuery query = new ConceptQuery(new CQExternal(Arrays.asList("ID", "DATE_SET"), data, false)); ExecutionManager executionManager = support.getNamespace().getExecutionManager(); - ManagedExecution managed = executionManager.createQuery(query, queryId, user, support.getNamespace().getDataset(), false); + ManagedExecution managed = executionManager.createExecution(query, queryId, user, support.getNamespace(), false); user.addPermission(managed.createPermission(AbilitySets.QUERY_CREATOR)); @@ -89,7 +89,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData UUID queryId = new UUID(0L, id++); ExecutionManager executionManager = support.getNamespace().getExecutionManager(); - ManagedExecution managed = executionManager.createQuery(query, queryId, user, support.getNamespace().getDataset(), false); + ManagedExecution managed = executionManager.createExecution(query, queryId, user, support.getNamespace(), false); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); @@ -247,7 +247,7 @@ public static void uploadConcept(StandaloneSupport support, Dataset dataset, Con } - private static List> getConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException { + private static List> getConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws IOException { return ConqueryTestSpec.parseSubTreeList( support, rawConcepts, diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 0fbda3acee..8f68962360 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -11,21 +11,17 @@ import java.util.Map; import java.util.OptionalLong; import java.util.concurrent.TimeUnit; - import jakarta.validation.Valid; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.forms.Form; -import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.common.ResourceFile; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.csv.CsvRenderer; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.forms.managed.ManagedForm; @@ -101,10 +97,10 @@ public void executeTest(StandaloneSupport support) throws Exception { ManagedInternalForm managedForm = (ManagedInternalForm) support .getNamespace() .getExecutionManager() - .runQuery(namespace, form, support.getTestUser(), support.getDataset(), support.getConfig(), false); + .runQuery(namespace, form, support.getTestUser(), support.getConfig(), false); - managedForm.awaitDone(10, TimeUnit.MINUTES); - if (managedForm.getState() != ExecutionState.DONE) { + ExecutionState executionState = namespace.getExecutionManager().awaitDone(managedForm, 10, TimeUnit.MINUTES); + if (executionState != ExecutionState.DONE) { if (managedForm.getState() == ExecutionState.FAILED) { fail(getLabel() + " Query failed"); } @@ -185,7 +181,7 @@ private void checkMultipleResult(Map> managedMapping, * * @see FormTest#checkMultipleResult(Map, ConqueryConfig, PrintSettings) */ - private void checkSingleResult(F managedForm, ConqueryConfig config, PrintSettings printSettings) + private & SingleTableResult> void checkSingleResult(F managedForm, ConqueryConfig config, PrintSettings printSettings) throws IOException { @@ -213,25 +209,6 @@ private void checkSingleResult(F man } - private static void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws JSONException, IOException { - if (rawConcepts == null) { - return; - } - - Dataset dataset = support.getDataset(); - - List> concepts = parseSubTreeList( - support, - rawConcepts, - Concept.class, - c -> c.setDataset(support.getDataset()) - ); - - for (Concept concept : concepts) { - LoadingUtil.uploadConcept(support, dataset, concept); - } - } - private Form parseForm(StandaloneSupport support) throws JSONException, IOException { return parseSubTree(support, rawForm, Form.class); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index e6cacdd25c..f3fab1bae2 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -10,13 +10,14 @@ import java.time.ZonedDateTime; import java.util.Collections; import java.util.List; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.apiv1.frontend.FrontendConfiguration; import com.bakdata.conquery.apiv1.frontend.VersionContainer; import com.bakdata.conquery.integration.common.IntegrationUtils; -import com.bakdata.conquery.io.result.ExternalResult; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.FormBackendConfig; @@ -26,14 +27,13 @@ import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner; +import com.bakdata.conquery.models.forms.managed.ExternalExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.resources.api.ConfigResource; import com.bakdata.conquery.resources.api.ResultExternalResource; import com.bakdata.conquery.resources.hierarchies.HierarchyHelper; import com.bakdata.conquery.util.support.StandaloneSupport; import com.bakdata.conquery.util.support.TestConquery; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.UriBuilder; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -97,19 +97,19 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Generate asset urls and check them in the status final ManagedExecution storedExecution = testConquery.getSupport(name).getMetaStorage().getExecution(managedExecutionId); final URI - downloadURLasset1 = - ResultExternalResource.getDownloadURL(apiUriBuilder.clone(), (ManagedExecution & ExternalResult) storedExecution, executionStatus.getResultUrls() - .get(0) - .getAssetId()); + downloadUrlAsset1 = + ResultExternalResource.getDownloadURL(apiUriBuilder.clone(), (ExternalExecution) storedExecution, executionStatus.getResultUrls() + .get(0) + .getAssetId()); final URI - downloadURLasset2 = - ResultExternalResource.getDownloadURL(apiUriBuilder.clone(), (ManagedExecution & ExternalResult) storedExecution, executionStatus.getResultUrls() + downloadUrlAsset2 = + ResultExternalResource.getDownloadURL(apiUriBuilder.clone(), (ExternalExecution) storedExecution, executionStatus.getResultUrls() .get(1) .getAssetId()); assertThat(executionStatus.getStatus()).isEqualTo(ExecutionState.DONE); - assertThat(executionStatus.getResultUrls()).containsExactly(new ResultAsset("Result", downloadURLasset1), new ResultAsset("Another Result", downloadURLasset2)); + assertThat(executionStatus.getResultUrls()).containsExactly(new ResultAsset("Result", downloadUrlAsset1), new ResultAsset("Another Result", downloadUrlAsset2)); log.info("Download Result"); final String diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java index eb1975b203..4cc81e80da 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultTestUtil.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.io.result; +import static org.mockito.Mockito.mock; + import java.util.Collections; import java.util.List; import java.util.OptionalLong; @@ -7,8 +9,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.ManagedQuery; @@ -69,7 +74,7 @@ public static List getTestEntityResults() { @NotNull public static ManagedQuery getTestQuery() { - return new ManagedQuery(null, null, null, null) { + return new ManagedQuery(mock(Query.class), mock(User.class), new Dataset(ResultTestUtil.class.getSimpleName()), null) { @Override public List getResultInfos() { return getResultTypes().stream() diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java index eff77b3c41..71c39c7b51 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/excel/ExcelResultRenderTest.java @@ -3,6 +3,7 @@ import static com.bakdata.conquery.io.result.ResultTestUtil.getResultTypes; import static com.bakdata.conquery.io.result.ResultTestUtil.getTestEntityResults; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -16,10 +17,13 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.result.ResultTestUtil; +import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.ExcelConfig; +import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.query.ManagedQuery; @@ -65,7 +69,7 @@ void writeAndRead() throws IOException { // The Shard nodes send Object[] but since Jackson is used for deserialization, nested collections are always a list because they are not further specialized List results = getTestEntityResults(); - ManagedQuery mquery = new ManagedQuery(null, null, null, null) { + ManagedQuery mquery = new ManagedQuery(mock(Query.class), mock(User.class), new Dataset(ExcelResultRenderTest.class.getSimpleName()), null) { public List getResultInfos() { return getResultTypes().stream() .map(ResultTestUtil.TypedSelectDummy::new) @@ -111,7 +115,6 @@ private List readComputed(InputStream inputStream, PrintSettings setting XSSFSheet sheet = workbook.getSheetAt(0); List computed = new ArrayList<>(); - int i = 0; for (Row row : sheet) { StringJoiner sj = new StringJoiner("\t"); DataFormatter formatter = new DataFormatter(settings.getLocale()); @@ -127,7 +130,6 @@ private List readComputed(InputStream inputStream, PrintSettings setting sj.add(formatted); } computed.add(sj.toString()); - i++; } return computed; } @@ -164,9 +166,7 @@ private void joinValue(PrintSettings settings, StringJoiner valueJoiner, Object String printVal = info.getType().printNullable(settings, val); if (info.getType().equals(ResultType.BooleanT.INSTANCE)) { - /** - * Even though we set the locale to GERMAN, poi's {@link DataFormatter#formatCellValue(Cell)} hardcoded english booleans - */ + // Even though we set the locale to GERMAN, poi's {@link DataFormatter#formatCellValue(Cell)} hardcoded english booleans printVal = (Boolean) val ? "TRUE" : "FALSE"; } diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index 96b4c394e9..069cdcb456 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -1,12 +1,15 @@ package com.bakdata.conquery.io.storage.xodus.stores; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; import java.io.File; import java.io.IOException; import java.util.concurrent.Executors; +import jakarta.validation.Validator; import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.io.jackson.Jackson; @@ -22,7 +25,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.io.Files; import io.dropwizard.jersey.validation.Validators; -import jakarta.validation.Validator; import jetbrains.exodus.env.Environment; import jetbrains.exodus.env.Environments; import lombok.extern.slf4j.Slf4j; @@ -42,7 +44,7 @@ public class SerializingStoreDumpTest { private ObjectMapper objectMapper; // Test data - private final ManagedQuery managedQuery = new ManagedQuery(null, null, new Dataset("dataset"), STORAGE); + private final ManagedQuery managedQuery = new ManagedQuery(mock(Query.class), mock(User.class), new Dataset("dataset"), STORAGE); private final ConceptQuery cQuery = new ConceptQuery( new CQReusedQuery(managedQuery.getId())); private final User user = new User("username", "userlabel", STORAGE); diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index a540a9a655..5b896054ee 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -8,7 +8,14 @@ import java.net.URISyntaxException; import java.time.LocalDate; import java.time.ZonedDateTime; -import java.util.*; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Random; +import java.util.UUID; import java.util.stream.Stream; import jakarta.validation.Validator; @@ -316,7 +323,7 @@ public void formConfig() throws JSONException, IOException { ObjectMapper mapper = FormConfigProcessor.getMAPPER(); JsonNode values = mapper.valueToTree(form); FormConfig formConfig = new FormConfig(form.getClass().getAnnotation(CPSType.class).id(), values); - formConfig.setDataset(dataset); + formConfig.setDataset(dataset.getId()); SerializationTestUtil .forType(FormConfig.class) diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index f2b1bece25..71c2be7212 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -3,6 +3,7 @@ import static com.bakdata.conquery.models.execution.ManagedExecution.AUTO_LABEL_SUFFIX; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; import java.time.LocalDateTime; import java.util.List; @@ -11,6 +12,7 @@ import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; @@ -31,13 +33,12 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; -import org.mockito.Mockito; public class DefaultLabelTest { private final static MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); - private static final Namespace NAMESPACE = Mockito.mock(LocalNamespace.class); + private static final Namespace NAMESPACE = mock(LocalNamespace.class); private static final Dataset DATASET = new Dataset("dataset"); private static final User user = new User("user","user", STORAGE); @@ -126,7 +127,7 @@ void autoLabelConceptQueryFallback(Locale locale, String autoLabel) { void autoLabelReusedQuery(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); - final ManagedQuery managedQuery = new ManagedQuery(null, null, DATASET, STORAGE); + final ManagedQuery managedQuery = new ManagedQuery(mock(Query.class), mock(User.class), DATASET, STORAGE); managedQuery.setQueryId(UUID.randomUUID()); CQReusedQuery reused = new CQReusedQuery(managedQuery.getId()); @@ -166,7 +167,7 @@ void autoLabelUploadQuery(Locale locale, String autoLabel) { void autoLabelComplexQuery(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); - final ManagedQuery managedQuery = new ManagedQuery(null, null, DATASET, STORAGE); + final ManagedQuery managedQuery = new ManagedQuery(mock(Query.class), mock(User.class), DATASET, STORAGE); managedQuery.setQueryId(UUID.randomUUID()); CQAnd and = new CQAnd(); @@ -199,7 +200,7 @@ void autoLabelComplexQuery(Locale locale, String autoLabel) { void autoLabelComplexQueryNullLabels(Locale locale, String autoLabel) { I18n.LOCALE.set(locale); - final ManagedQuery managedQuery = new ManagedQuery(null, null, DATASET, STORAGE); + final ManagedQuery managedQuery = new ManagedQuery(mock(Query.class), mock(User.class), DATASET, STORAGE); managedQuery.setQueryId(UUID.randomUUID()); CQAnd and = new CQAnd(); diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java index 301a2597dc..39202011ea 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/PermissionCleanupTaskTest.java @@ -3,6 +3,7 @@ import static com.bakdata.conquery.tasks.PermissionCleanupTask.deletePermissionsOfOwnedInstances; import static com.bakdata.conquery.tasks.PermissionCleanupTask.deleteQueryPermissionsWithMissingRef; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; import java.time.Instant; import java.time.LocalDateTime; @@ -41,7 +42,7 @@ private ManagedQuery createManagedQuery() { ConceptQuery query = new ConceptQuery(root); - final ManagedQuery managedQuery = new ManagedQuery(query, null, new Dataset("test"), STORAGE); + final ManagedQuery managedQuery = new ManagedQuery(query, mock(User.class), new Dataset("test"), STORAGE); managedQuery.setCreationTime(LocalDateTime.now().minusDays(1)); diff --git a/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java b/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java index c8d9a47a17..34707104d6 100644 --- a/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java +++ b/backend/src/test/java/com/bakdata/conquery/tasks/QueryCleanupTaskTest.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.tasks; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; import java.time.Duration; import java.time.LocalDateTime; @@ -13,6 +14,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.util.NonPersistentStoreFactory; @@ -34,7 +36,7 @@ private ManagedQuery createManagedQuery() { ConceptQuery query = new ConceptQuery(root); - final ManagedQuery managedQuery = new ManagedQuery(query, null, new Dataset("test"), STORAGE); + final ManagedQuery managedQuery = new ManagedQuery(query, mock(User.class), new Dataset("test"), STORAGE); managedQuery.setCreationTime(LocalDateTime.now().minus(queryExpiration).minusDays(1)); diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 00f9822cd3..45191a5178 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -10,7 +10,6 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import jakarta.validation.Validator; import jakarta.ws.rs.client.Client; @@ -61,12 +60,10 @@ public class TestConquery { private StandaloneCommand standaloneCommand; @Getter private DropwizardTestSupport dropwizard; - private Set openSupports = new HashSet<>(); + private final Set openSupports = new HashSet<>(); @Getter private Client client; - private AtomicBoolean started = new AtomicBoolean(false); - /** * Returns the extension context used by the beforeAll-callback. * @@ -117,7 +114,7 @@ public void beforeAll() throws Exception { // define server - dropwizard = new DropwizardTestSupport(TestBootstrappingConquery.class, config, app -> { + dropwizard = new DropwizardTestSupport<>(TestBootstrappingConquery.class, config, app -> { if (config.getSqlConnectorConfig().isEnabled()) { standaloneCommand = new SqlStandaloneCommand((Conquery) app); } @@ -131,12 +128,12 @@ public void beforeAll() throws Exception { // create HTTP client for api tests client = new JerseyClientBuilder(this.getDropwizard().getEnvironment()) - .withProperty(ClientProperties.CONNECT_TIMEOUT, 10000) - .withProperty(ClientProperties.READ_TIMEOUT, 10000) + .withProperty(ClientProperties.CONNECT_TIMEOUT, 100000) + .withProperty(ClientProperties.READ_TIMEOUT, 100000) .build("test client"); } - public void afterAll() throws Exception { + public void afterAll() { client.close(); dropwizard.after(); FileUtils.deleteQuietly(tmpDir); From 9f6e7eab4bd57e1898c2da85f19e8edc998ac13f Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:00:43 +0200 Subject: [PATCH 10/40] removes unused member from class --- .../conquery/models/forms/managed/ExternalExecution.java | 2 +- .../com/bakdata/conquery/models/query/ExternalResultImpl.java | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java index ffa07cbe6d..85b2e00d0e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java @@ -95,7 +95,7 @@ public void start() { final ExternalTaskState externalTaskState = api.postForm(getSubmitted(), originalUser, serviceUser, dataset); - executionManager.addResult(this, new ExternalResultImpl(new CountDownLatch(0), api, formBackendConfig, serviceUser)); + executionManager.addResult(this, new ExternalResultImpl(new CountDownLatch(0), api, serviceUser)); externalTaskId = externalTaskState.getId(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java index 79936f5a45..3575e4dad0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java @@ -10,7 +10,6 @@ import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; import com.bakdata.conquery.io.result.ExternalResult; import com.bakdata.conquery.models.auth.entities.User; -import com.bakdata.conquery.models.config.FormBackendConfig; import com.google.common.collect.MoreCollectors; import it.unimi.dsi.fastutil.Pair; import lombok.Getter; @@ -24,8 +23,6 @@ public class ExternalResultImpl implements ExternalResult { @Getter private final ExternalFormBackendApi api; - private final FormBackendConfig formBackendConfig; - @Getter private final User serviceUser; From 4d90441cd89bef687d68bcf4b8541ace343239ab Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:05:01 +0200 Subject: [PATCH 11/40] revert client timeouts --- .../java/com/bakdata/conquery/util/support/TestConquery.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 45191a5178..d55f20e988 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -128,8 +128,8 @@ public void beforeAll() throws Exception { // create HTTP client for api tests client = new JerseyClientBuilder(this.getDropwizard().getEnvironment()) - .withProperty(ClientProperties.CONNECT_TIMEOUT, 100000) - .withProperty(ClientProperties.READ_TIMEOUT, 100000) + .withProperty(ClientProperties.CONNECT_TIMEOUT, 10000) + .withProperty(ClientProperties.READ_TIMEOUT, 10000) .build("test client"); } From bdd05d9bebc5d55750a54f23ce197aee54e3a03e Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 8 Aug 2024 12:07:14 +0200 Subject: [PATCH 12/40] cleanup --- .../bakdata/conquery/util/support/TestConquery.java | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index d55f20e988..965f91ee9c 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -43,7 +43,6 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; import org.glassfish.jersey.client.ClientProperties; -import org.junit.jupiter.api.extension.ExtensionContext; /** * Represents the test instance of Conquery. @@ -64,13 +63,6 @@ public class TestConquery { @Getter private Client client; - /** - * Returns the extension context used by the beforeAll-callback. - * - * @return The context. - */ - @Getter - private ExtensionContext beforeAllContext; // Initial user which is set before each test from the config. private User testUser; @@ -139,7 +131,7 @@ public void afterAll() { FileUtils.deleteQuietly(tmpDir); } - public void afterEach() throws Exception { + public void afterEach() { synchronized (openSupports) { for (StandaloneSupport openSupport : openSupports) { removeSupportDataset(openSupport); From 9b78a5c8651426dba3eb1c1c7be401aa4b7db93a Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 9 Aug 2024 14:06:53 +0200 Subject: [PATCH 13/40] enable healthcheck for sql connection --- .../conquery/commands/ManagerNode.java | 2 +- .../conquery/mode/NamespaceHandler.java | 3 +- .../mode/cluster/ClusterNamespaceHandler.java | 3 +- .../mode/local/LocalNamespaceHandler.java | 5 +- .../models/worker/DatasetRegistry.java | 9 +- .../resources/admin/AdminServlet.java | 28 ++++++- .../admin/rest/AdminDatasetProcessor.java | 17 ++-- .../conquery/sql/DslContextFactory.java | 9 +- .../sql/dialect/HanaSqlIntegrationTests.java | 4 +- .../dialect/PostgreSqlIntegrationTests.java | 84 ++++++++++++++++--- .../io/AbstractSerializationTest.java | 3 +- 11 files changed, 129 insertions(+), 38 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index d48094fbad..3005de93e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -241,7 +241,7 @@ public void loadNamespaces() { final Collection namespaceStorages = getConfig().getStorage().discoverNamespaceStorages(); for (NamespaceStorage namespaceStorage : namespaceStorages) { loaders.submit(() -> { - registry.createNamespace(namespaceStorage, getMetaStorage()); + registry.createNamespace(namespaceStorage, getMetaStorage(), getEnvironment()); }); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java index af83f42f13..9e132e7139 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java @@ -15,6 +15,7 @@ import com.bakdata.conquery.models.query.FilterSearch; import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.databind.ObjectMapper; +import io.dropwizard.core.setup.Environment; /** * Handler of namespaces in a ConQuery instance. @@ -23,7 +24,7 @@ */ public interface NamespaceHandler { - N createNamespace(NamespaceStorage storage, MetaStorage metaStorage, IndexService indexService); + N createNamespace(NamespaceStorage storage, MetaStorage metaStorage, IndexService indexService, Environment environment); void removeNamespace(DatasetId id, N namespace); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java index 617f166d72..9ea889b2b0 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.models.worker.WorkerHandler; +import io.dropwizard.core.setup.Environment; import lombok.RequiredArgsConstructor; @RequiredArgsConstructor @@ -23,7 +24,7 @@ public class ClusterNamespaceHandler implements NamespaceHandler { private final SqlDialectFactory dialectFactory; @Override - public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, IndexService indexService) { + public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, IndexService indexService, Environment environment) { NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator, indexService); @@ -45,7 +46,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig(); DatabaseConfig databaseConfig = sqlConnectorConfig.getDatabaseConfig(namespaceStorage.getDataset()); - DSLContextWrapper dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig); + DSLContextWrapper dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig, environment.healthChecks()); DSLContext dslContext = dslContextWrapper.getDslContext(); SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index 64606760ff..c633260b01 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -28,6 +28,7 @@ import com.fasterxml.jackson.annotation.JsonIgnoreType; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.cache.CacheStats; +import io.dropwizard.core.setup.Environment; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -50,7 +51,7 @@ public class DatasetRegistry extends IdResolveContext imple private final IndexService indexService; - public N createNamespace(Dataset dataset, MetaStorage metaStorage) throws IOException { + public N createNamespace(Dataset dataset, MetaStorage metaStorage, Environment environment) throws IOException { // Prepare empty storage NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName()); final ObjectMapper persistenceMapper = internalObjectMapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); @@ -63,11 +64,11 @@ public N createNamespace(Dataset dataset, MetaStorage metaStorage) throws IOExce datasetStorage.setPreviewConfig(new PreviewConfig()); datasetStorage.close(); - return createNamespace(datasetStorage, metaStorage); + return createNamespace(datasetStorage, metaStorage, environment); } - public N createNamespace(NamespaceStorage datasetStorage, MetaStorage metaStorage) { - final N namespace = namespaceHandler.createNamespace(datasetStorage, metaStorage, indexService); + public N createNamespace(NamespaceStorage datasetStorage, MetaStorage metaStorage, Environment environment) { + final N namespace = namespaceHandler.createNamespace(datasetStorage, metaStorage, indexService, environment); add(namespace); return namespace; } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java index a41e6e8600..f697609f60 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java @@ -19,8 +19,28 @@ import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.resources.admin.rest.*; -import com.bakdata.conquery.resources.admin.ui.*; +import com.bakdata.conquery.resources.admin.rest.AdminConceptsResource; +import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; +import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource; +import com.bakdata.conquery.resources.admin.rest.AdminDatasetsResource; +import com.bakdata.conquery.resources.admin.rest.AdminProcessor; +import com.bakdata.conquery.resources.admin.rest.AdminResource; +import com.bakdata.conquery.resources.admin.rest.AdminTablesResource; +import com.bakdata.conquery.resources.admin.rest.AuthOverviewResource; +import com.bakdata.conquery.resources.admin.rest.GroupResource; +import com.bakdata.conquery.resources.admin.rest.PermissionResource; +import com.bakdata.conquery.resources.admin.rest.RoleResource; +import com.bakdata.conquery.resources.admin.rest.UIProcessor; +import com.bakdata.conquery.resources.admin.rest.UserResource; +import com.bakdata.conquery.resources.admin.ui.AdminUIResource; +import com.bakdata.conquery.resources.admin.ui.AuthOverviewUIResource; +import com.bakdata.conquery.resources.admin.ui.ConceptsUIResource; +import com.bakdata.conquery.resources.admin.ui.DatasetsUIResource; +import com.bakdata.conquery.resources.admin.ui.GroupUIResource; +import com.bakdata.conquery.resources.admin.ui.IndexServiceUIResource; +import com.bakdata.conquery.resources.admin.ui.RoleUIResource; +import com.bakdata.conquery.resources.admin.ui.TablesUIResource; +import com.bakdata.conquery.resources.admin.ui.UserUIResource; import com.bakdata.conquery.resources.admin.ui.model.ConnectorUIResource; import io.dropwizard.core.setup.AdminEnvironment; import io.dropwizard.jersey.DropwizardResourceConfig; @@ -79,12 +99,12 @@ public AdminServlet(ManagerNode manager) { adminDatasetProcessor = new AdminDatasetProcessor( manager.getConfig(), - manager.getValidator(), manager.getDatasetRegistry(), manager.getMetaStorage(), manager.getJobManager(), manager.getImportHandler(), - manager.getStorageListener() + manager.getStorageListener(), + manager.getEnvironment() ); jerseyConfig.register(new AbstractBinder() { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index 875b65abd4..20d65886af 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -9,7 +9,6 @@ import java.util.Set; import java.util.stream.Collectors; import jakarta.inject.Inject; -import jakarta.validation.Validator; import jakarta.ws.rs.ForbiddenException; import jakarta.ws.rs.NotFoundException; import jakarta.ws.rs.WebApplicationException; @@ -42,6 +41,7 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.univocity.parsers.csv.CsvParser; +import io.dropwizard.core.setup.Environment; import lombok.Data; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -58,12 +58,13 @@ public class AdminDatasetProcessor { private static final String ABBREVIATION_MARKER = "\u2026"; private final ConqueryConfig config; - private final Validator validator; private final DatasetRegistry datasetRegistry; private final MetaStorage metaStorage; private final JobManager jobManager; private final ImportHandler importHandler; private final StorageListener storageListener; + private final Environment environment; + /** @@ -76,7 +77,7 @@ public synchronized Dataset addDataset(Dataset dataset) throws IOException { throw new WebApplicationException("Dataset already exists", Response.Status.CONFLICT); } - return datasetRegistry.createNamespace(dataset, metaStorage).getDataset(); + return datasetRegistry.createNamespace(dataset, metaStorage, environment).getDataset(); } /** @@ -168,7 +169,7 @@ else if (!table.getDataset().equals(dataset)) { throw new WebApplicationException("Table already exists", Response.Status.CONFLICT); } - ValidatorHelper.failOnError(log, validator.validate(table)); + ValidatorHelper.failOnError(log, environment.getValidator().validate(table)); namespace.getStorage().addTable(table); storageListener.onAddTable(table); @@ -194,7 +195,7 @@ public synchronized void updateConcept(@NonNull Dataset dataset, @NonNull Concep */ public synchronized void addConcept(@NonNull Dataset dataset, @NonNull Concept concept, boolean force) { concept.setDataset(dataset); - ValidatorHelper.failOnError(log, validator.validate(concept)); + ValidatorHelper.failOnError(log, environment.getValidator().validate(concept)); if (datasetRegistry.get(dataset.getId()).getStorage().hasConcept(concept.getId())) { if (!force) { @@ -215,7 +216,7 @@ public synchronized void addConcept(@NonNull Dataset dataset, @NonNull Concept POSTGRESQL_CONTAINER = new PostgreSQLContainer<>(postgreSqlImageName) - .withDatabaseName(DATABASE_NAME) - .withUsername(USERNAME) - .withPassword(PASSWORD); @BeforeAll static void before() { - POSTGRESQL_CONTAINER.start(); - databaseConfig = DatabaseConfig.builder() - .dialect(Dialect.POSTGRESQL) - .jdbcConnectionUrl(POSTGRESQL_CONTAINER.getJdbcUrl()) - .databaseUsername(USERNAME) - .databasePassword(PASSWORD) - .build(); + TestContextProvider provider = useLocalPostgresDb + ? new PortgresTestcontainerContextProvider() + : new RemotePostgresContextProvider(); + + databaseConfig = provider.getDatabaseConfig(); sqlConfig = new TestSqlConnectorConfig(databaseConfig); - dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConfig); + dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConfig, null); testSqlDialect = new TestPostgreSqlDialect(); testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContextWrapper.getDslContext(), testSqlDialect, databaseConfig)); } @@ -131,4 +134,59 @@ protected TestSqlQuery(String sql) { } } + @Getter + private static class PortgresTestcontainerContextProvider implements TestContextProvider { + + private final DSLContextWrapper dslContextWrapper; + private final DatabaseConfig databaseConfig; + private final TestSqlConnectorConfig sqlConnectorConfig; + + @Container + private final PostgreSQLContainer postgreSQLContainer; + + public PortgresTestcontainerContextProvider() { + this.postgreSQLContainer = new PostgreSQLContainer<>(postgreSqlImageName) + .withDatabaseName(DATABASE_NAME) + .withUsername(USERNAME) + .withPassword(PASSWORD); + this.postgreSQLContainer.start(); + this.databaseConfig = DatabaseConfig.builder() + .dialect(Dialect.POSTGRESQL) + .jdbcConnectionUrl(postgreSQLContainer.getJdbcUrl()) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); + + this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); + this.dslContextWrapper = DslContextFactory.create(this.databaseConfig, sqlConnectorConfig, null); + } + + } + + + @Getter + private static class RemotePostgresContextProvider implements TestContextProvider { + + private final static String PORT = Objects.requireNonNullElse(System.getenv("CONQUERY_SQL_PORT"), "39041"); + private final static String HOST = System.getenv("CONQUERY_SQL_HOST"); + private final static String DATABASE = System.getenv("CONQUERY_SQL_DATABASE"); + private final static String CONNECTION_URL = "jdbc:postgresql://%s:%s/%s".formatted(HOST, PORT, DATABASE); + private final static String USERNAME = System.getenv("CONQUERY_SQL_USER"); + private final static String PASSWORD = System.getenv("CONQUERY_SQL_PASSWORD"); + private final DSLContextWrapper dslContextWrapper; + private final DatabaseConfig databaseConfig; + private final TestSqlConnectorConfig sqlConnectorConfig; + + public RemotePostgresContextProvider() { + this.databaseConfig = DatabaseConfig.builder() + .dialect(Dialect.POSTGRESQL) + .jdbcConnectionUrl(CONNECTION_URL) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); + this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); + this.dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig, null); + } + + } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index 11ead26f2c..1991157c6e 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -24,6 +24,7 @@ import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.codahale.metrics.SharedMetricRegistries; import com.fasterxml.jackson.databind.ObjectMapper; +import io.dropwizard.core.setup.Environment; import io.dropwizard.jersey.validation.Validators; import lombok.Getter; import org.junit.jupiter.api.BeforeAll; @@ -65,7 +66,7 @@ public void before() throws IOException { datasetRegistry = new DatasetRegistry<>(0, config, creator, clusterNamespaceHandler, indexService); creator.init(datasetRegistry); - namespace = datasetRegistry.createNamespace(new Dataset("serialization_test"), metaStorage); + namespace = datasetRegistry.createNamespace(new Dataset("serialization_test"), metaStorage, new Environment(this.getClass().getSimpleName())); // Prepare manager meta internal mapper managerMetaInternalMapper = creator.createInternalObjectMapper(View.Persistence.Manager.class); From 6bc226745643f9395f351c7f839f2fa7c5b3d966 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 9 Aug 2024 14:29:46 +0200 Subject: [PATCH 14/40] makes connectivityCheckTimeoutMs configurable --- .../conquery/models/config/SqlConnectorConfig.java | 8 +++++++- .../java/com/bakdata/conquery/sql/DslContextFactory.java | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index c8140ef205..fd11752e78 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.models.config; import java.util.Map; +import jakarta.validation.Valid; import com.bakdata.conquery.models.datasets.Dataset; import com.fasterxml.jackson.annotation.JsonIgnore; +import io.dropwizard.util.Duration; import io.dropwizard.validation.ValidationMethod; -import jakarta.validation.Valid; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -31,6 +32,11 @@ public class SqlConnectorConfig { */ private Map databaseConfigs; + /** + * Timeout duration after which a database connection is considered unhealthy (defaults to connection timeout) + */ + private Duration connectivityCheckTimeout; + public DatabaseConfig getDatabaseConfig(Dataset dataset) { return databaseConfigs.get(dataset.getName()); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java index a7953cfd42..59f13756c6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java @@ -24,6 +24,10 @@ public static DSLContextWrapper create(DatabaseConfig config, SqlConnectorConfig if (healthCheckRegistry != null) { hikariConfig.setHealthCheckRegistry(healthCheckRegistry); + if (connectorConfig.getConnectivityCheckTimeout() != null) { + long connectivityTimeoutMs = connectorConfig.getConnectivityCheckTimeout().toMilliseconds(); + hikariConfig.addHealthCheckProperty("connectivityCheckTimeoutMs", Long.toString(connectivityTimeoutMs)); + } } HikariDataSource hikariDataSource = new HikariDataSource(hikariConfig); From 03fb266e2d352afd654f43a3c7b8dc64ed0477d5 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 13 Aug 2024 09:29:04 +0200 Subject: [PATCH 15/40] fix whitespaces that broke freemarker --- .../resources/admin/ui/queries.html.ftl | 489 +++++++++--------- 1 file changed, 244 insertions(+), 245 deletions(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/queries.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/queries.html.ftl index c73c81cd19..032b5ed33a 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/queries.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/queries.html.ftl @@ -1,256 +1,255 @@ <#import "templates/template.html.ftl" as layout> - <@layout.layout> - + -

Queries

- -
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
-
-
- -
-
-
-
-
-
-
-
-
- \ No newline at end of file + function handleUpdateCheck(event) { + if (event.checked) { + reloader = setInterval(getQueries, 5000); + return; + } + clearInterval(reloader); + reloader = 0; + } + <#noparse > + function getHtmltemplate(data, queryCounter) { + return ` +
+
+
+
+
+ ${data.label} ( ${data.ownerName} , ${data.id.split('.')[0]} ) : ${data.id.split('.')[1]} +
+
+
+
+ Creation-Time : ${((new Date(data.createdAt)).toLocaleString(languageTag))} +
+
+ Start-Time : ${((new Date(data.startTime)).toLocaleString(languageTag))} +
+
+ Finish-Time : ${((new Date(data.finishTime)).toLocaleString(languageTag))} +
+
+
+
+ Requested-Time : ${data.requiredTime} ms +
+
+ Query-Type : ${data.queryType} +
+
+ + +
+
+
+
+
+
+ ${(data.progress && data.progress != null ? data.progress*100 : 0 )} % +
+
+
+
+
+
+
+
+ 
  ${(data.query ? JSON.stringify(data.query, undefined, 2) : '')}
+
+
+
+
+
+
+ 
  ${(data.error ? JSON.stringify(data.error, undefined, 2) : '')}
+
+
+
+
+
+
+
+ ` + } + + +

Queries

+ +
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+
+
+ +
+
+
+
+
+
+
+
+
+ \ No newline at end of file From 08f94d6d130993abca1afb4036e781d34ccc7cd9 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 13 Aug 2024 11:02:02 +0200 Subject: [PATCH 16/40] adds smoke tests for queries and jobs ui --- .../backend-admin-ui/test_3_smoketest.cy.js | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 cypress/e2e/backend-admin-ui/test_3_smoketest.cy.js diff --git a/cypress/e2e/backend-admin-ui/test_3_smoketest.cy.js b/cypress/e2e/backend-admin-ui/test_3_smoketest.cy.js new file mode 100644 index 0000000000..a44dd12331 --- /dev/null +++ b/cypress/e2e/backend-admin-ui/test_3_smoketest.cy.js @@ -0,0 +1,25 @@ +/// +import { visitAdminUI } from "../../integration-helpers/visitAdminUI"; + +context("Simplest Smoke Tests", () => { + + describe("UI renders", () => { + + it("Query Overview", () => { + + visitAdminUI("queries"); + + // Disable updates so the test ends + cy.get('#updateCheckBox').click() + cy.root().should('not.contain.text', 'FreeMarker template error') + }); + + + it("Jobs Overview", () => { + + visitAdminUI("jobs"); + + cy.root().should('not.contain.text', 'FreeMarker template error') + }); + }); +}) \ No newline at end of file From 6018205278bc53f2126cf76d1e49fe0b4c2ce29f Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 13 Aug 2024 12:17:58 +0200 Subject: [PATCH 17/40] fix missing argument --- .../integration/sql/dialect/TestSqlConnectorConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java index 5eeddd3500..bf3c1a5325 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java @@ -15,7 +15,7 @@ public class TestSqlConnectorConfig extends SqlConnectorConfig { private static final String TEST_DATASET = "test"; public TestSqlConnectorConfig(DatabaseConfig databaseConfig) { - super(true, true, Map.of(TEST_DATASET, databaseConfig)); + super(true, true, Map.of(TEST_DATASET, databaseConfig), null); } @Override From 2a67e1da4b57fd3476fd3b351c4ca7b95cbe82d0 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 14 Aug 2024 14:18:54 +0200 Subject: [PATCH 18/40] sort dropdown options --- .../conquery/resources/admin/ui/templates/groupHandler.html.ftl | 2 +- .../conquery/resources/admin/ui/templates/roleHandler.html.ftl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/groupHandler.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/groupHandler.html.ftl index 4b89d2367a..dbef9a7327 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/groupHandler.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/groupHandler.html.ftl @@ -6,7 +6,7 @@
diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/roleHandler.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/roleHandler.html.ftl index c6d1e27c42..640fc0e0c6 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/roleHandler.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/roleHandler.html.ftl @@ -6,7 +6,7 @@
From 982afcfa7598200a92ceebd9a36ba8b4382a1866 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:30:11 +0200 Subject: [PATCH 19/40] capsule object mapper creation and configuration in one factory --- .../DistributedStandaloneCommand.java | 3 +- .../conquery/commands/ManagerNode.java | 55 +------- .../bakdata/conquery/commands/ShardNode.java | 46 +----- .../conquery/mode/DelegateManager.java | 7 +- .../mode/InternalObjectMapperCreator.java | 67 --------- .../com/bakdata/conquery/mode/Manager.java | 3 +- .../conquery/mode/ManagerProvider.java | 12 +- .../conquery/mode/NamespaceHandler.java | 18 +-- .../conquery/mode/NamespaceSetupData.java | 2 - .../cluster/ClusterConnectionManager.java | 8 +- .../mode/cluster/ClusterConnectionShard.java | 5 +- .../mode/cluster/ClusterManagerProvider.java | 17 ++- .../mode/cluster/ClusterNamespaceHandler.java | 11 +- .../mode/cluster/InternalMapperFactory.java | 132 ++++++++++++++++++ .../mode/local/LocalManagerProvider.java | 11 +- .../mode/local/LocalNamespaceHandler.java | 12 +- .../models/worker/DatasetRegistry.java | 15 +- .../models/worker/DistributedNamespace.java | 5 +- .../models/worker/LocalNamespace.java | 5 +- .../conquery/models/worker/Namespace.java | 12 +- .../conquery/models/worker/Workers.java | 25 ++-- .../admin/rest/AdminDatasetProcessor.java | 18 ++- .../admin/rest/AdminDatasetResource.java | 34 ++--- .../io/AbstractSerializationTest.java | 32 ++--- .../types/ColumnStoreSerializationTests.java | 9 +- 25 files changed, 247 insertions(+), 317 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java diff --git a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java index 23f867b39c..311c72cf5f 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java @@ -88,10 +88,9 @@ public void startStandalone(Environment environment, Namespace namespace, Conque sc.run(clone, environment); } - ConqueryMDC.setLocation("ManagerNode"); - log.debug("Waiting for ShardNodes to start"); // starts the Jersey Server + ConqueryMDC.setLocation("ManagerNode"); log.debug("Starting REST Server"); ConqueryMDC.setLocation(null); super.run(environment, namespace, config); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index d1c5686040..2a59b5c401 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -11,9 +11,7 @@ import jakarta.validation.Validator; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.PathParamInjector; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.jersey.RESTServer; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; @@ -31,9 +29,7 @@ import com.bakdata.conquery.tasks.PermissionCleanupTask; import com.bakdata.conquery.tasks.QueryCleanupTask; import com.bakdata.conquery.tasks.ReloadMetaStorageTask; -import com.fasterxml.jackson.databind.DeserializationConfig; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationConfig; import com.google.common.base.Throwables; import io.dropwizard.core.setup.Environment; import io.dropwizard.jersey.DropwizardResourceConfig; @@ -84,8 +80,8 @@ public void run(Manager manager) throws InterruptedException { this.manager = manager; - final ObjectMapper objectMapper = environment.getObjectMapper(); - customizeApiObjectMapper(objectMapper); + final ObjectMapper apiObjectMapper = environment.getObjectMapper(); + getInternalMapperFactory().customizeApiObjectMapper(apiObjectMapper, getDatasetRegistry(), getMetaStorage()); // FormScanner needs to be instantiated before plugins are initialized @@ -172,54 +168,9 @@ protected void configure() { jerseyConfig.register(PathParamInjector.class); } - /** - * Customize the mapper from the environment, that is used in the REST-API. - * In contrast to the internal object mapper this uses textual JSON representation - * instead of the binary smile format. It also does not expose internal fields through serialization. - *

- * Internal and external mapper have in common that they might process the same classes/objects and that - * they are configured to understand certain Conquery specific data types. - * - * @param objectMapper to be configured (should be a JSON mapper) - */ - public void customizeApiObjectMapper(ObjectMapper objectMapper) { - - // Set serialization config - SerializationConfig serializationConfig = objectMapper.getSerializationConfig(); - - serializationConfig = serializationConfig.withView(View.Api.class); - - objectMapper.setConfig(serializationConfig); - - // Set deserialization config - DeserializationConfig deserializationConfig = objectMapper.getDeserializationConfig(); - - deserializationConfig = deserializationConfig.withView(View.Api.class); - - objectMapper.setConfig(deserializationConfig); - - final MutableInjectableValues injectableValues = new MutableInjectableValues(); - objectMapper.setInjectableValues(injectableValues); - injectableValues.add(Validator.class, getValidator()); - - getDatasetRegistry().injectInto(objectMapper); - getMetaStorage().injectInto(objectMapper); - getConfig().injectInto(objectMapper); - } - - /** - * Create a new internal object mapper for binary (de-)serialization that is equipped with {@link ManagerNode} related injectables. - * - * @return a preconfigured binary object mapper - * @see ManagerNode#customizeApiObjectMapper(ObjectMapper) - */ - public ObjectMapper createInternalObjectMapper(Class viewClass) { - return getInternalObjectMapperCreator().createInternalObjectMapper(viewClass); - } - private void loadMetaStorage() { log.info("Opening MetaStorage"); - getMetaStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class)); + getMetaStorage().openStores(getInternalMapperFactory().createManagerPersistenceMapper(getDatasetRegistry(), getMetaStorage())); log.info("Loading MetaStorage"); getMetaStorage().loadData(); log.info("MetaStorage loaded {}", getMetaStorage()); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index 3f9b65ba82..aac321e32a 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -5,20 +5,14 @@ import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.TimeUnit; -import jakarta.validation.Validator; -import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.WorkerStorage; import com.bakdata.conquery.mode.cluster.ClusterConnectionShard; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.worker.Worker; import com.bakdata.conquery.models.worker.Workers; import com.bakdata.conquery.util.io.ConqueryMDC; -import com.fasterxml.jackson.databind.DeserializationConfig; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationConfig; import io.dropwizard.core.ConfiguredBundle; import io.dropwizard.core.setup.Environment; import io.dropwizard.lifecycle.setup.LifecycleEnvironment; @@ -55,19 +49,19 @@ public ShardNode(String name) { public void run(ConqueryConfig config, Environment environment) throws Exception { LifecycleEnvironment lifecycle = environment.lifecycle(); + + InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, environment.getValidator()); workers = new Workers( config.getQueries().getExecutionPool(), - () -> createInternalObjectMapper(View.Persistence.Shard.class, config, environment.getValidator()), - () -> createInternalObjectMapper(View.InternalCommunication.class, config, environment.getValidator()), + internalMapperFactory, config.getCluster().getEntityBucketSize(), config.getQueries().getSecondaryIdSubPlanRetention() ); lifecycle.manage(workers); - clusterConnection = - new ClusterConnectionShard(config, environment, workers, () -> createInternalObjectMapper(View.InternalCommunication.class, config, environment.getValidator())); + new ClusterConnectionShard(config, environment, workers, internalMapperFactory); lifecycle.manage(clusterConnection); @@ -102,37 +96,7 @@ public void run(ConqueryConfig config, Environment environment) throws Exception log.info("All Worker loaded: {}", workers.getWorkers().size()); } - /** - * Pendant to {@link ManagerNode#createInternalObjectMapper(Class)}. - *

- * TODO May move to {@link ConqueryCommand} - * - * @return a preconfigured binary object mapper - */ - public static ObjectMapper createInternalObjectMapper(Class viewClass, ConqueryConfig config, Validator validator) { - final ObjectMapper objectMapper = config.configureObjectMapper(Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER)); - - final MutableInjectableValues injectableValues = new MutableInjectableValues(); - objectMapper.setInjectableValues(injectableValues); - injectableValues.add(Validator.class, validator); - - - // Set serialization config - SerializationConfig serializationConfig = objectMapper.getSerializationConfig(); - - serializationConfig = serializationConfig.withView(viewClass); - objectMapper.setConfig(serializationConfig); - - // Set deserialization config - DeserializationConfig deserializationConfig = objectMapper.getDeserializationConfig(); - - deserializationConfig = deserializationConfig.withView(viewClass); - - objectMapper.setConfig(deserializationConfig); - - return objectMapper; - } public boolean isBusy() { return clusterConnection.isBusy() || workers.isBusy(); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java index 53ada11ccc..e24429a056 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java @@ -5,6 +5,7 @@ import java.util.function.Supplier; import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.DatasetRegistry; @@ -29,13 +30,9 @@ public class DelegateManager implements Manager { StorageListener storageListener; Supplier> nodeProvider; List adminTasks; - InternalObjectMapperCreator internalObjectMapperCreator; + InternalMapperFactory internalMapperFactory; JobManager jobManager; - @Override - public void start() throws Exception { - } - @Override public void stop() throws Exception { jobManager.close(); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java deleted file mode 100644 index 8fdcf63076..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java +++ /dev/null @@ -1,67 +0,0 @@ -package com.bakdata.conquery.mode; - -import javax.annotation.Nullable; -import jakarta.validation.Validator; - -import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.MutableInjectableValues; -import com.bakdata.conquery.io.jackson.View; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.Namespace; -import com.fasterxml.jackson.databind.DeserializationConfig; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationConfig; -import lombok.Getter; -import lombok.RequiredArgsConstructor; - -/** - * Creator for internal object mapper in the manager. - */ -@Getter -@RequiredArgsConstructor -public class InternalObjectMapperCreator { - private final ConqueryConfig config; - private final MetaStorage storage; - private final Validator validator; - private DatasetRegistry datasetRegistry = null; - - public void init(DatasetRegistry datasetRegistry) { - this.datasetRegistry = datasetRegistry; - } - - public ObjectMapper createInternalObjectMapper(@Nullable Class viewClass) { - if (datasetRegistry == null || storage == null) { - throw new IllegalStateException("%s must be initialized by calling its init method".formatted(this.getClass().getSimpleName())); - } - - final ObjectMapper objectMapper = getConfig().configureObjectMapper(Jackson.BINARY_MAPPER.copy()); - - final MutableInjectableValues injectableValues = new MutableInjectableValues(); - objectMapper.setInjectableValues(injectableValues); - injectableValues.add(Validator.class, getValidator()); - getDatasetRegistry().injectInto(objectMapper); - getStorage().injectInto(objectMapper); - getConfig().injectInto(objectMapper); - - - if (viewClass != null) { - // Set serialization config - SerializationConfig serializationConfig = objectMapper.getSerializationConfig(); - - serializationConfig = serializationConfig.withView(viewClass); - - objectMapper.setConfig(serializationConfig); - - // Set deserialization config - DeserializationConfig deserializationConfig = objectMapper.getDeserializationConfig(); - - deserializationConfig = deserializationConfig.withView(viewClass); - - objectMapper.setConfig(deserializationConfig); - } - - return objectMapper; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java index f8de4d3035..6d538f99b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java @@ -5,6 +5,7 @@ import java.util.function.Supplier; import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.DatasetRegistry; @@ -25,7 +26,7 @@ public interface Manager extends Managed { StorageListener getStorageListener(); Supplier> getNodeProvider(); List getAdminTasks(); - InternalObjectMapperCreator getInternalObjectMapperCreator(); + InternalMapperFactory getInternalMapperFactory(); JobManager getJobManager(); MetaStorage getMetaStorage(); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java index fe45f4ecbe..e3160e58b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java @@ -1,8 +1,6 @@ package com.bakdata.conquery.mode; -import jakarta.validation.Validator; - -import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; @@ -23,20 +21,16 @@ static JobManager newJobManager(ConqueryConfig config) { return new JobManager(JOB_MANAGER_NAME, config.isFailOnError()); } - static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig config, MetaStorage metaStorage, Validator validator) { - return new InternalObjectMapperCreator(config, metaStorage, validator); - } - static DatasetRegistry createDatasetRegistry( NamespaceHandler namespaceHandler, ConqueryConfig config, - InternalObjectMapperCreator creator + InternalMapperFactory internalMapperFactory ) { final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), config.getIndex().getEmptyLabel()); return new DatasetRegistry<>( config.getCluster().getEntityBucketSize(), config, - creator, + internalMapperFactory, namespaceHandler, indexService ); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java index fef5334580..86be55f702 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java @@ -5,14 +5,14 @@ import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.query.FilterSearch; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.databind.ObjectMapper; @@ -23,20 +23,20 @@ */ public interface NamespaceHandler { - N createNamespace(NamespaceStorage storage, MetaStorage metaStorage, IndexService indexService); + N createNamespace(NamespaceStorage storage, MetaStorage metaStorage, DatasetRegistry datasetRegistry); void removeNamespace(DatasetId id, N namespace); /** * Creates the {@link NamespaceSetupData} that is shared by all {@link Namespace} types. */ - static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalObjectMapperCreator mapperCreator, IndexService indexService) { + static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalMapperFactory internalMapperFactory, DatasetRegistry datasetRegistry) { List injectables = new ArrayList<>(); - injectables.add(indexService); + injectables.add(datasetRegistry); - ObjectMapper persistenceMapper = mapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); - ObjectMapper communicationMapper = mapperCreator.createInternalObjectMapper(View.InternalCommunication.class); - ObjectMapper preprocessMapper = mapperCreator.createInternalObjectMapper(null); + ObjectMapper persistenceMapper = internalMapperFactory.createNamespacePersistenceMapper(datasetRegistry); + ObjectMapper communicationMapper = internalMapperFactory.createManagerCommunicationMapper(datasetRegistry); + ObjectMapper preprocessMapper = internalMapperFactory.createPreprocessMapper(datasetRegistry); injectables.forEach(i -> i.injectInto(persistenceMapper)); injectables.forEach(i -> i.injectInto(communicationMapper)); @@ -50,7 +50,7 @@ static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final C JobManager jobManager = new JobManager(storage.getDataset().getName(), config.isFailOnError()); FilterSearch filterSearch = new FilterSearch(config.getIndex()); - return new NamespaceSetupData(injectables, indexService, communicationMapper, preprocessMapper, jobManager, filterSearch); + return new NamespaceSetupData(injectables, communicationMapper, preprocessMapper, jobManager, filterSearch); } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceSetupData.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceSetupData.java index 779205b73d..021379f93a 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceSetupData.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceSetupData.java @@ -3,7 +3,6 @@ import java.util.List; import com.bakdata.conquery.io.jackson.Injectable; -import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.query.FilterSearch; import com.fasterxml.jackson.databind.ObjectMapper; @@ -15,7 +14,6 @@ @Value public class NamespaceSetupData { List injectables; - IndexService indexService; ObjectMapper communicationMapper; ObjectMapper preprocessMapper; JobManager jobManager; diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java index 1d1ed236b4..3208880ab7 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java @@ -4,7 +4,6 @@ import java.net.InetSocketAddress; import jakarta.validation.Validator; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.mina.BinaryJacksonCoder; import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; import com.bakdata.conquery.io.mina.ChunkReader; @@ -12,7 +11,6 @@ import com.bakdata.conquery.io.mina.MdcFilter; import com.bakdata.conquery.io.mina.MinaAttributes; import com.bakdata.conquery.io.mina.NetworkSession; -import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.Job; import com.bakdata.conquery.models.jobs.JobManager; @@ -43,7 +41,7 @@ public class ClusterConnectionManager extends IoHandlerAdapter { private final JobManager jobManager; private final Validator validator; private final ConqueryConfig config; - private final InternalObjectMapperCreator internalObjectMapperCreator; + private final InternalMapperFactory internalMapperFactory; @Getter private final ClusterState clusterState; @@ -93,8 +91,8 @@ public void start() throws IOException { acceptor = new NioSocketAcceptor(); acceptor.getFilterChain().addFirst("mdc", new MdcFilter("Manager[%s]")); - ObjectMapper om = internalObjectMapperCreator.createInternalObjectMapper(View.InternalCommunication.class); - config.configureObjectMapper(om); + ObjectMapper om = internalMapperFactory.createManagerCommunicationMapper(datasetRegistry); + BinaryJacksonCoder coder = new BinaryJacksonCoder(datasetRegistry, validator, om); acceptor.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om))); acceptor.setHandler(this); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java index fc7c28b8b1..777213b06b 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -4,7 +4,6 @@ import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; import com.bakdata.conquery.io.mina.BinaryJacksonCoder; import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; @@ -51,7 +50,7 @@ public class ClusterConnectionShard implements Managed, IoHandler { private final ConqueryConfig config; private final Environment environment; private final Workers workers; - private final Supplier communicationMapperSupplier; + private final InternalMapperFactory internalMapperFactory; private JobManager jobManager; private ScheduledExecutorService scheduler; @@ -205,7 +204,7 @@ private void disconnectFromCluster() { @NotNull private NioSocketConnector getClusterConnector(IdResolveContext workers) { - ObjectMapper om = communicationMapperSupplier.get(); + ObjectMapper om = internalMapperFactory.createShardCommunicationMapper(); NioSocketConnector connector = new NioSocketConnector(); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java index cafc855713..955c90dff6 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java @@ -5,7 +5,11 @@ import java.util.function.Supplier; import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.mode.*; +import com.bakdata.conquery.mode.DelegateManager; +import com.bakdata.conquery.mode.ImportHandler; +import com.bakdata.conquery.mode.ManagerProvider; +import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.mode.StorageListener; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.ClusterHealthCheck; @@ -21,14 +25,13 @@ public class ClusterManagerProvider implements ManagerProvider { public ClusterManager provideManager(ConqueryConfig config, Environment environment) { final JobManager jobManager = ManagerProvider.newJobManager(config); final MetaStorage storage = new MetaStorage(config.getStorage()); - final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, storage, environment.getValidator()); + final InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, environment.getValidator()); final ClusterState clusterState = new ClusterState(); - final NamespaceHandler namespaceHandler = new ClusterNamespaceHandler(clusterState, config, creator); - final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); - creator.init(datasetRegistry); + final NamespaceHandler namespaceHandler = new ClusterNamespaceHandler(clusterState, config, internalMapperFactory); + final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, internalMapperFactory); final ClusterConnectionManager connectionManager = - new ClusterConnectionManager(datasetRegistry, jobManager, environment.getValidator(), config, creator, clusterState); + new ClusterConnectionManager(datasetRegistry, jobManager, environment.getValidator(), config, internalMapperFactory, clusterState); final ImportHandler importHandler = new ClusterImportHandler(config, datasetRegistry); final StorageListener extension = new ClusterStorageListener(jobManager, datasetRegistry); @@ -37,7 +40,7 @@ public ClusterManager provideManager(ConqueryConfig config, Environment environm final DelegateManager delegate = - new DelegateManager<>(config, environment, datasetRegistry, storage, importHandler, extension, nodeProvider, adminTasks, creator, jobManager); + new DelegateManager<>(config, environment, datasetRegistry, storage, importHandler, extension, nodeProvider, adminTasks, internalMapperFactory, jobManager); environment.healthChecks().register("cluster", new ClusterHealthCheck(clusterState)); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java index 617f166d72..120e229128 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java @@ -2,15 +2,14 @@ import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.mode.NamespaceSetupData; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.messages.network.specific.AddWorker; import com.bakdata.conquery.models.messages.network.specific.RemoveWorker; import com.bakdata.conquery.models.query.DistributedExecutionManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.models.worker.WorkerHandler; @@ -20,23 +19,21 @@ public class ClusterNamespaceHandler implements NamespaceHandler { private final ClusterState clusterState; private final ConqueryConfig config; - private final InternalObjectMapperCreator mapperCreator; + private final InternalMapperFactory internalMapperFactory; @Override - public DistributedNamespace createNamespace(NamespaceStorage storage, final MetaStorage metaStorage, IndexService indexService) { - NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(storage, config, mapperCreator, indexService); + public DistributedNamespace createNamespace(NamespaceStorage storage, final MetaStorage metaStorage, DatasetRegistry datasetRegistry) { + NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(storage, config, internalMapperFactory, datasetRegistry); DistributedExecutionManager executionManager = new DistributedExecutionManager(metaStorage, clusterState); WorkerHandler workerHandler = new WorkerHandler(namespaceData.getCommunicationMapper(), storage); clusterState.getWorkerHandlers().put(storage.getDataset().getId(), workerHandler); DistributedNamespace distributedNamespace = new DistributedNamespace( namespaceData.getPreprocessMapper(), - namespaceData.getCommunicationMapper(), storage, executionManager, namespaceData.getJobManager(), namespaceData.getFilterSearch(), - namespaceData.getIndexService(), new ClusterEntityResolver(), namespaceData.getInjectables(), workerHandler diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java new file mode 100644 index 0000000000..e4ab7af4bd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/InternalMapperFactory.java @@ -0,0 +1,132 @@ +package com.bakdata.conquery.mode.cluster; + +import jakarta.validation.Validator; + +import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; +import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Workers; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; + +public record InternalMapperFactory(ConqueryConfig config, Validator validator) { + + public ObjectMapper createShardCommunicationMapper() { + return createInternalObjectMapper(View.InternalCommunication.class); + } + + public ObjectMapper createWorkerCommunicationMapper(Workers workers) { + final ObjectMapper objectMapper = createInternalObjectMapper(View.InternalCommunication.class); + + workers.injectInto(objectMapper); + + return objectMapper; + } + + public ObjectMapper createWorkerPersistenceMapper(Workers workers) { + final ObjectMapper objectMapper = createInternalObjectMapper(View.Persistence.Shard.class); + + workers.injectInto(objectMapper); + + return objectMapper; + } + + public ObjectMapper createNamespacePersistenceMapper(DatasetRegistry datasetRegistry) { + final ObjectMapper objectMapper = createInternalObjectMapper(View.Persistence.Shard.class); + + datasetRegistry.injectInto(objectMapper); + + return objectMapper; + } + + public ObjectMapper createManagerPersistenceMapper(DatasetRegistry datasetRegistry, MetaStorage metaStorage) { + ObjectMapper objectMapper = createInternalObjectMapper(View.Persistence.Manager.class); + + datasetRegistry.injectInto(objectMapper); + metaStorage.injectInto(objectMapper); + + return objectMapper; + } + + public ObjectMapper createManagerCommunicationMapper(DatasetRegistry datasetRegistry) { + ObjectMapper objectMapper = createInternalObjectMapper(View.InternalCommunication.class); + + datasetRegistry.injectInto(objectMapper); + + return objectMapper; + } + + + + public ObjectMapper createPreprocessMapper(DatasetRegistry datasetRegistry) { + ObjectMapper objectMapper = createInternalObjectMapper(null); + + datasetRegistry.injectInto(objectMapper); + + return objectMapper; + } + + /** + * @return a preconfigured binary object mapper + */ + private ObjectMapper createInternalObjectMapper(Class viewClass) { + final ObjectMapper objectMapper = config.configureObjectMapper(Jackson.copyMapperAndInjectables(Jackson.BINARY_MAPPER)); + + final MutableInjectableValues injectableValues = new MutableInjectableValues(); + objectMapper.setInjectableValues(injectableValues); + + injectableValues.add(Validator.class, validator); + config.injectInto(objectMapper); + + if (viewClass != null) { + setViewClass(objectMapper, viewClass); + } + + return objectMapper; + } + + public static void setViewClass(ObjectMapper objectMapper, Class viewClass) { + // Set serialization config + SerializationConfig serializationConfig = objectMapper.getSerializationConfig(); + + serializationConfig = serializationConfig.withView(viewClass); + + objectMapper.setConfig(serializationConfig); + + // Set deserialization config + DeserializationConfig deserializationConfig = objectMapper.getDeserializationConfig(); + + deserializationConfig = deserializationConfig.withView(viewClass); + + objectMapper.setConfig(deserializationConfig); + } + + + + /** + * Customize the mapper from the environment, that is used in the REST-API. + * In contrast to the internal object mapper this uses textual JSON representation + * instead of the binary smile format. It also does not expose internal fields through serialization. + *

+ * Internal and external mapper have in common that they might process the same classes/objects and that + * they are configured to understand certain Conquery specific data types. + * + * @param objectMapper to be configured (should be a JSON mapper) + */ + public void customizeApiObjectMapper(ObjectMapper objectMapper, DatasetRegistry datasetRegistry, MetaStorage metaStorage) { + + InternalMapperFactory.setViewClass(objectMapper, View.Api.class); + + final MutableInjectableValues injectableValues = new MutableInjectableValues(); + objectMapper.setInjectableValues(injectableValues); + injectableValues.add(Validator.class, validator); + + datasetRegistry.injectInto(objectMapper); + metaStorage.injectInto(objectMapper); + config.injectInto(objectMapper); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index 41d3895cf3..a8d9e2e88d 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -7,9 +7,9 @@ import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.DelegateManager; -import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.mode.ManagerProvider; import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; @@ -34,11 +34,10 @@ public LocalManagerProvider(SqlDialectFactory dialectFactory) { public DelegateManager provideManager(ConqueryConfig config, Environment environment) { final MetaStorage storage = new MetaStorage(config.getStorage()); - final InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, storage, environment.getValidator()); - final NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory); - final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + final InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, environment.getValidator()); + final NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, internalMapperFactory, dialectFactory); + final DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, internalMapperFactory); - creator.init(datasetRegistry); return new DelegateManager<>( config, @@ -49,7 +48,7 @@ public DelegateManager provideManager(ConqueryConfig config, Env new LocalStorageListener(), EMPTY_NODE_PROVIDER, List.of(), - creator, + internalMapperFactory, ManagerProvider.newJobManager(config) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index a0f7f23c2d..5f38d411d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -2,16 +2,16 @@ import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.mode.NamespaceSetupData; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.query.ExecutionManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.sql.DSLContextWrapper; import com.bakdata.conquery.sql.DslContextFactory; @@ -33,13 +33,13 @@ public class LocalNamespaceHandler implements NamespaceHandler { private final ConqueryConfig config; - private final InternalObjectMapperCreator mapperCreator; + private final InternalMapperFactory internalMapperFactory; private final SqlDialectFactory dialectFactory; @Override - public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, IndexService indexService) { + public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, DatasetRegistry datasetRegistry) { - NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator, indexService); + NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, internalMapperFactory, datasetRegistry); IdColumnConfig idColumns = config.getIdColumns(); SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig(); @@ -59,14 +59,12 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto return new LocalNamespace( namespaceData.getPreprocessMapper(), - namespaceData.getCommunicationMapper(), namespaceStorage, executionManager, dslContextWrapper, sqlStorageHandler, namespaceData.getJobManager(), namespaceData.getFilterSearch(), - namespaceData.getIndexService(), sqlEntityResolver, namespaceData.getInjectables() ); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index 64606760ff..2024a67df3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -12,11 +12,10 @@ import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; @@ -44,7 +43,7 @@ public class DatasetRegistry extends IdResolveContext imple @Getter private final ConqueryConfig config; - private final InternalObjectMapperCreator internalObjectMapperCreator; + private final InternalMapperFactory internalMapperFactory; private final NamespaceHandler namespaceHandler; @@ -53,7 +52,7 @@ public class DatasetRegistry extends IdResolveContext imple public N createNamespace(Dataset dataset, MetaStorage metaStorage) throws IOException { // Prepare empty storage NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName()); - final ObjectMapper persistenceMapper = internalObjectMapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); + final ObjectMapper persistenceMapper = internalMapperFactory.createNamespacePersistenceMapper(this); // Each store injects its own IdResolveCtx so each needs its own mapper datasetStorage.openStores(Jackson.copyMapperAndInjectables((persistenceMapper))); @@ -67,7 +66,7 @@ public N createNamespace(Dataset dataset, MetaStorage metaStorage) throws IOExce } public N createNamespace(NamespaceStorage datasetStorage, MetaStorage metaStorage) { - final N namespace = namespaceHandler.createNamespace(datasetStorage, metaStorage, indexService); + final N namespace = namespaceHandler.createNamespace(datasetStorage, metaStorage, this); add(namespace); return namespace; } @@ -129,7 +128,11 @@ public void close() { @Override public MutableInjectableValues inject(MutableInjectableValues values) { // Make this class also available under DatasetRegistry - return super.inject(values).add(DatasetRegistry.class, this); + super.inject(values).add(DatasetRegistry.class, this); + + indexService.inject(values); + + return values; } public void resetIndexService() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java index 76a5d4664f..6314706ca3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java @@ -12,7 +12,6 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.messages.namespaces.specific.CollectColumnValuesJob; import com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage; @@ -39,17 +38,15 @@ public class DistributedNamespace extends Namespace { public DistributedNamespace( ObjectMapper preprocessMapper, - ObjectMapper communicationMapper, NamespaceStorage storage, DistributedExecutionManager executionManager, JobManager jobManager, FilterSearch filterSearch, - IndexService indexService, ClusterEntityResolver clusterEntityResolver, List injectables, WorkerHandler workerHandler ) { - super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, clusterEntityResolver, injectables); + super(preprocessMapper, storage, executionManager, jobManager, filterSearch, clusterEntityResolver, injectables); this.executionManager = executionManager; this.workerHandler = workerHandler; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java index 99fb6340a6..871e9eced7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -11,7 +11,6 @@ import com.bakdata.conquery.mode.local.SqlEntityResolver; import com.bakdata.conquery.mode.local.SqlStorageHandler; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.FilterSearch; @@ -30,18 +29,16 @@ public class LocalNamespace extends Namespace { public LocalNamespace( ObjectMapper preprocessMapper, - ObjectMapper communicationMapper, NamespaceStorage storage, ExecutionManager executionManager, DSLContextWrapper dslContextWrapper, SqlStorageHandler storageHandler, JobManager jobManager, FilterSearch filterSearch, - IndexService indexService, SqlEntityResolver sqlEntityResolver, List injectables ) { - super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, sqlEntityResolver, injectables); + super(preprocessMapper, storage, executionManager, jobManager, filterSearch, sqlEntityResolver, injectables); this.dslContextWrapper = dslContextWrapper; this.storageHandler = storageHandler; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java index dcee848050..264e75f83c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java @@ -16,7 +16,6 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.jobs.SimpleJob; import com.bakdata.conquery.models.jobs.UpdateFilterSearchJob; @@ -36,20 +35,16 @@ public abstract class Namespace extends IdResolveContext { private final ObjectMapper preprocessMapper; - private final ObjectMapper communicationMapper; - @ToString.Include private final NamespaceStorage storage; - private final ExecutionManager executionManager; + private final ExecutionManager executionManager; // TODO: 01.07.2020 FK: This is not used a lot, as NamespacedMessages are highly convoluted and hard to decouple as is. private final JobManager jobManager; private final FilterSearch filterSearch; - private final IndexService indexService; - private final EntityResolver entityResolver; // Jackson's injectables that are available when deserializing requests (see PathParamInjector) or items from the storage @@ -109,10 +104,6 @@ public void updateInternToExternMappings() { .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s.getMapping()::init))); } - public void clearIndexCache() { - indexService.evictCache(); - } - public PreviewConfig getPreviewConfig() { return getStorage().getPreviewConfig(); } @@ -142,7 +133,6 @@ final void updateFilterSearch() { * and registers them in the namespace's {@link FilterSearch#registerValues(Searchable, Collection)}. * After value registration for a column is complete, {@link FilterSearch#shrinkSearch(Searchable)} should be called. * - * @param columns */ abstract void registerColumnValuesInSearch(Set columns); diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java index 0f4981d03b..cd420a0c64 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Workers.java @@ -7,11 +7,11 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Supplier; import jakarta.validation.Validator; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.storage.WorkerStorage; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.StoreFactory; import com.bakdata.conquery.models.config.ThreadPoolDefinition; import com.bakdata.conquery.models.datasets.Dataset; @@ -29,7 +29,7 @@ /** * {@link ShardNode} container of {@link Worker}. - * + *

* Each Shard contains one {@link Worker} per {@link Dataset}. */ @Slf4j @@ -47,22 +47,20 @@ public class Workers extends IdResolveContext implements Managed { private final ThreadPoolExecutor jobsThreadPool; private final ThreadPoolDefinition queryThreadPoolDefinition; - private final Supplier persistenceMapperSupplier; - private final Supplier communicationMapperSupplier; + private final InternalMapperFactory internalMapperFactory; private final int entityBucketSize; private final int secondaryIdSubPlanRetention; - public Workers(ThreadPoolDefinition queryThreadPoolDefinition, Supplier persistenceMapperSupplier, Supplier communicationMapperSupplier, int entityBucketSize, int secondaryIdSubPlanRetention) { + public Workers(ThreadPoolDefinition queryThreadPoolDefinition, InternalMapperFactory internalMapperFactory, int entityBucketSize, int secondaryIdSubPlanRetention) { this.queryThreadPoolDefinition = queryThreadPoolDefinition; // TODO This shouldn't be coupled to the query thread pool definition jobsThreadPool = queryThreadPoolDefinition.createService("Workers"); - this.persistenceMapperSupplier = persistenceMapperSupplier; - this.communicationMapperSupplier = communicationMapperSupplier; + this.internalMapperFactory = internalMapperFactory; this.entityBucketSize = entityBucketSize; this.secondaryIdSubPlanRetention = secondaryIdSubPlanRetention; @@ -71,11 +69,8 @@ public Workers(ThreadPoolDefinition queryThreadPoolDefinition, Supplier deleteInternToExternMapping(InternToExternMapper internTo return dependentConcepts.stream().map(Concept::getId).collect(Collectors.toList()); } - public void clearIndexCache(Namespace namespace) { - namespace.clearIndexCache(); + public void clearIndexCache() { + datasetRegistry.resetIndexService(); } public void addSearchIndex(Namespace namespace, SearchIndex searchIndex) { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java index 7feeecc553..ef0592a939 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetResource.java @@ -9,21 +9,6 @@ import java.util.List; import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; - -import com.bakdata.conquery.io.jersey.ExtraMimeTypes; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.PreviewConfig; -import com.bakdata.conquery.models.datasets.SecondaryIdDescription; -import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.datasets.concepts.StructureNode; -import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; -import com.bakdata.conquery.models.identifiable.ids.specific.TableId; -import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.index.InternToExternMapper; -import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.worker.Namespace; -import com.bakdata.conquery.util.io.FileUtil; import jakarta.annotation.PostConstruct; import jakarta.inject.Inject; import jakarta.validation.Valid; @@ -43,6 +28,21 @@ import jakarta.ws.rs.WebApplicationException; import jakarta.ws.rs.core.MediaType; import jakarta.ws.rs.core.Response.Status; + +import com.bakdata.conquery.io.jersey.ExtraMimeTypes; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.PreviewConfig; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.StructureNode; +import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; +import com.bakdata.conquery.models.identifiable.ids.specific.TableId; +import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.index.InternToExternMapper; +import com.bakdata.conquery.models.index.search.SearchIndex; +import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.util.io.FileUtil; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; @@ -186,7 +186,7 @@ public void addConcept(@QueryParam("force") @DefaultValue("false") boolean force @PUT @Path("concepts") - public void updateConcept(Concept concept) { + public void updateConcept(Concept concept) { processor.updateConcept(namespace.getDataset(), concept); } @@ -256,7 +256,7 @@ public void postprocessNamespace(@PathParam(DATASET) Dataset dataset) { @POST @Path("clear-index-cache") public void clearIndexCache() { - processor.clearIndexCache(namespace); + processor.clearIndexCache(); } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index da50cecfe4..9dc098947c 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -1,22 +1,19 @@ package com.bakdata.conquery.io; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.*; +import static org.mockito.Mockito.mock; import jakarta.validation.Validator; -import com.bakdata.conquery.commands.ManagerNode; -import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.jackson.Jackson; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.mode.InternalObjectMapperCreator; import com.bakdata.conquery.mode.cluster.ClusterNamespaceHandler; import com.bakdata.conquery.mode.cluster.ClusterState; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.Workers; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.jersey.validation.Validators; @@ -38,34 +35,25 @@ public abstract class AbstractSerializationTest { @BeforeEach public void before() { + final InternalMapperFactory internalMapperFactory = new InternalMapperFactory(config, validator); metaStorage = new MetaStorage(new NonPersistentStoreFactory()); - InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, metaStorage, validator); final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings(), "emptyDefaultLabel"); - final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler(new ClusterState(), config, creator); - datasetRegistry = new DatasetRegistry<>(0, config, creator, clusterNamespaceHandler, indexService); - creator.init(datasetRegistry); + final ClusterNamespaceHandler clusterNamespaceHandler = new ClusterNamespaceHandler(new ClusterState(), config, internalMapperFactory); + datasetRegistry = new DatasetRegistry<>(0, config, internalMapperFactory, clusterNamespaceHandler, indexService); // Prepare manager node internal mapper - final ManagerNode managerNode = mock(ManagerNode.class); - when(managerNode.getConfig()).thenReturn(config); - when(managerNode.getValidator()).thenReturn(validator); - doReturn(datasetRegistry).when(managerNode).getDatasetRegistry(); - when(managerNode.getMetaStorage()).thenReturn(metaStorage); - when(managerNode.getInternalObjectMapperCreator()).thenReturn(creator); - - when(managerNode.createInternalObjectMapper(any())).thenCallRealMethod(); - managerInternalMapper = managerNode.createInternalObjectMapper(View.Persistence.Manager.class); + managerInternalMapper = internalMapperFactory.createManagerPersistenceMapper(datasetRegistry, metaStorage); metaStorage.openStores(managerInternalMapper); metaStorage.loadData(); // Prepare shard node internal mapper - shardInternalMapper = ShardNode.createInternalObjectMapper(View.Persistence.Shard.class, config, validator); + final Workers workers = mock(Workers.class); + shardInternalMapper = internalMapperFactory.createWorkerPersistenceMapper(workers); // Prepare api response mapper - doCallRealMethod().when(managerNode).customizeApiObjectMapper(any(ObjectMapper.class)); apiMapper = Jackson.copyMapperAndInjectables(Jackson.MAPPER); - managerNode.customizeApiObjectMapper(apiMapper); + internalMapperFactory.customizeApiObjectMapper(apiMapper, datasetRegistry, metaStorage); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java index cb0d7b484a..06f41ef571 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/events/stores/types/ColumnStoreSerializationTests.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.models.events.stores.types; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; import java.io.IOException; import java.util.Arrays; @@ -8,10 +9,9 @@ import java.util.Set; import java.util.stream.Collectors; -import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.jackson.serializer.SerializationTestUtil; +import com.bakdata.conquery.mode.cluster.InternalMapperFactory; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.events.EmptyStore; @@ -34,6 +34,7 @@ import com.bakdata.conquery.models.events.stores.specific.ScaledDecimalStore; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.identifiable.CentralRegistry; +import com.bakdata.conquery.models.worker.Workers; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.Sets; import io.dropwizard.jersey.validation.Validators; @@ -60,8 +61,8 @@ public static void setupRegistry() { // Prepare shard node internal mapper - - shardInternalMapper = ShardNode.createInternalObjectMapper(View.Persistence.Shard.class, new ConqueryConfig(), Validators.newValidator()); + InternalMapperFactory internalMapperFactory = new InternalMapperFactory(new ConqueryConfig(), Validators.newValidator()); + shardInternalMapper = internalMapperFactory.createWorkerPersistenceMapper(mock(Workers.class)); } @Test From 87534edf0e91147de57d22d60227091d82b638cf Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:40:44 +0200 Subject: [PATCH 20/40] improve log message --- .../bakdata/conquery/mode/cluster/ClusterConnectionShard.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java index fc7c28b8b1..d93b07202e 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionShard.java @@ -61,7 +61,7 @@ public class ClusterConnectionShard implements Managed, IoHandler { @Override public void sessionCreated(IoSession session) { - log.debug("Session created."); + log.debug("Session created: {}", session); } From 7666907104497ed34022a1dfef7ec6c4a0d2c147 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:58:40 +0200 Subject: [PATCH 21/40] use list instead of class map to retrieve query visitors --- .../conquery/apiv1/QueryProcessor.java | 29 +++++++------------ .../bakdata/conquery/apiv1/forms/Form.java | 4 +-- .../apiv1/query/QueryDescription.java | 16 ++++------ .../query/preview/EntityPreviewForm.java | 5 ++-- .../com/bakdata/conquery/util/QueryUtils.java | 9 +++--- 5 files changed, 25 insertions(+), 38 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index ac9cdabcbc..3d33798f86 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -89,8 +89,6 @@ import com.bakdata.conquery.util.QueryUtils; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.bakdata.conquery.util.io.IdColumnUtil; -import com.google.common.collect.ClassToInstanceMap; -import com.google.common.collect.MutableClassToInstanceMap; import lombok.AllArgsConstructor; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -385,21 +383,23 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Subje // This maps works as long as we have query visitors that are not configured in anyway. // So adding a visitor twice would replace the previous one but both would have yielded the same result. // For the future a better data structure might be desired that also regards similar QueryVisitors of different configuration - final ClassToInstanceMap visitors = MutableClassToInstanceMap.create(); + final List visitors = new ArrayList<>(); query.addVisitors(visitors); // Initialize checks that need to traverse the query tree - visitors.putInstance(QueryUtils.OnlyReusingChecker.class, new QueryUtils.OnlyReusingChecker()); - visitors.putInstance(NamespacedIdentifiableCollector.class, new NamespacedIdentifiableCollector()); + QueryUtils.OnlyReusingChecker onlyReusingChecker = new QueryUtils.OnlyReusingChecker(); + visitors.add(onlyReusingChecker); + NamespacedIdentifiableCollector namespacedIdentifiableCollector = new NamespacedIdentifiableCollector(); + visitors.add(namespacedIdentifiableCollector); final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(subject, storage).map(Group::getName).orElse("none"); - - visitors.putInstance(ExecutionMetrics.QueryMetricsReporter.class, new ExecutionMetrics.QueryMetricsReporter(primaryGroupName)); + ExecutionMetrics.QueryMetricsReporter queryMetricsReporter = new ExecutionMetrics.QueryMetricsReporter(primaryGroupName); + visitors.add(queryMetricsReporter); // Chain all Consumers Consumer consumerChain = QueryUtils.getNoOpEntryPoint(); - for (QueryVisitor visitor : visitors.values()) { + for (QueryVisitor visitor : visitors) { consumerChain = consumerChain.andThen(visitor); } @@ -410,11 +410,8 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Subje query.authorize(subject, dataset, visitors, storage); // After all authorization checks we can now use the actual subject to invoke the query and do not to bubble down the Userish in methods - NamespacedIdentifiableCollector namespaceIdCollector = visitors.getInstance(NamespacedIdentifiableCollector.class); - if (namespaceIdCollector == null) { - throw new IllegalStateException("NamespacedIdentifiableCollector was not registered properly"); - } - ExecutionMetrics.reportNamespacedIds(namespaceIdCollector.getIdentifiables(), primaryGroupName); + + ExecutionMetrics.reportNamespacedIds(namespacedIdentifiableCollector.getIdentifiables(), primaryGroupName); ExecutionMetrics.reportQueryClassUsage(query.getClass(), primaryGroupName); @@ -424,11 +421,7 @@ public ManagedExecution postQuery(Dataset dataset, QueryDescription query, Subje // If this is only a re-executing query, try to execute the underlying query instead. { - QueryUtils.OnlyReusingChecker reusedChecker = visitors.getInstance(QueryUtils.OnlyReusingChecker.class); - if (reusedChecker == null) { - throw new IllegalStateException("OnlyReusingChecker was not registered properly"); - } - final Optional executionId = reusedChecker.getOnlyReused(); + final Optional executionId = onlyReusingChecker.getOnlyReused(); final Optional execution = diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java index c9844ce5c4..bd12ada5f7 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.apiv1.forms; +import java.util.List; import javax.annotation.Nullable; import com.bakdata.conquery.apiv1.query.QueryDescription; @@ -15,7 +16,6 @@ import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ClassToInstanceMap; import lombok.EqualsAndHashCode; import lombok.NonNull; @@ -41,7 +41,7 @@ public String getFormType() { @Override - public void authorize(Subject subject, Dataset submittedDataset, @NonNull ClassToInstanceMap visitors, MetaStorage storage) { + public void authorize(Subject subject, Dataset submittedDataset, @NonNull List visitors, MetaStorage storage) { QueryDescription.super.authorize(subject, submittedDataset, visitors, storage); // Check if subject is allowed to create this form final FormType formType = FormScanner.resolveFormType(getFormType()); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java index 091d78a5e3..73a0282b40 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/QueryDescription.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.apiv1.query; +import java.util.List; import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -25,9 +26,7 @@ import com.bakdata.conquery.util.QueryUtils.ExternalIdChecker; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import com.google.common.collect.ClassToInstanceMap; import lombok.NonNull; -import org.jetbrains.annotations.NotNull; @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, property = "type") @CPSBase @@ -62,25 +61,22 @@ public interface QueryDescription extends Visitable { * All visitors are concatenated so only a single traverse needs to be done. * @param visitors The structure to which new visitors need to be added. */ - default void addVisitors(@NonNull ClassToInstanceMap visitors) { + default void addVisitors(@NonNull List visitors) { // Register visitors for permission checks - visitors.putInstance(NamespacedIdentifiableCollector.class, new NamespacedIdentifiableCollector()); - visitors.putInstance(QueryUtils.ExternalIdChecker.class, new QueryUtils.ExternalIdChecker()); + visitors.add(new QueryUtils.ExternalIdChecker()); } /** * Check implementation specific permissions. Is called after all visitors have been registered and executed. */ - default void authorize(Subject subject, Dataset submittedDataset, @NonNull ClassToInstanceMap visitors, MetaStorage storage) { + default void authorize(Subject subject, Dataset submittedDataset, List visitors, MetaStorage storage) { authorizeQuery(this, subject, submittedDataset, visitors, storage); } - public static void authorizeQuery(QueryDescription queryDescription, Subject subject, Dataset submittedDataset, @NotNull ClassToInstanceMap visitors, MetaStorage storage) { + static void authorizeQuery(QueryDescription queryDescription, Subject subject, Dataset submittedDataset, List visitors, MetaStorage storage) { NamespacedIdentifiableCollector nsIdCollector = QueryUtils.getVisitor(visitors, NamespacedIdentifiableCollector.class); ExternalIdChecker externalIdChecker = QueryUtils.getVisitor(visitors, ExternalIdChecker.class); - if (nsIdCollector == null) { - throw new IllegalStateException(); - } + // Generate DatasetPermissions final Set datasets = nsIdCollector.getIdentifiables().stream() .map(NamespacedIdentifiable::getDataset) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java index 3ac2552767..fae63122e6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java @@ -8,6 +8,7 @@ import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; +import jakarta.validation.Valid; import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.apiv1.forms.InternalForm; @@ -39,8 +40,6 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.databind.JsonNode; -import com.google.common.collect.ClassToInstanceMap; -import jakarta.validation.Valid; import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -161,7 +160,7 @@ public Map createSubQueries() { } @Override - public void authorize(Subject subject, Dataset submittedDataset, @NonNull ClassToInstanceMap visitors, MetaStorage storage) { + public void authorize(Subject subject, Dataset submittedDataset, @NonNull List visitors, MetaStorage storage) { QueryDescription.authorizeQuery(this, subject, submittedDataset, visitors, storage); } diff --git a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java index 5d0e14bab2..cc78dbaa55 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java @@ -7,7 +7,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; @@ -43,7 +42,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.google.common.base.Strings; -import com.google.common.collect.ClassToInstanceMap; +import com.google.common.collect.MoreCollectors; import lombok.Getter; import lombok.NonNull; import lombok.experimental.UtilityClass; @@ -60,12 +59,12 @@ public class QueryUtils { public static Consumer getNoOpEntryPoint() { return (whatever) -> {}; } - + /** * Gets the specified visitor from the map. If none was found an exception is raised. */ - public static T getVisitor(ClassToInstanceMap visitors, Class clazz){ - return Objects.requireNonNull(visitors.getInstance(clazz),String.format("Among the visitor that traversed the query no %s could be found", clazz)); + public static T getVisitor(List visitors, Class clazz){ + return (T) visitors.stream().filter(clazz::isInstance).collect(MoreCollectors.onlyElement()); } public static String createDefaultMultiLabel(List elements, String delimiter, Locale locale) { From e0ef22f94c19e8b16f0cd187fc67be46374fc8e7 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 16 Aug 2024 12:03:09 +0200 Subject: [PATCH 22/40] unlocks external executions after completion --- .../forms/managed/ExternalExecution.java | 20 ++++++++++++------- .../query/DistributedExecutionManager.java | 4 ++-- .../models/query/ExecutionManager.java | 9 +++++++-- .../models/query/results/FormShardResult.java | 4 ++-- .../sql/conquery/SqlExecutionManager.java | 4 ++-- 5 files changed, 26 insertions(+), 15 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java index 85b2e00d0e..0e184fde03 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java @@ -166,16 +166,22 @@ public void cancel() { @Override public void finish(ExecutionState executionState, ExecutionManager executionManager) { - if (getState().equals(executionState)) { - return; - } - User serviceUser = executionManager.getExternalResult(this.getId()).getServiceUser(); + try { + if (getState().equals(executionState)) { + return; + } + User serviceUser = executionManager.getExternalResult(this.getId()).getServiceUser(); - super.finish(executionState, executionManager); + super.finish(executionState, executionManager); - synchronized (this) { - AuthUtil.cleanUpUserAndBelongings(serviceUser, getMetaStorage()); + synchronized (this) { + AuthUtil.cleanUpUserAndBelongings(serviceUser, getMetaStorage()); + } + } + finally { + executionManager.clearLockExternalExecution(this.getId()); } + } @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java index e967778005..4158f2f814 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java @@ -90,7 +90,7 @@ public if (result.getError().isPresent()) { execution.fail(result.getError().get(), this); - clearLock(execution.getId()); + clearLockInternalExecution(execution.getId()); } else { @@ -103,7 +103,7 @@ public // If all known workers have returned a result, the query is DONE. if (finishedWorkers.equals(getWorkerHandler(execution).getAllWorkerIds())) { execution.finish(ExecutionState.DONE, this); - clearLock(execution.getId()); + clearLockInternalExecution(execution.getId()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index 03be13e3c8..258a8a3170 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -201,11 +201,16 @@ public Stream streamQueryResults(ManagedExecution execution) { } - public void clearLock(ManagedExecutionId id) { + public void clearLockInternalExecution(ManagedExecutionId id) { R result = Objects.requireNonNull(executionResults.getIfPresent(id), "Cannot clear lock on absent execution result"); result.getExecutingLock().countDown(); - //TODO external execution + } + + public void clearLockExternalExecution(ManagedExecutionId id) { + ExternalResult result = Objects.requireNonNull(externalExecutionResults.getIfPresent(id), "Cannot clear lock on absent execution result"); + + result.getExecutingLock().countDown(); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java index c72692fb2a..c7853a2a08 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java @@ -50,7 +50,7 @@ public void addResult(DistributedExecutionManager executionManager) { ); // Signal to waiting threads that the form finished - executionManager.clearLock(formId); + executionManager.clearLockInternalExecution(formId); } if (managedInternalForm.allSubQueriesDone()) { @@ -58,7 +58,7 @@ public void addResult(DistributedExecutionManager executionManager) { managedInternalForm.finish(ExecutionState.DONE, executionManager); // Signal to waiting threads that the form finished - executionManager.clearLock(formId); + executionManager.clearLockInternalExecution(formId); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index 29cbef83c8..75f850f7ec 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -54,7 +54,7 @@ protected > void doExecute(E e }).toArray(CompletableFuture[]::new)) .thenRun(() -> { managedForm.finish(ExecutionState.DONE, this); - clearLock(managedForm.getId()); + clearLockInternalExecution(managedForm.getId()); }); return; } @@ -100,7 +100,7 @@ private CompletableFuture executeAsync(ManagedQuery managedQuery, SqlExecu runningExecutions.remove(id); // Unlock waiting requests - clearLock(id); + clearLockInternalExecution(id); }) .exceptionally(e -> { managedQuery.fail(ConqueryError.asConqueryError(e), this); From 1cbe5704b73b359030948a4cb92e55a0182ec47d Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Mon, 19 Aug 2024 10:32:05 +0200 Subject: [PATCH 23/40] Fix negation conversion if negation is not the first node (#3527) --- .../cqelement/CQNegationConverter.java | 6 -- .../tests/sql/not/with_sibling/content.csv | 9 ++ .../tests/sql/not/with_sibling/expected.csv | 7 ++ .../tests/sql/not/with_sibling/not.spec.json | 101 ++++++++++++++++++ 4 files changed, 117 insertions(+), 6 deletions(-) create mode 100644 backend/src/test/resources/tests/sql/not/with_sibling/content.csv create mode 100644 backend/src/test/resources/tests/sql/not/with_sibling/expected.csv create mode 100644 backend/src/test/resources/tests/sql/not/with_sibling/not.spec.json diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java index e184f5894b..eb0cc34406 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java @@ -4,7 +4,6 @@ import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.google.common.base.Preconditions; public class CQNegationConverter implements NodeConverter { @@ -21,12 +20,7 @@ public ConversionContext convert(CQNegation negationNode, ConversionContext cont .convert(negationNode.getChild(), context.withNegation(true)) .withNegation(false); - Preconditions.checkArgument( - converted.getQuerySteps().size() == 1, - "As we convert only 1 child CQElement, their should be only a single query step." - ); QueryStep queryStep = converted.getLastConvertedStep(); - if (negationNode.getDateAction() != DateAggregationAction.NEGATE) { QueryStep withBlockedValidityDate = queryStep.toBuilder() .selects(queryStep.getSelects().blockValidityDate()) diff --git a/backend/src/test/resources/tests/sql/not/with_sibling/content.csv b/backend/src/test/resources/tests/sql/not/with_sibling/content.csv new file mode 100644 index 0000000000..8dab40f969 --- /dev/null +++ b/backend/src/test/resources/tests/sql/not/with_sibling/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2013-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11, +6,2012-11-11, +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/not/with_sibling/expected.csv b/backend/src/test/resources/tests/sql/not/with_sibling/expected.csv new file mode 100644 index 0000000000..4c60b15bec --- /dev/null +++ b/backend/src/test/resources/tests/sql/not/with_sibling/expected.csv @@ -0,0 +1,7 @@ +result,dates +2,{} +4,{} +5,{} +6,{} +7,{} +8,{} diff --git a/backend/src/test/resources/tests/sql/not/with_sibling/not.spec.json b/backend/src/test/resources/tests/sql/not/with_sibling/not.spec.json new file mode 100644 index 0000000000..d8d0a06e7c --- /dev/null +++ b/backend/src/test/resources/tests/sql/not/with_sibling/not.spec.json @@ -0,0 +1,101 @@ +{ + "label": "Simple negation query where NEGATION is not the first node to convert", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, + "expectedCsv": "tests/sql/not/with_sibling/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "OR", + "children": [ + { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "m" + ] + } + ] + } + ] + }, + { + "type": "NEGATION", + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + } + ] + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/not/with_sibling/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} From 12315916bd776b17f6305f803f0d6708aa1e9cd9 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 20 Aug 2024 09:38:33 +0200 Subject: [PATCH 24/40] update java parser --- autodoc/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/autodoc/pom.xml b/autodoc/pom.xml index 214cd4d877..5ef335a393 100644 --- a/autodoc/pom.xml +++ b/autodoc/pom.xml @@ -5,7 +5,7 @@ - 3.24.4 + 3.26.1 From be5cd58751331940dc0040c414239b07fdcf2e9a Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 20 Aug 2024 09:39:07 +0200 Subject: [PATCH 25/40] adds docs and registers them to autodoc --- .../java/com/bakdata/conquery/Constants.java | 5 +++++ .../conquery/introspection/Introspection.java | 4 ++-- .../models/config/DatabaseConfig.java | 21 +++++++++++++++++++ .../conquery/models/config/Dialect.java | 5 +++++ .../models/config/SqlConnectorConfig.java | 7 +++++++ 5 files changed, 40 insertions(+), 2 deletions(-) diff --git a/autodoc/src/main/java/com/bakdata/conquery/Constants.java b/autodoc/src/main/java/com/bakdata/conquery/Constants.java index 021cf535a1..405dc6d19c 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/Constants.java +++ b/autodoc/src/main/java/com/bakdata/conquery/Constants.java @@ -4,6 +4,7 @@ import java.nio.charset.Charset; import java.time.ZonedDateTime; import java.util.Currency; +import java.util.List; import java.util.Locale; import java.util.Set; import jakarta.ws.rs.DELETE; @@ -41,12 +42,15 @@ import com.bakdata.conquery.models.config.CSVConfig; import com.bakdata.conquery.models.config.ClusterConfig; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.config.LocaleConfig; import com.bakdata.conquery.models.config.MinaConfig; import com.bakdata.conquery.models.config.PluginConfig; import com.bakdata.conquery.models.config.PreprocessingConfig; import com.bakdata.conquery.models.config.QueryConfig; +import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.config.StandaloneConfig; import com.bakdata.conquery.models.config.XodusConfig; import com.bakdata.conquery.models.config.XodusStoreFactory; @@ -129,6 +133,7 @@ public class Constants { .otherClass(MinaConfig.class) .otherClass(FrontendConfig.CurrencyConfig.class) .otherClass(XodusConfig.class) + .otherClasses(List.of(SqlConnectorConfig.class, DatabaseConfig.class, Dialect.class)) .hide(Charset.class) .hide(Currency.class) .hide(InetAddress.class) diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java index f0e79c06ac..8e304352a6 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java @@ -13,8 +13,8 @@ public interface Introspection { String getDescription(); String getExample(); File getFile(); - - public static Introspection from(File root, ClassInfo cl) { + + static Introspection from(File root, ClassInfo cl) { File f = new File(root, "backend/src/main/java/"+cl.getName().replace('.', '/')+".java"); try { if(cl.isInnerClass()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java index db23de6553..e7ffaad173 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java @@ -6,6 +6,11 @@ import lombok.NoArgsConstructor; import lombok.extern.jackson.Jacksonized; +/** + * Connection properties for a SQL database. + *

+ * Currently supported are HANA and Prostgres databases, see {@link DatabaseConfig#dialect}. + */ @Data @Builder @Jacksonized @@ -15,14 +20,30 @@ public class DatabaseConfig { private static final String DEFAULT_PRIMARY_COLUMN = "pid"; + /** + * SQL vendor specific dialect used to transform queries to SQL + */ private Dialect dialect; + /** + * Username used to connect to the database. + */ private String databaseUsername; + + /** + * Password used to connect to the database. + */ private String databasePassword; + /** + * Connections url in JDBC notation. + */ private String jdbcConnectionUrl; + /** + * Name of the column which is shared among the table and all aggregations are grouped by. + */ @Builder.Default private String primaryColumn = DEFAULT_PRIMARY_COLUMN; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java index 70787c0d94..094d512c1d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -3,6 +3,11 @@ import lombok.Getter; import org.jooq.SQLDialect; +/** + * The dialect sets SQL vendor specific query transformation rules. + *

+ * There is no fallback dialect, so the dialect must fit the targeted database. + */ @Getter public enum Dialect { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index fd11752e78..ef66c3ca8a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -13,6 +13,13 @@ import lombok.NoArgsConstructor; import lombok.extern.jackson.Jacksonized; +/** + * Configuration for SQL databases to send dataset queries to. + *

+ * Multiple databases can be configured for different datasets. + * + * @implNote At the moment, dataset names are statically mapped to a database by the {@link SqlConnectorConfig#databaseConfigs}-map. + */ @Data @Builder @Jacksonized From 8aeff62bb170ca6547c57e249d0d83e2e1de9696 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 20 Aug 2024 12:54:12 +0200 Subject: [PATCH 26/40] support enums and records --- .../java/com/bakdata/conquery/AutoDoc.java | 11 ++- .../conquery/handler/GroupHandler.java | 23 ++--- .../AbstractJavadocIntrospection.java | 40 ++++---- .../AbstractNodeWithMemberIntrospection.java | 93 +++++++++++++++++++ .../introspection/ClassIntrospection.java | 57 +----------- .../introspection/EnumIntrospection.java | 26 ++++++ .../conquery/introspection/Introspection.java | 28 ++++-- .../introspection/RecordIntrospection.java | 13 +++ .../conquery/models/config/Dialect.java | 6 ++ 9 files changed, 201 insertions(+), 96 deletions(-) create mode 100644 autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java create mode 100644 autodoc/src/main/java/com/bakdata/conquery/introspection/EnumIntrospection.java create mode 100644 autodoc/src/main/java/com/bakdata/conquery/introspection/RecordIntrospection.java diff --git a/autodoc/src/main/java/com/bakdata/conquery/AutoDoc.java b/autodoc/src/main/java/com/bakdata/conquery/AutoDoc.java index dcc0721776..b934d59102 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/AutoDoc.java +++ b/autodoc/src/main/java/com/bakdata/conquery/AutoDoc.java @@ -8,6 +8,8 @@ import com.bakdata.conquery.handler.GroupHandler; import com.bakdata.conquery.handler.SimpleWriter; import com.bakdata.conquery.model.Group; +import com.github.javaparser.ParserConfiguration; +import com.github.javaparser.StaticJavaParser; import com.github.powerlibraries.io.Out; import io.github.classgraph.ClassGraph; import io.github.classgraph.ScanResult; @@ -17,10 +19,17 @@ public class AutoDoc { public static void main(String[] args) throws IOException { + configureJavaParser(); + new AutoDoc().start(new File(args.length == 0 ? "./docs/" : args[0])); } - private ScanResult scan; + private static void configureJavaParser() { + ParserConfiguration parserConfiguration = StaticJavaParser.getParserConfiguration(); + parserConfiguration.setLanguageLevel(ParserConfiguration.LanguageLevel.JAVA_17); + } + + private final ScanResult scan; public AutoDoc() { scan = new ClassGraph() diff --git a/autodoc/src/main/java/com/bakdata/conquery/handler/GroupHandler.java b/autodoc/src/main/java/com/bakdata/conquery/handler/GroupHandler.java index 9e7db57843..ea99912a73 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/handler/GroupHandler.java +++ b/autodoc/src/main/java/com/bakdata/conquery/handler/GroupHandler.java @@ -15,7 +15,6 @@ import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; - import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.introspection.Introspection; @@ -98,7 +97,7 @@ public void handle() throws IOException { } if (!endpoints.isEmpty()) { out.heading("REST endpoints"); - for (Pair endpoint : endpoints.stream().sorted(Comparator.comparing(Pair::getLeft)).collect(Collectors.toList())) { + for (Pair endpoint : endpoints.stream().sorted(Comparator.comparing(Pair::getLeft)).toList()) { handleEndpoint(endpoint.getLeft(), endpoint.getRight()); } } @@ -108,13 +107,13 @@ public void handle() throws IOException { } out.subHeading("Other Types"); - for (Class t : group.getOtherClasses().stream().sorted(Comparator.comparing(Class::getSimpleName)).collect(Collectors.toList())) { + for (Class t : group.getOtherClasses().stream().sorted(Comparator.comparing(Class::getSimpleName)).toList()) { handleClass(typeTitle(t), scan.getClassInfo(t.getName())); } if (!group.getMarkerInterfaces().isEmpty()) { out.subHeading("Marker Interfaces"); - for (Class t : group.getMarkerInterfaces().stream().sorted(Comparator.comparing(Class::getSimpleName)).collect(Collectors.toList())) { + for (Class t : group.getMarkerInterfaces().stream().sorted(Comparator.comparing(Class::getSimpleName)).toList()) { handleMarkerInterface(markerTitle(t), scan.getClassInfo(t.getName())); } } @@ -135,7 +134,7 @@ private void handleEndpoint(String url, MethodInfo method) throws IOException { } } - private void collectEndpoints(Class resource) throws IOException { + private void collectEndpoints(Class resource) { final ClassInfo info = scan.getClassInfo(resource.getName()); for (MethodInfo method : info.getMethodInfo()) { @@ -172,7 +171,7 @@ public void handleBase(Base base) throws IOException { + code(typeProperty) + " to one of the following values:"); - for (Pair pair : content.get(base).stream().sorted(Comparator.comparing(p -> p.getLeft().id())).collect(Collectors.toList())) { + for (Pair pair : content.get(base).stream().sorted(Comparator.comparing(p -> p.getLeft().id())).toList()) { handleClass(pair.getLeft(), pair.getRight()); } @@ -198,7 +197,7 @@ private void handleClass(String name, ClassInfo c) throws IOException { out.line("Supported Fields:"); out.tableHeader("", "Field", "Type", "Default", "Example", "Description"); - for (FieldInfo field : c.getFieldInfo().stream().sorted().collect(Collectors.toList())) { + for (FieldInfo field : c.getFieldInfo().stream().sorted().toList()) { handleField(c, field); } } @@ -226,12 +225,7 @@ private Closeable details(String name, ClassInfo c, Introspection source) throws ); } - return new Closeable() { - @Override - public void close() throws IOException { - out.line("

"); - } - }; + return () -> out.line("

"); } private void handleMarkerInterface(String name, ClassInfo c) throws IOException { @@ -487,6 +481,7 @@ private boolean isJSONSettableField(FieldInfo field) { } } - return false; + // is record + return field.getClassInfo().isRecord(); } } diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractJavadocIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractJavadocIntrospection.java index 133eba024a..8304cc0fbb 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractJavadocIntrospection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractJavadocIntrospection.java @@ -10,53 +10,55 @@ import io.github.classgraph.MethodInfo; import lombok.Getter; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +@Slf4j @RequiredArgsConstructor -public class AbstractJavadocIntrospection&NodeWithRange> implements Introspection { +public class AbstractJavadocIntrospection & NodeWithRange> implements Introspection { @Getter protected final File file; @Getter protected final VALUE value; - + @Getter(lazy = true) private final String description = extractDescription(); @Getter(lazy = true) private final String example = extractExample(); - private String extractDescription() { - if(!value.getJavadoc().isPresent()) { + protected String extractDescription() { + if (value.getJavadoc().isEmpty()) { return ""; } var javadoc = value.getJavadoc().get(); return javadoc.getDescription().toText().replaceAll("\\s+", " "); } - + private String extractExample() { - if(!value.getJavadoc().isPresent()) { + if (value.getJavadoc().isEmpty()) { return ""; } var javadoc = value.getJavadoc().get(); return javadoc.getBlockTags() - .stream() - .filter(b->b.getTagName().equals("jsonExample")) - .findAny() - .map(JavadocBlockTag::getContent) - .map(JavadocDescription::toText) - .orElse(""); + .stream() + .filter(b -> b.getTagName().equals("jsonExample")) + .findAny() + .map(JavadocBlockTag::getContent) + .map(JavadocDescription::toText) + .orElse(""); } - + @Override public String getLine() { - if(value.getJavadocComment().isPresent()) { + if (value.getJavadocComment().isPresent()) { return "L" - + value.getJavadocComment().get().getBegin().get().line - + "-L" - + value.getJavadocComment().get().getEnd().get().line; + + value.getJavadocComment().get().getBegin().get().line + + "-L" + + value.getJavadocComment().get().getEnd().get().line; } - return "L"+value.getBegin().get().line; + return "L" + value.getBegin().get().line; } - + @Override public Introspection findMethod(MethodInfo method) { return new SimpleIntrospection(file); diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java new file mode 100644 index 0000000000..b5df9dbb21 --- /dev/null +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java @@ -0,0 +1,93 @@ +package com.bakdata.conquery.introspection; + +import java.io.File; +import java.util.Arrays; + +import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.FieldDeclaration; +import com.github.javaparser.ast.body.RecordDeclaration; +import com.github.javaparser.ast.nodeTypes.NodeWithJavadoc; +import com.github.javaparser.ast.nodeTypes.NodeWithMembers; +import com.github.javaparser.ast.nodeTypes.NodeWithRange; +import com.github.javaparser.ast.nodeTypes.NodeWithSimpleName; +import com.google.common.collect.MoreCollectors; +import io.github.classgraph.ArrayTypeSignature; +import io.github.classgraph.BaseTypeSignature; +import io.github.classgraph.ClassRefTypeSignature; +import io.github.classgraph.FieldInfo; +import io.github.classgraph.MethodInfo; +import io.github.classgraph.MethodParameterInfo; +import io.github.classgraph.TypeSignature; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class AbstractNodeWithMemberIntrospection & NodeWithRange & NodeWithMembers> extends AbstractJavadocIntrospection { + + public AbstractNodeWithMemberIntrospection(File file, VALUE nodeWithJavadoc) { + super(file, nodeWithJavadoc); + } + + @Override + public Introspection findMethod(MethodInfo method) { + var types = Arrays.stream(method.getParameterInfo()) + .map(MethodParameterInfo::getTypeSignatureOrTypeDescriptor) + .map(this::toClass) + .toArray(Class[]::new); + + return new AbstractJavadocIntrospection<>( + file, + value + .getMethodsByParameterTypes(types) + .stream() + .filter(md -> md.getNameAsString().equals(method.getName())) + .collect(MoreCollectors.onlyElement()) + ); + + } + + @Override + public Introspection findField(FieldInfo field) { + + var f = value.getFieldByName(field.getName()); + if (f.isPresent()) { + FieldDeclaration fieldDeclaration = f.get(); + return new AbstractJavadocIntrospection<>(file, fieldDeclaration); + } + log.warn("Could not find field '{}'", field.getName()); + return new SimpleIntrospection(file); + } + + public Introspection findInnerType(String simpleName) { + for (var decl : value.getMembers()) { + if (decl instanceof NodeWithSimpleName node) { + if (!node.getNameAsString().equals(simpleName)) { + continue; + } + } + else { + continue; + } + if (decl instanceof ClassOrInterfaceDeclaration cType) { + return new ClassIntrospection(file, cType); + } + if (decl instanceof RecordDeclaration recordDeclaration) { + return new RecordIntrospection(file, recordDeclaration); + } + } + throw new IllegalStateException(value.getNameAsString() + " has no inner type " + simpleName); + } + + private Class toClass(TypeSignature sig) { + if (sig instanceof BaseTypeSignature) { + return ((BaseTypeSignature) sig).getType(); + } + else if (sig instanceof ArrayTypeSignature) { + return ((ArrayTypeSignature) sig).loadClass(); + } + else if (sig instanceof ClassRefTypeSignature) { + return ((ClassRefTypeSignature) sig).loadClass(); + } + throw new IllegalStateException("Can't find class for signature " + sig); + } + +} diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/ClassIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/ClassIntrospection.java index 9217810b56..7e25dbe241 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/ClassIntrospection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/ClassIntrospection.java @@ -1,71 +1,16 @@ package com.bakdata.conquery.introspection; import java.io.File; -import java.util.Arrays; import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; -import com.google.common.collect.MoreCollectors; -import io.github.classgraph.ArrayTypeSignature; -import io.github.classgraph.BaseTypeSignature; -import io.github.classgraph.ClassRefTypeSignature; -import io.github.classgraph.FieldInfo; -import io.github.classgraph.MethodInfo; -import io.github.classgraph.TypeSignature; import lombok.extern.slf4j.Slf4j; @Slf4j -public class ClassIntrospection extends AbstractJavadocIntrospection { +public class ClassIntrospection extends AbstractNodeWithMemberIntrospection { public ClassIntrospection(File file, ClassOrInterfaceDeclaration value) { super(file, value); } - public ClassIntrospection findInnerType(String simpleName) { - for(var decl : value.getMembers()) { - if(decl.isClassOrInterfaceDeclaration()) { - var cType = decl.asClassOrInterfaceDeclaration(); - if(cType.getNameAsString().equals(simpleName)) { - return new ClassIntrospection(file, cType); - } - } - } - throw new IllegalStateException(value.getNameAsString()+" has no inner type "+simpleName); - } - public Introspection findMethod(MethodInfo method) { - var types = Arrays.stream(method.getParameterInfo()) - .map(p->p.getTypeSignatureOrTypeDescriptor()) - .map(this::toClass) - .toArray(Class[]::new); - - return new AbstractJavadocIntrospection<>(file, - value - .getMethodsByParameterTypes(types) - .stream() - .filter(md->md.getNameAsString().equals(method.getName())) - .collect(MoreCollectors.onlyElement()) - ); - } - - private Class toClass(TypeSignature sig) { - if(sig instanceof BaseTypeSignature) { - return ((BaseTypeSignature) sig).getType(); - } - else if(sig instanceof ArrayTypeSignature) { - return ((ArrayTypeSignature) sig).loadClass(); - } - else if(sig instanceof ClassRefTypeSignature) { - return ((ClassRefTypeSignature) sig).loadClass(); - } - throw new IllegalStateException("Can't find class for signature "+sig); - } - - public Introspection findField(FieldInfo field) { - var f = value.getFieldByName(field.getName()); - if(f.isPresent()) { - return new AbstractJavadocIntrospection<>(file, f.get()); - } - log.warn("Could not find field "+field.getName()); - return super.findField(field); - } } diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/EnumIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/EnumIntrospection.java new file mode 100644 index 0000000000..d6f271b97b --- /dev/null +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/EnumIntrospection.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.introspection; + +import java.io.File; +import java.util.stream.Collectors; + +import com.github.javaparser.ast.body.EnumDeclaration; +import com.github.javaparser.javadoc.Javadoc; + +public class EnumIntrospection extends AbstractNodeWithMemberIntrospection { + + + public EnumIntrospection(File file, EnumDeclaration enumDeclaration) { + super(file, enumDeclaration); + } + + @Override + protected String extractDescription() { + String description = super.extractDescription(); + + String values = value.getEntries().stream() + .map(e -> "`%s`: %s".formatted(e.getNameAsString(), e.getJavadoc().map(Javadoc::toText).orElse("No description available"))) + .collect(Collectors.joining("\n- ", "\nValues:\n- ", "")); + description += values; + return description; + } +} diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java index 8e304352a6..3ef4c317de 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java @@ -4,6 +4,8 @@ import com.github.javaparser.StaticJavaParser; import com.github.javaparser.ast.CompilationUnit; +import com.github.javaparser.ast.body.EnumDeclaration; +import com.github.javaparser.ast.body.TypeDeclaration; import io.github.classgraph.ClassInfo; import io.github.classgraph.FieldInfo; import io.github.classgraph.MethodInfo; @@ -18,17 +20,31 @@ static Introspection from(File root, ClassInfo cl) { File f = new File(root, "backend/src/main/java/"+cl.getName().replace('.', '/')+".java"); try { if(cl.isInnerClass()) { - ClassIntrospection parent = (ClassIntrospection)from(root, cl.getOuterClasses().get(cl.getOuterClasses().size()-1)); - return parent.findInnerType(cl.getSimpleName()); + Introspection from = from(root, cl.getOuterClasses().get(cl.getOuterClasses().size() - 1)); + if (from instanceof SimpleIntrospection simpleIntrospection) { + return simpleIntrospection; + } + + if (from instanceof AbstractNodeWithMemberIntrospection nodeWithMemberIntrospection) { + return nodeWithMemberIntrospection.findInnerType(cl.getSimpleName()); + } + else { + return new SimpleIntrospection(f); + } } CompilationUnit cu = StaticJavaParser.parse(f); - - var type = cu.getPrimaryType().get().asClassOrInterfaceDeclaration(); - return new ClassIntrospection(f, type); + + TypeDeclaration typeDeclaration = cu.getPrimaryType().get(); + + if (typeDeclaration instanceof EnumDeclaration enumDeclaration) { + return new EnumIntrospection(f, enumDeclaration); + } + return new AbstractNodeWithMemberIntrospection<>(f, typeDeclaration); + // return new ClassIntrospection(f, typeDeclaration); } catch(Exception e) { - LoggerFactory.getLogger(Introspection.class).warn("Could not create compilation unit for "+cl.getName(), e); + LoggerFactory.getLogger(Introspection.class).warn("Could not create compilation unit for {}", cl.getName(), e); return new SimpleIntrospection(f); } } diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/RecordIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/RecordIntrospection.java new file mode 100644 index 0000000000..f023e8472a --- /dev/null +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/RecordIntrospection.java @@ -0,0 +1,13 @@ +package com.bakdata.conquery.introspection; + +import java.io.File; + +import com.github.javaparser.ast.body.RecordDeclaration; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class RecordIntrospection extends AbstractNodeWithMemberIntrospection { + public RecordIntrospection(File file, RecordDeclaration recordDeclaration) { + super(file, recordDeclaration); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java index 094d512c1d..edd431f90e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -11,7 +11,13 @@ @Getter public enum Dialect { + /** + * Dialect for PostgreSQL database + */ POSTGRESQL(SQLDialect.POSTGRES, 63), + /** + * Dialect for SAP HANA database + */ HANA(SQLDialect.DEFAULT, 127); private final SQLDialect jooqDialect; From 9d5022e017f63153b009785f15fb5b965ece1b86 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 21 Aug 2024 16:12:32 +0200 Subject: [PATCH 27/40] Adds timeout check to ClusterHealthCheck --- .../mode/cluster/ClusterManagerProvider.java | 3 ++- .../conquery/models/config/ClusterConfig.java | 1 + .../models/jobs/JobManagerStatus.java | 9 ++++--- .../models/worker/ClusterHealthCheck.java | 27 ++++++++++++++++--- .../models/worker/ShardNodeInformation.java | 1 + 5 files changed, 34 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java index 5d53758afb..9f93b0c108 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java @@ -42,7 +42,8 @@ public ClusterManager provideManager(ConqueryConfig config, Environment environm delegate = new DelegateManager<>(config, environment, datasetRegistry, storage, importHandler, extension, nodeProvider, adminTasks, internalMapperFactory, jobManager); - environment.healthChecks().register("cluster", new ClusterHealthCheck(clusterState)); + environment.healthChecks() + .register("cluster", new ClusterHealthCheck(clusterState, nodeProvider, config.getCluster().getHeartbeatTimeout().toJavaDuration())); return new ClusterManager(delegate, connectionManager); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java index c71c7fde1c..f2f0ea1910 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java @@ -26,6 +26,7 @@ public class ClusterConfig extends Configuration { private int entityBucketSize = 1000; private Duration idleTimeOut = Duration.minutes(5); + private Duration heartbeatTimeout = Duration.minutes(1); private Duration connectRetryTimeout = Duration.seconds(30); /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java index 6314bb75e4..674505939c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java @@ -3,7 +3,6 @@ import java.time.Duration; import java.time.LocalDateTime; import java.util.List; - import javax.annotation.Nullable; import jakarta.validation.constraints.NotNull; @@ -40,11 +39,15 @@ public int size() { return jobs.size(); } + @JsonIgnore + public Duration getAge() { + return Duration.between(timestamp, LocalDateTime.now()); + } + // Used in AdminUIResource/jobs @JsonIgnore public String getAgeString() { - final Duration duration = Duration.between(timestamp, LocalDateTime.now()); - return DurationFormatUtils.formatDurationWords(duration.toMillis(), true, true); + return DurationFormatUtils.formatDurationWords(getAge().toMillis(), true, true); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java index a52e35667d..1211f76a14 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java @@ -1,9 +1,15 @@ package com.bakdata.conquery.models.worker; +import java.net.SocketAddress; +import java.time.Duration; +import java.time.LocalDateTime; import java.util.Collection; import java.util.List; import java.util.function.Predicate; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import com.bakdata.conquery.io.mina.NetworkSession; import com.bakdata.conquery.mode.cluster.ClusterState; import com.codahale.metrics.health.HealthCheck; import lombok.Data; @@ -13,6 +19,8 @@ public class ClusterHealthCheck extends HealthCheck { public static final String HEALTHY_MESSAGE_FMT = "All %d known shards are connected."; private final ClusterState clusterState; + private final Supplier> nodeProvider; + private final Duration heartbeatTimeout; @Override protected Result check() throws Exception { @@ -24,10 +32,23 @@ protected Result check() throws Exception { .map(ShardNodeInformation::toString) .toList(); - if (disconnectedWorkers.isEmpty()){ - return Result.healthy(HEALTHY_MESSAGE_FMT, knownShards.size()); + if (!disconnectedWorkers.isEmpty()) { + return Result.unhealthy("The shard(s) %s are no longer connected.".formatted(String.join(",", disconnectedWorkers))); } - return Result.unhealthy("The shard(s) %s are no longer connected.".formatted(String.join(",", disconnectedWorkers))); + LocalDateTime now = LocalDateTime.now(); + List timeoutShards = nodeProvider.get().stream() + .filter((status) -> heartbeatTimeout.minus(Duration.between(now, status.getLastStatusTime())) + .isNegative()).toList(); + + if (!timeoutShards.isEmpty()) { + return Result.unhealthy("Shards timed out:%s".formatted(timeoutShards.stream() + .map(ShardNodeInformation::getSession) + .map(NetworkSession::getRemoteAddress) + .map(SocketAddress::toString) + .collect(Collectors.joining(", ")))); + } + + return Result.healthy(HEALTHY_MESSAGE_FMT, knownShards.size()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java index cc6c780ddf..e7e0bde136 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java @@ -38,6 +38,7 @@ public class ShardNodeInformation extends MessageSender.Simple jobManagerStatus = new HashSet<>(); private final AtomicBoolean full = new AtomicBoolean(false); + @Getter private LocalDateTime lastStatusTime = LocalDateTime.now(); public ShardNodeInformation(NetworkSession session, int backpressure) { From 146aed3e64ac740dc9e887c4af430519eebe96b4 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 21 Aug 2024 17:30:21 +0200 Subject: [PATCH 28/40] adds missing changes to universally handy nodes with members --- .../AbstractNodeWithMemberIntrospection.java | 12 ++++++------ .../introspection/ClassIntrospection.java | 16 ---------------- .../conquery/introspection/Introspection.java | 1 - .../introspection/RecordIntrospection.java | 13 ------------- 4 files changed, 6 insertions(+), 36 deletions(-) delete mode 100644 autodoc/src/main/java/com/bakdata/conquery/introspection/ClassIntrospection.java delete mode 100644 autodoc/src/main/java/com/bakdata/conquery/introspection/RecordIntrospection.java diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java index b5df9dbb21..63a6abba4c 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/AbstractNodeWithMemberIntrospection.java @@ -3,9 +3,9 @@ import java.io.File; import java.util.Arrays; -import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; +import com.github.javaparser.ast.body.EnumDeclaration; import com.github.javaparser.ast.body.FieldDeclaration; -import com.github.javaparser.ast.body.RecordDeclaration; +import com.github.javaparser.ast.body.TypeDeclaration; import com.github.javaparser.ast.nodeTypes.NodeWithJavadoc; import com.github.javaparser.ast.nodeTypes.NodeWithMembers; import com.github.javaparser.ast.nodeTypes.NodeWithRange; @@ -67,11 +67,11 @@ public Introspection findInnerType(String simpleName) { else { continue; } - if (decl instanceof ClassOrInterfaceDeclaration cType) { - return new ClassIntrospection(file, cType); + if (decl instanceof EnumDeclaration enumDeclaration) { + return new EnumIntrospection(file, enumDeclaration); } - if (decl instanceof RecordDeclaration recordDeclaration) { - return new RecordIntrospection(file, recordDeclaration); + if (decl instanceof TypeDeclaration cType) { + return new AbstractNodeWithMemberIntrospection<>(file, cType); } } throw new IllegalStateException(value.getNameAsString() + " has no inner type " + simpleName); diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/ClassIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/ClassIntrospection.java deleted file mode 100644 index 7e25dbe241..0000000000 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/ClassIntrospection.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.bakdata.conquery.introspection; - -import java.io.File; - -import com.github.javaparser.ast.body.ClassOrInterfaceDeclaration; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class ClassIntrospection extends AbstractNodeWithMemberIntrospection { - - public ClassIntrospection(File file, ClassOrInterfaceDeclaration value) { - super(file, value); - } - - -} diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java index 3ef4c317de..b5c9191b0a 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java +++ b/autodoc/src/main/java/com/bakdata/conquery/introspection/Introspection.java @@ -42,7 +42,6 @@ static Introspection from(File root, ClassInfo cl) { return new EnumIntrospection(f, enumDeclaration); } return new AbstractNodeWithMemberIntrospection<>(f, typeDeclaration); - // return new ClassIntrospection(f, typeDeclaration); } catch(Exception e) { LoggerFactory.getLogger(Introspection.class).warn("Could not create compilation unit for {}", cl.getName(), e); return new SimpleIntrospection(f); diff --git a/autodoc/src/main/java/com/bakdata/conquery/introspection/RecordIntrospection.java b/autodoc/src/main/java/com/bakdata/conquery/introspection/RecordIntrospection.java deleted file mode 100644 index f023e8472a..0000000000 --- a/autodoc/src/main/java/com/bakdata/conquery/introspection/RecordIntrospection.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.bakdata.conquery.introspection; - -import java.io.File; - -import com.github.javaparser.ast.body.RecordDeclaration; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -public class RecordIntrospection extends AbstractNodeWithMemberIntrospection { - public RecordIntrospection(File file, RecordDeclaration recordDeclaration) { - super(file, recordDeclaration); - } -} From 4a0c0d9b2f8edb381caef7f30ff8da9cb7807108 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 22 Aug 2024 11:06:28 +0200 Subject: [PATCH 29/40] fix sign mistake in calculation --- .../bakdata/conquery/models/worker/ClusterHealthCheck.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java index 1211f76a14..41c6f8d6c7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ClusterHealthCheck.java @@ -12,9 +12,9 @@ import com.bakdata.conquery.io.mina.NetworkSession; import com.bakdata.conquery.mode.cluster.ClusterState; import com.codahale.metrics.health.HealthCheck; -import lombok.Data; +import lombok.RequiredArgsConstructor; -@Data +@RequiredArgsConstructor public class ClusterHealthCheck extends HealthCheck { public static final String HEALTHY_MESSAGE_FMT = "All %d known shards are connected."; @@ -38,7 +38,7 @@ protected Result check() throws Exception { LocalDateTime now = LocalDateTime.now(); List timeoutShards = nodeProvider.get().stream() - .filter((status) -> heartbeatTimeout.minus(Duration.between(now, status.getLastStatusTime())) + .filter((status) -> heartbeatTimeout.minus(Duration.between(status.getLastStatusTime(), now).abs()) .isNegative()).toList(); if (!timeoutShards.isEmpty()) { From 59b68f685f523c564c7017ee9139211138a7a0d9 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:15:38 +0200 Subject: [PATCH 30/40] cleanup and renaming --- ...ExternalResult.java => ExternalState.java} | 4 +- .../external/ExternalResultProcessor.java | 4 +- .../models/execution/InternalExecution.java | 11 +-- .../models/execution/ManagedExecution.java | 2 + .../forms/managed/ExternalExecution.java | 18 ++--- .../forms/managed/ManagedInternalForm.java | 17 ++-- .../query/DistributedExecutionManager.java | 33 ++++++-- .../models/query/ExecutionManager.java | 78 +++++++++++-------- ...ResultImpl.java => ExternalStateImpl.java} | 4 +- .../conquery/models/query/ManagedQuery.java | 13 +--- .../query/preview/EntityPreviewExecution.java | 13 ---- .../models/query/results/FormShardResult.java | 9 --- .../sql/conquery/SqlExecutionManager.java | 16 ++-- .../sql/execution/SqlExecutionResult.java | 2 +- 14 files changed, 107 insertions(+), 117 deletions(-) rename backend/src/main/java/com/bakdata/conquery/io/result/{ExternalResult.java => ExternalState.java} (92%) rename backend/src/main/java/com/bakdata/conquery/models/query/{ExternalResultImpl.java => ExternalStateImpl.java} (92%) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ExternalResult.java b/backend/src/main/java/com/bakdata/conquery/io/result/ExternalState.java similarity index 92% rename from backend/src/main/java/com/bakdata/conquery/io/result/ExternalResult.java rename to backend/src/main/java/com/bakdata/conquery/io/result/ExternalState.java index a4c56a883c..65db0fc592 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ExternalResult.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ExternalState.java @@ -16,7 +16,7 @@ /** * Interface for executions, whose final result is produced externally. */ -public interface ExternalResult extends ExecutionManager.Result { +public interface ExternalState extends ExecutionManager.State { /** * Returns the api object for the external form backend. @@ -30,7 +30,7 @@ public interface ExternalResult extends ExecutionManager.Result { /** * Returns assert builders for all results registered by an {@link com.bakdata.conquery.models.forms.managed.ExternalExecution} (see {@link AssetBuilder}). - * The provided assetId is the one that is used by {@link ExternalResult#fetchExternalResult(String)} to retrieve the download. + * The provided assetId is the one that is used by {@link ExternalState#fetchExternalResult(String)} to retrieve the download. */ @JsonIgnore Stream getResultAssets(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/external/ExternalResultProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/external/ExternalResultProcessor.java index e4a9a290e8..65b6e630cb 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/external/ExternalResultProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/external/ExternalResultProcessor.java @@ -3,7 +3,7 @@ import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; -import com.bakdata.conquery.io.result.ExternalResult; +import com.bakdata.conquery.io.result.ExternalState; import com.bakdata.conquery.io.result.ResultUtil; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.forms.managed.ExternalExecution; @@ -21,7 +21,7 @@ public Response getResult(Subject subject, ExternalExecution execution, String f ResultUtil.authorizeExecutable(subject, execution); ExecutionManager executionManager = datasetRegistry.get(execution.getDataset().getId()).getExecutionManager(); - ExternalResult externalResult = executionManager.getExternalResult(execution.getId()); + ExternalState externalResult = executionManager.getExternalResult(execution.getId()); return externalResult.fetchExternalResult(fileName); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/InternalExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/InternalExecution.java index 2474583b01..f56322b3a4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/InternalExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/InternalExecution.java @@ -1,18 +1,11 @@ package com.bakdata.conquery.models.execution; -import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; -import com.bakdata.conquery.models.query.results.ShardResult; - /** * This interface must be implemented if a {@link ManagedExecution} requires direct computation using the query engine on the shard nodes. * - * @param The type of result, the execution assumes to be returned from the shards. */ -public interface InternalExecution { +public interface InternalExecution { + - /** - * The message sent to shard nodes, to execute the query - */ - WorkerMessage createExecutionMessage(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index e3d8489e13..06821e92f5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -230,6 +230,8 @@ public void finish(ExecutionState executionState, ExecutionManager executionM } + + log.info("{} {} {} within {}", getState(), queryId, getClass().getSimpleName(), getExecutionTime()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java index 0e184fde03..a46a4a0a92 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ExternalExecution.java @@ -14,7 +14,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; import com.bakdata.conquery.io.external.form.ExternalTaskState; -import com.bakdata.conquery.io.result.ExternalResult; +import com.bakdata.conquery.io.result.ExternalState; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.entities.User; @@ -24,7 +24,7 @@ import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.query.ExecutionManager; -import com.bakdata.conquery.models.query.ExternalResultImpl; +import com.bakdata.conquery.models.query.ExternalStateImpl; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.api.ResultExternalResource; import com.bakdata.conquery.util.AuthUtil; @@ -95,7 +95,7 @@ public void start() { final ExternalTaskState externalTaskState = api.postForm(getSubmitted(), originalUser, serviceUser, dataset); - executionManager.addResult(this, new ExternalResultImpl(new CountDownLatch(0), api, serviceUser)); + executionManager.addState(this.getId(), new ExternalStateImpl(new CountDownLatch(0), api, serviceUser)); externalTaskId = externalTaskState.getId(); @@ -120,7 +120,7 @@ private void updateStatus(ExternalTaskState formState, ExecutionManager execu } case FAILURE -> fail(formState.getError(), executionManager); case SUCCESS -> { - List> resultsAssetMap = registerResultAssets(formState); + List> resultsAssetMap = registerResultAssets(formState); this.executionManager.getExternalResult(this.getId()).setResultsAssetMap(resultsAssetMap); finish(ExecutionState.DONE, executionManager); } @@ -128,8 +128,8 @@ private void updateStatus(ExternalTaskState formState, ExecutionManager execu } } - private List> registerResultAssets(ExternalTaskState response) { - final List> assetMap = new ArrayList<>(); + private List> registerResultAssets(ExternalTaskState response) { + final List> assetMap = new ArrayList<>(); response.getResults().forEach(asset -> assetMap.add(Pair.of(asset, createResultAssetBuilder(asset)))); return assetMap; } @@ -137,7 +137,7 @@ private List> registerResultAsset /** * The {@link ResultAsset} is request-dependent, so we can prepare only builder here which takes an url builder. */ - private ExternalResult.AssetBuilder createResultAssetBuilder(ResultAsset asset) { + private ExternalState.AssetBuilder createResultAssetBuilder(ResultAsset asset) { return (uriBuilder) -> { try { final URI externalDownloadURL = ResultExternalResource.getDownloadURL(uriBuilder.clone(), this, asset.getAssetId()); @@ -179,13 +179,13 @@ public void finish(ExecutionState executionState, ExecutionManager executionM } } finally { - executionManager.clearLockExternalExecution(this.getId()); + executionManager.clearBarrierExternalExecution(this.getId()); } } @JsonIgnore - public Stream getResultAssets() { + public Stream getResultAssets() { return executionManager.getExternalResult(this.getId()).getResultAssets(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java index 98944728a0..778fa6353a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java @@ -20,15 +20,13 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.IdMap; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; -import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteForm; import com.bakdata.conquery.models.query.ColumnDescriptor; +import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.models.query.results.FormShardResult; import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.AccessLevel; @@ -47,7 +45,7 @@ @Getter @EqualsAndHashCode(callSuper = true) @NoArgsConstructor(access = AccessLevel.PROTECTED) -public class ManagedInternalForm extends ManagedForm implements SingleTableResult, InternalExecution { +public class ManagedInternalForm extends ManagedForm implements SingleTableResult, InternalExecution { /** @@ -111,7 +109,7 @@ public List generateColumnDescriptions(boolean isInitialized, } - protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { + protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status, Namespace namespace) { // Set the ColumnDescription if the Form only consits of a single subquery if (subQueries == null) { // If subqueries was not set the Execution was not initialized, do it manually @@ -162,11 +160,12 @@ public long resultRowCount() { } @Override - public WorkerMessage createExecutionMessage() { - return new ExecuteForm(getId(), flatSubQueries.entrySet().stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getQuery()))); - } + public void finish(ExecutionState executionState, ExecutionManager executionManager) { + super.finish(executionState, executionManager); + // Signal to waiting threads that the form finished + executionManager.clearBarrierInternalExecution(this); + } public boolean allSubQueriesDone() { synchronized (this) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java index 4158f2f814..a0d0121163 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java @@ -6,6 +6,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.bakdata.conquery.io.storage.MetaStorage; @@ -18,17 +19,21 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.managed.ManagedInternalForm; import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; +import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; import com.bakdata.conquery.models.messages.namespaces.specific.CancelQuery; +import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteForm; +import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteQuery; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.WorkerHandler; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.NotImplementedException; @Slf4j public class DistributedExecutionManager extends ExecutionManager { - public record DistributedResult(Map> results, CountDownLatch executingLock) implements ExecutionManager.InternalResult { + public record DistributedResult(Map> results, CountDownLatch executingLock) implements InternalState { @Override public Stream streamQueryResults() { @@ -51,19 +56,33 @@ public DistributedExecutionManager(MetaStorage storage, ClusterState state) { @Override - protected > void doExecute(E execution) { + protected void doExecute(E execution) { log.info("Executing Query[{}] in Dataset[{}]", execution.getQueryId(), execution.getDataset()); - addResult(execution.getId(), new DistributedResult(new ConcurrentHashMap<>(), new CountDownLatch(1))); + addState(execution.getId(), new DistributedResult(new ConcurrentHashMap<>(), new CountDownLatch(1))); if (execution instanceof ManagedInternalForm form) { - form.getSubQueries().values().forEach((query) -> addResult(query.getId(), new DistributedResult(new ConcurrentHashMap<>(), new CountDownLatch(1)))); + form.getSubQueries().values().forEach((query) -> addState(query.getId(), new DistributedResult(new ConcurrentHashMap<>(), new CountDownLatch(1)))); } final WorkerHandler workerHandler = getWorkerHandler(execution); - workerHandler.sendToAll(execution.createExecutionMessage()); + workerHandler.sendToAll(createExecutionMessage(execution)); + } + + private WorkerMessage createExecutionMessage(ManagedExecution execution) { + if (execution instanceof ManagedQuery mq) { + return new ExecuteQuery(mq.getId(), mq.getQuery()); + } + else if (execution instanceof ManagedInternalForm form) { + return new ExecuteForm(form.getId(), form.getFlatSubQueries() + .entrySet() + .stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getQuery()))); + } + throw new NotImplementedException("Unable to build execution message for " + execution.getClass()); + } private WorkerHandler getWorkerHandler(ManagedExecution execution) { @@ -77,7 +96,7 @@ private WorkerHandler getWorkerHandler(ManagedExecution execution) { * @implNote subQueries of Forms are managed by the form itself, so need to be passed from outside. */ @SneakyThrows - public > void handleQueryResult(R result, E execution) { + public void handleQueryResult(R result, E execution) { log.debug("Received Result[size={}] for Query[{}]", result.getResults().size(), result.getQueryId()); @@ -90,7 +109,6 @@ public if (result.getError().isPresent()) { execution.fail(result.getError().get(), this); - clearLockInternalExecution(execution.getId()); } else { @@ -103,7 +121,6 @@ public // If all known workers have returned a result, the query is DONE. if (finishedWorkers.equals(getWorkerHandler(execution).getAllWorkerIds())) { execution.finish(ExecutionState.DONE, this); - clearLockInternalExecution(execution.getId()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index 258a8a3170..db21511aab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -8,7 +8,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.io.result.ExternalResult; +import com.bakdata.conquery.io.result.ExternalState; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.metrics.ExecutionMetrics; import com.bakdata.conquery.models.auth.AuthorizationHelper; @@ -32,30 +32,40 @@ @Data @Slf4j -public abstract class ExecutionManager { +public abstract class ExecutionManager { /** - * TODO Result might not be the appropriate name any more, because we will put everything in this instance - * belonging to an execution, which cannot/should not be serialized/cached in a store. + * Holds all informations about an execution, which cannot/should not be serialized/cached in a store. */ - public interface Result { + public interface State { + /** + * Synchronization barrier for web requests. + * Barrier is activated upon starting an execution so request can wait for execution completion. + * When the execution is finished the barrier is removed. + */ CountDownLatch getExecutingLock(); } - public interface InternalResult extends Result{ + public interface InternalState extends State{ Stream streamQueryResults(); } private final MetaStorage storage; - private final Cache executionResults = + /** + * Cache for internal execution state. + */ + private final Cache internalExecutionStates = CacheBuilder.newBuilder() .softValues() .removalListener(this::executionRemoved) .build(); - private final Cache externalExecutionResults = + /** + * Cache for external execution state. + */ + private final Cache externalExecutionStates = CacheBuilder.newBuilder() .softValues() .removalListener(this::executionRemoved) @@ -64,7 +74,7 @@ public interface InternalResult extends Result{ /** * Manage state of evicted Queries, setting them to NEW. */ - private void executionRemoved(RemovalNotification removalNotification) { + private void executionRemoved(RemovalNotification removalNotification) { // If removal was done manually we assume it was also handled properly if (!removalNotification.wasEvicted()) { return; @@ -88,22 +98,22 @@ public ManagedExecution getExecution(ManagedExecutionId execution) { } protected R getResult(ManagedExecutionId id) throws ExecutionException { - return executionResults.getIfPresent(id); + return internalExecutionStates.getIfPresent(id); } - public ExternalResult getExternalResult(ManagedExecutionId id) { - return externalExecutionResults.getIfPresent(id); + public ExternalState getExternalResult(ManagedExecutionId id) { + return externalExecutionStates.getIfPresent(id); } - protected void addResult(ManagedExecutionId id, R result) { - executionResults.put(id, result); + protected void addState(ManagedExecutionId id, R result) { + internalExecutionStates.put(id, result); } /** * Is called upon start by the external execution */ - public void addResult(ExternalExecution execution, ExternalResult result) { - externalExecutionResults.put(execution.getId(), result); + public void addState(ManagedExecutionId execution, ExternalState result) { + externalExecutionStates.put(execution, result); } public final ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, ConqueryConfig config, boolean system) { @@ -143,8 +153,8 @@ public final void execute(Namespace namespace, ManagedExecution execution, Conqu final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner(), storage).map(Group::getName).orElse("none"); ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).inc(); - if (execution instanceof InternalExecution internalExecution) { - doExecute((ManagedExecution & InternalExecution) internalExecution); + if (execution instanceof InternalExecution internalExecution) { + doExecute((ManagedExecution & InternalExecution) internalExecution); } } catch (Exception e) { @@ -153,7 +163,7 @@ public final void execute(Namespace namespace, ManagedExecution execution, Conqu } } - protected abstract > void doExecute(E execution); + protected abstract void doExecute(E execution); // Visible for testing public final ManagedExecution createExecution(QueryDescription query, User user, Namespace namespace, boolean system) { @@ -176,7 +186,7 @@ public final ManagedExecution createExecution(QueryDescription query, UUID query public final void cancelQuery(final ManagedExecution execution) { if (execution instanceof ExternalExecution externalExecution) { externalExecution.cancel(); - externalExecutionResults.invalidate(execution.getId()); + externalExecutionStates.invalidate(execution.getId()); return; } @@ -187,13 +197,15 @@ public final void cancelQuery(final ManagedExecution execution) { public abstract void doCancelQuery(final ManagedExecution execution); public void clearQueryResults(ManagedExecution execution) { - executionResults.invalidate(execution.getId()); - - //TODO external execution + if (execution instanceof InternalExecution) { + internalExecutionStates.invalidate(execution.getId()); + return; + } + externalExecutionStates.invalidate(execution.getId()); } - public Stream streamQueryResults(ManagedExecution execution) { - final R resultParts = executionResults.getIfPresent(execution.getId()); + public Stream streamQueryResults(E execution) { + final R resultParts = internalExecutionStates.getIfPresent(execution.getId()); return resultParts == null ? Stream.empty() @@ -201,14 +213,14 @@ public Stream streamQueryResults(ManagedExecution execution) { } - public void clearLockInternalExecution(ManagedExecutionId id) { - R result = Objects.requireNonNull(executionResults.getIfPresent(id), "Cannot clear lock on absent execution result"); + public void clearBarrierInternalExecution(E execution) { + R result = Objects.requireNonNull(internalExecutionStates.getIfPresent(execution.getId()), "Cannot clear lock on absent execution result"); result.getExecutingLock().countDown(); } - public void clearLockExternalExecution(ManagedExecutionId id) { - ExternalResult result = Objects.requireNonNull(externalExecutionResults.getIfPresent(id), "Cannot clear lock on absent execution result"); + public void clearBarrierExternalExecution(ManagedExecutionId id) { + ExternalState result = Objects.requireNonNull(externalExecutionStates.getIfPresent(id), "Cannot clear lock on absent execution result"); result.getExecutingLock().countDown(); } @@ -223,12 +235,12 @@ public ExecutionState awaitDone(ManagedExecution execution, int time, TimeUnit u return state; } - Result result; - if (execution instanceof InternalExecution) { - result = executionResults.getIfPresent(id); + State result; + if (execution instanceof InternalExecution) { + result = internalExecutionStates.getIfPresent(id); } else { - result = externalExecutionResults.getIfPresent(id); + result = externalExecutionStates.getIfPresent(id); } if (result == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExternalStateImpl.java similarity index 92% rename from backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java rename to backend/src/main/java/com/bakdata/conquery/models/query/ExternalStateImpl.java index 3575e4dad0..68911f1039 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExternalResultImpl.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExternalStateImpl.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; -import com.bakdata.conquery.io.result.ExternalResult; +import com.bakdata.conquery.io.result.ExternalState; import com.bakdata.conquery.models.auth.entities.User; import com.google.common.collect.MoreCollectors; import it.unimi.dsi.fastutil.Pair; @@ -17,7 +17,7 @@ import lombok.Setter; @RequiredArgsConstructor -public class ExternalResultImpl implements ExternalResult { +public class ExternalStateImpl implements ExternalState { private final CountDownLatch latch; @Getter diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 437e24e4fc..0b27fe578d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -22,11 +22,8 @@ import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.InternalExecution; import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; -import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteQuery; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -44,7 +41,7 @@ @Slf4j @CPSType(base = ManagedExecution.class, id = "MANAGED_QUERY") @NoArgsConstructor(access = AccessLevel.PROTECTED) -public class ManagedQuery extends ManagedExecution implements SingleTableResult, InternalExecution { +public class ManagedQuery extends ManagedExecution implements SingleTableResult, InternalExecution { // Needs to be resolved externally before being executed private Query query; @@ -74,6 +71,9 @@ public void finish(ExecutionState executionState, ExecutionManager executionM lastResultCount = query.countResults(streamResults(OptionalLong.empty())); super.finish(executionState, executionManager); + + // Signal to waiting threads that the form finished + executionManager.clearBarrierInternalExecution(this); } @@ -155,11 +155,6 @@ protected String makeDefaultLabel(PrintSettings cfg) { return QueryUtils.makeQueryLabel(query, cfg, getId()); } - @Override - public WorkerMessage createExecutionMessage() { - return new ExecuteQuery(getId(), getQuery()); - } - @Override public void visit(Consumer visitor) { visitor.accept(this); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index bf9c27122a..98a133f391 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -30,8 +30,6 @@ import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.ids.specific.SelectId; -import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; -import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteForm; import com.bakdata.conquery.models.query.ColumnDescriptor; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; @@ -72,8 +70,6 @@ public EntityPreviewExecution(EntityPreviewForm entityPreviewQuery, User user, D /** * Query contains both YEARS and QUARTERS lines: Group them. - * - * @return */ private static Map> getQuarterLines(EntityResult entityResult) { final Map> quarterLines = new HashMap<>(); @@ -97,8 +93,6 @@ private static Map> getQuarterLines(EntityResult /** * Query contains both YEARS and QUARTERS lines: Group them. - * - * @return */ private static Map getYearLines(EntityResult entityResult) { @@ -432,13 +426,6 @@ protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecu status.setColumnDescriptions(generateColumnDescriptions(isInitialized(), getConfig())); } - @Override - public WorkerMessage createExecutionMessage() { - return new ExecuteForm(getId(), getFlatSubQueries().entrySet() - .stream() - .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getQuery()))); - } - @Override public List getResultInfos() { return getValuesQuery().getResultInfos(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java b/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java index c7853a2a08..84bcdee3f7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/results/FormShardResult.java @@ -25,8 +25,6 @@ public FormShardResult(ManagedExecutionId formId, ManagedExecutionId subQueryId, /** * Distribute the result to a sub query. - * - * @param executionManager */ @Override public void addResult(DistributedExecutionManager executionManager) { @@ -48,17 +46,10 @@ public void addResult(DistributedExecutionManager executionManager) { ), executionManager ); - - // Signal to waiting threads that the form finished - executionManager.clearLockInternalExecution(formId); } if (managedInternalForm.allSubQueriesDone()) { - managedInternalForm.finish(ExecutionState.DONE, executionManager); - - // Signal to waiting threads that the form finished - executionManager.clearLockInternalExecution(formId); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index 75f850f7ec..c125d209f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -36,9 +36,9 @@ public SqlExecutionManager(SqlConverter sqlConverter, SqlExecutionService sqlExe } @Override - protected > void doExecute(E execution) { + protected void doExecute(E execution) { - addResult(execution.getId(), new SqlExecutionResult()); + addState(execution.getId(), new SqlExecutionResult()); if (execution instanceof ManagedQuery managedQuery) { CompletableFuture sqlQueryExecution = executeAsync(managedQuery, this); @@ -48,14 +48,11 @@ protected > void doExecute(E e if (execution instanceof ManagedInternalForm managedForm) { CompletableFuture.allOf(managedForm.getSubQueries().values().stream().map(managedQuery -> { - addResult(managedQuery.getId(), new SqlExecutionResult()); + addState(managedQuery.getId(), new SqlExecutionResult()); return executeAsync(managedQuery, this); }).toArray(CompletableFuture[]::new)) - .thenRun(() -> { - managedForm.finish(ExecutionState.DONE, this); - clearLockInternalExecution(managedForm.getId()); - }); + .thenRun(() -> managedForm.finish(ExecutionState.DONE, this)); return; } @@ -90,7 +87,7 @@ private CompletableFuture executeAsync(ManagedQuery managedQuery, SqlExecu SqlExecutionResult finishResult = new SqlExecutionResult(result.getColumnNames(), result.getTable(), startResult.getExecutingLock()); - addResult(id, finishResult); + addState(id, finishResult); } catch (ExecutionException e) { throw new RuntimeException(e); @@ -98,9 +95,6 @@ private CompletableFuture executeAsync(ManagedQuery managedQuery, SqlExecu managedQuery.setLastResultCount(((long) result.getRowCount())); managedQuery.finish(ExecutionState.DONE, executionManager); runningExecutions.remove(id); - - // Unlock waiting requests - clearLockInternalExecution(id); }) .exceptionally(e -> { managedQuery.fail(ConqueryError.asConqueryError(e), this); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java index 63b4bfa516..3c0e223287 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java @@ -9,7 +9,7 @@ import lombok.Value; @Value -public class SqlExecutionResult implements ExecutionManager.InternalResult { +public class SqlExecutionResult implements ExecutionManager.InternalState { List columnNames; List table; From 0854f7449adae7c4d1006cd237ebb3b41b163527 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 22 Aug 2024 16:34:18 +0200 Subject: [PATCH 31/40] restructures ResultType and decouples printing into ResultPrinters: (#3512) - restructures ResultType and decouples printing into ResultPrinters: - ResultType is now very simple mix of primitive enum and LIST. (We hope to remove even more logic from it) - Result printing was always weird with Mappings embedded in StringT. We've moved that responsibility into ResultInfo via SelectResultInfo for mapped Selects. - This PR opens the possibility to embed anonymization into the ResultInfo. --- .../bakdata/conquery/ConqueryConstants.java | 36 +- .../com/bakdata/conquery/ResultHeaders.java | 72 ++++ .../conquery/apiv1/QueryProcessor.java | 15 +- .../conquery/apiv1/forms/FeatureGroup.java | 21 +- .../apiv1/query/ArrayConceptQuery.java | 20 +- .../conquery/apiv1/query/CQElement.java | 3 +- .../bakdata/conquery/apiv1/query/CQYes.java | 3 +- .../conquery/apiv1/query/ConceptQuery.java | 19 +- .../bakdata/conquery/apiv1/query/Query.java | 3 +- .../apiv1/query/SecondaryIdQuery.java | 11 +- .../apiv1/query/TableExportQuery.java | 55 ++- .../apiv1/query/concept/specific/CQAnd.java | 27 +- .../query/concept/specific/CQConcept.java | 7 +- .../concept/specific/CQDateRestriction.java | 5 +- .../query/concept/specific/CQNegation.java | 5 +- .../apiv1/query/concept/specific/CQOr.java | 22 +- .../query/concept/specific/CQReusedQuery.java | 5 +- .../concept/specific/ResultInfoDecorator.java | 82 ---- .../concept/specific/external/CQExternal.java | 12 +- .../temporal/CQAbstractTemporalQuery.java | 7 +- .../io/result/arrow/ArrowRenderer.java | 48 +-- .../conquery/io/result/arrow/ArrowUtil.java | 31 +- .../io/result/arrow/ResultArrowProcessor.java | 12 +- .../conquery/io/result/csv/CsvRenderer.java | 6 +- .../io/result/csv/ResultCsvProcessor.java | 4 +- .../io/result/excel/ExcelRenderer.java | 44 +- .../io/result/excel/ResultExcelProcessor.java | 4 +- .../parquet/EntityResultWriteSupport.java | 52 +-- .../parquet/ResultParquetProcessor.java | 16 +- .../mode/local/LocalNamespaceHandler.java | 2 +- .../models/common/LocalizedToString.java | 7 + .../models/config/ExcelResultProvider.java | 21 +- .../models/config/IdColumnConfig.java | 41 +- .../models/datasets/concepts/Concept.java | 12 +- .../datasets/concepts/ConceptElement.java | 6 - .../datasets/concepts/select/Select.java | 12 +- .../select/concept/ConceptColumnSelect.java | 29 +- .../specific/EventDateUnionSelect.java | 4 +- .../specific/EventDurationSumSelect.java | 4 +- .../select/concept/specific/ExistsSelect.java | 4 +- .../concept/specific/QuarterSelect.java | 4 +- .../select/connector/DistinctSelect.java | 11 + .../select/connector/SingleColumnSelect.java | 8 +- .../specific/CountOccurencesSelect.java | 2 +- .../specific/CountQuartersSelect.java | 5 +- .../connector/specific/CountSelect.java | 5 +- .../specific/DateDistanceSelect.java | 4 +- .../connector/specific/DateUnionSelect.java | 4 +- .../connector/specific/DurationSumSelect.java | 4 +- .../select/connector/specific/FlagSelect.java | 4 +- .../specific/MappableSingleColumnSelect.java | 46 +-- .../connector/specific/PrefixSelect.java | 4 +- .../specific/QuartersInYearSelect.java | 4 +- .../select/connector/specific/SumSelect.java | 14 +- .../datasets/concepts/tree/TreeConcept.java | 33 +- .../models/execution/ManagedExecution.java | 2 +- .../forms/export/RelExportGenerator.java | 21 +- .../forms/managed/AbsoluteFormQuery.java | 15 +- .../models/forms/managed/EntityDateQuery.java | 13 +- .../forms/managed/ManagedInternalForm.java | 5 +- .../forms/managed/RelativeFormQuery.java | 23 +- .../models/forms/util/Resolution.java | 61 +-- .../conquery/models/query/C10nCache.java | 18 + .../conquery/models/query/ManagedQuery.java | 4 +- .../conquery/models/query/PrintSettings.java | 10 +- .../models/query/SingleTableResult.java | 10 +- .../query/preview/EntityPreviewExecution.java | 78 +--- .../specific/ExistsAggregator.java | 9 +- .../queryplan/specific/ExternalNode.java | 5 +- .../query/resultinfo/ColumnResultInfo.java | 30 +- .../query/resultinfo/ExternalResultInfo.java | 43 ++ .../resultinfo/FixedLabelResultInfo.java | 74 ++++ .../LocalizedDefaultResultInfo.java | 77 ---- .../models/query/resultinfo/ResultInfo.java | 64 ++- .../query/resultinfo/SelectResultInfo.java | 54 +-- .../query/resultinfo/SimpleResultInfo.java | 38 -- .../models/query/resultinfo/UniqueNamer.java | 2 +- .../resultinfo/printers/ResultPrinters.java | 227 +++++++++++ .../printers/SecondaryIdResultInfo.java | 44 ++ .../BooleanColumnStatsCollector.java | 12 +- .../statistics/ColumnStatsCollector.java | 44 +- .../statistics/DateColumnStatsCollector.java | 16 +- .../statistics/ListColumnStatsCollector.java | 4 +- .../NumberColumnStatsCollector.java | 46 +-- .../query/statistics/ResultStatistics.java | 8 +- .../StringColumnStatsCollector.java | 12 +- .../conquery/models/types/ResultType.java | 377 ++++-------------- .../sql/conquery/SqlExecutionManager.java | 12 +- .../sql/conversion/NodeConversions.java | 8 +- .../conquery/sql/conversion/SqlConverter.java | 10 +- .../cqelement/ConversionContext.java | 3 + .../cqelement/concept/TablePath.java | 11 +- .../sql/conversion/model/NameGenerator.java | 8 +- .../query/AbsoluteFormQueryConverter.java | 2 +- .../query/ConceptQueryConverter.java | 2 +- .../query/EntityDateQueryConverter.java | 2 +- .../query/RelativFormQueryConverter.java | 2 +- .../query/SecondaryIdQueryConverter.java | 2 +- .../query/TableExportQueryConverter.java | 4 +- .../sql/execution/SqlExecutionService.java | 6 +- .../com/bakdata/conquery/util/QueryUtils.java | 1 + .../api/StoredQueriesProcessorTest.java | 7 +- .../DefaultSqlCDateSetParserTest.java | 5 +- .../json/AbstractQueryEngineTest.java | 6 +- .../conquery/integration/json/FormTest.java | 30 +- .../integration/tests/EntityExportTest.java | 4 +- .../conquery/io/result/ResultTestUtil.java | 70 ++-- .../arrow/ArrowResultGenerationTest.java | 31 +- .../result/csv/CsvResultGenerationTest.java | 10 +- .../result/excel/ExcelResultRenderTest.java | 21 +- .../parquet/ParquetResultGenerationTest.java | 20 +- .../models/query/DefaultColumnNameTest.java | 9 +- .../conquery/models/query/UniqueNameTest.java | 14 +- .../conquery/models/types/ResultTypeTest.java | 128 +++--- 114 files changed, 1378 insertions(+), 1437 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/ResultHeaders.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/ResultInfoDecorator.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/common/LocalizedToString.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/C10nCache.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ExternalResultInfo.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/FixedLabelResultInfo.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/LocalizedDefaultResultInfo.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SimpleResultInfo.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/ResultPrinters.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/printers/SecondaryIdResultInfo.java diff --git a/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java b/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java index 4c21a4cc77..8f4eca6c4f 100644 --- a/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/ConqueryConstants.java @@ -1,15 +1,5 @@ package com.bakdata.conquery; -import java.util.Set; - -import c10n.C10N; -import com.bakdata.conquery.apiv1.forms.FeatureGroup; -import com.bakdata.conquery.internationalization.ResultHeadersC10n; -import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.models.types.SemanticType; import lombok.experimental.UtilityClass; @UtilityClass @@ -18,35 +8,11 @@ public class ConqueryConstants { public static final String ALL_IDS_TABLE = "ALL_IDS"; public static final String EXTENSION_PREPROCESSED = ".cqpp"; public static final String EXTENSION_DESCRIPTION = ".import.json"; - - public static final ResultInfo DATES_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).dates(), new ResultType.ListT(ResultType.DateRangeT.INSTANCE), Set.of(new SemanticType.EventDateT())); - - public static final ResultInfo DATES_INFO_HISTORY = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).dates(), new ResultType.ListT(ResultType.DateRangeT.INSTANCE), Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT())); - - - public static final ResultInfo - SOURCE_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).source(), ResultType.StringT.INSTANCE, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT())); - // Form related constants public static final String SINGLE_RESULT_TABLE_NAME = "results"; - public static final ResultInfo CONTEXT_INDEX_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).index(), ResultType.IntegerT.INSTANCE, Set.of()); - public static final ResultInfo DATE_RANGE_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).dateRange(), ResultType.DateRangeT.INSTANCE, Set.of()); - public static final ResultInfo RESOLUTION_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).resolution(), new ResultType.StringT(Resolution::localizeValue), Set.of(new SemanticType.ResolutionT())); - public static final ResultInfo EVENT_DATE_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).eventDate(), ResultType.DateT.INSTANCE, Set.of()); - - public static final ResultInfo - OBSERVATION_SCOPE_INFO = - new LocalizedDefaultResultInfo((l) -> C10N.get(ResultHeadersC10n.class, l).observationScope(), new ResultType.StringT(FeatureGroup::localizeValue), Set.of()); - /** * Drawn from random.org */ public static final long RANDOM_SEED = 694011229L; + } diff --git a/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java new file mode 100644 index 0000000000..9d3eb527e3 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/ResultHeaders.java @@ -0,0 +1,72 @@ +package com.bakdata.conquery; + +import java.util.Set; + +import com.bakdata.conquery.apiv1.forms.FeatureGroup; +import com.bakdata.conquery.internationalization.ResultHeadersC10n; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.models.query.C10nCache; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.models.types.SemanticType; +import lombok.experimental.UtilityClass; + +@UtilityClass +public class ResultHeaders { + public static ResultInfo datesInfo(PrintSettings settings) { + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).dates(); + + final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); + + return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT()), settings, ResultPrinters.printerFor(type, settings)); + } + + public static ResultInfo historyDatesInfo(PrintSettings settings) { + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).dates(); + + final ResultType.ListT type = new ResultType.ListT<>(ResultType.Primitive.DATE_RANGE); + + return new FixedLabelResultInfo(label, label, type, Set.of(new SemanticType.EventDateT(), new SemanticType.GroupT()), settings, ResultPrinters.printerFor(type, settings)); + } + + public static ResultInfo sourceInfo(PrintSettings settings) { + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).source(); + + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.SourcesT(), new SemanticType.CategoricalT(), new SemanticType.GroupT()), settings, ResultPrinters.printerFor(ResultType.Primitive.STRING, settings)); + } + + public static ResultInfo formContextInfo(PrintSettings settings) { + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).index(); + + return new FixedLabelResultInfo(label, label, ResultType.Primitive.INTEGER, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.INTEGER, settings)); + } + + public static ResultInfo formDateRangeInfo(PrintSettings settings) { + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()) + .dateRange(); + + return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE_RANGE, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.DATE_RANGE, settings)); + } + + public static ResultInfo formResolutionInfo(PrintSettings settings) { + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).resolution(); + + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings, new ResultPrinters.LocalizedEnumPrinter<>(settings, Resolution.class)); + } + + public static ResultInfo formEventDateInfo(PrintSettings settings) { + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()) + .eventDate(); + + return new FixedLabelResultInfo(label, label, ResultType.Primitive.DATE, Set.of(), settings, ResultPrinters.printerFor(ResultType.Primitive.DATE, settings)); + } + + public static ResultInfo formObservationScopeInfo(PrintSettings settings) { + final String label = C10nCache.getLocalized(ResultHeadersC10n.class, settings.getLocale()).observationScope(); + + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(), settings, new ResultPrinters.LocalizedEnumPrinter<>(settings, FeatureGroup.class)); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 92d2b9b463..7bb95fa2f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -560,13 +560,6 @@ public Stream> resolveEntities(Subject subject, List resultInfos = managedQuery.getResultInfos(); - - final Optional - dateInfo = - resultInfos.stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst(); - - final Optional dateIndex = dateInfo.map(resultInfos::indexOf); final Locale locale = I18n.LOCALE.get(); final NumberFormat decimalFormat = NumberFormat.getNumberInstance(locale); @@ -579,6 +572,14 @@ public ResultStatistics getResultStatistics(SingleTableResult managedQuery) { new PrintSettings(true, locale, managedQuery.getNamespace(), config, null, null, decimalFormat, integerFormat); final UniqueNamer uniqueNamer = new UniqueNamer(printSettings); + final List resultInfos = managedQuery.getResultInfos(printSettings); + + final Optional + dateInfo = + resultInfos.stream().filter(info -> info.getSemantics().contains(new SemanticType.EventDateT())).findFirst(); + + final Optional dateIndex = dateInfo.map(resultInfos::indexOf); + return ResultStatistics.collectResultStatistics(managedQuery, resultInfos, dateInfo, dateIndex, printSettings, uniqueNamer, config); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FeatureGroup.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FeatureGroup.java index 75fc4ca3d0..80c329348f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FeatureGroup.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/FeatureGroup.java @@ -4,8 +4,7 @@ import c10n.C10N; import com.bakdata.conquery.internationalization.ResultHeadersC10n; -import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.common.LocalizedToString; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -16,7 +15,7 @@ */ @Getter @RequiredArgsConstructor -public enum FeatureGroup { +public enum FeatureGroup implements LocalizedToString { FEATURE() { @Override public String toString(Locale locale) { @@ -34,21 +33,5 @@ public String toString(Locale locale) { public String toString(Locale locale) { return ""; } - }; - - public abstract String toString(Locale locale); - - public static String localizeValue(Object value, PrintSettings cfg) { - if (value instanceof Resolution) { - return ((Resolution) value).toString(cfg.getLocale()); - } - try { - // If the object was parsed as a simple string, try to convert it to a - // FeatureGroup to get Internationalization - return FeatureGroup.valueOf((String) value).toString(cfg.getLocale()); - } - catch (Exception e) { - throw new IllegalArgumentException(value + " is not a valid resolution.", e); - } } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java index cf9344539b..32e7871b23 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ArrayConceptQuery.java @@ -6,16 +6,18 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import com.bakdata.conquery.ConqueryConstants; +import com.bakdata.conquery.ResultHeaders; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.queryplan.ArrayConceptQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonView; import jakarta.validation.Valid; @@ -59,9 +61,6 @@ public static ArrayConceptQuery createFromFeatures(List features) { } public ArrayConceptQuery(@NonNull List queries, @NonNull DateAggregationMode dateAggregationMode) { - if (queries == null) { - throw new IllegalArgumentException("No sub query list provided."); - } this.childQueries = queries; this.dateAggregationMode = dateAggregationMode; } @@ -83,7 +82,7 @@ public void resolve(QueryResolveContext context) { @Override public ArrayConceptQueryPlan createQueryPlan(QueryPlanContext context) { // Make sure the constructor and the adding is called with the same context. - ArrayConceptQueryPlan aq = new ArrayConceptQueryPlan(!DateAggregationMode.NONE.equals(resolvedDateAggregationMode)); + ArrayConceptQueryPlan aq = new ArrayConceptQueryPlan(resolvedDateAggregationMode != DateAggregationMode.NONE); aq.addChildPlans(childQueries, context); return aq; } @@ -94,21 +93,22 @@ public void collectRequiredQueries(Set requiredQueries) { } @Override - public List getResultInfos() { + public List getResultInfos(PrintSettings printSettings) { final List resultInfos = new ArrayList<>(); - ResultInfo dateInfo = ConqueryConstants.DATES_INFO; + ResultInfo dateInfo = ResultHeaders.datesInfo(printSettings); - if(!DateAggregationMode.NONE.equals(getResolvedDateAggregationMode())){ + if(getResolvedDateAggregationMode() != DateAggregationMode.NONE){ // Add one DateInfo for the whole Query resultInfos.add(0, dateInfo); } int lastIndex = resultInfos.size(); - childQueries.forEach(q -> resultInfos.addAll(q.getResultInfos())); + childQueries.forEach(q -> resultInfos.addAll(q.getResultInfos(printSettings))); if(!resultInfos.isEmpty()) { // Remove DateInfo from each childQuery - resultInfos.subList(lastIndex, resultInfos.size()).removeAll(List.of(dateInfo)); + resultInfos.subList(lastIndex, resultInfos.size()) + .removeIf(resultInfo -> resultInfo.getSemantics().contains(new SemanticType.EventDateT())); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java index 61806adfb4..cf309dcf90 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQElement.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -78,7 +79,7 @@ public final Set collectRequiredQueries() { public abstract void collectRequiredQueries(Set requiredQueries) ; @JsonIgnore - public abstract List getResultInfos(); + public abstract List getResultInfos(PrintSettings settings); public void visit(Consumer visitor) { visitor.accept(this); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java index 8bdc6a00d6..2e3e34a727 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; @@ -32,7 +33,7 @@ public void collectRequiredQueries(Set requiredQueries) { } @Override - public List getResultInfos() { + public List getResultInfos(PrintSettings settings) { return Collections.emptyList(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java index 3ddece071f..6f028f3684 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/ConceptQuery.java @@ -5,11 +5,12 @@ import java.util.Set; import java.util.function.Consumer; -import com.bakdata.conquery.ConqueryConstants; +import com.bakdata.conquery.ResultHeaders; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -56,7 +57,7 @@ public ConceptQuery(CQElement root, DateAggregationMode dateAggregationMode) { @Override public ConceptQueryPlan createQueryPlan(QueryPlanContext context) { - ConceptQueryPlan qp = new ConceptQueryPlan(!DateAggregationMode.NONE.equals(resolvedDateAggregationMode)); + ConceptQueryPlan qp = new ConceptQueryPlan(resolvedDateAggregationMode != DateAggregationMode.NONE); qp.setChild(root.createQueryPlan(context, qp)); qp.getDateAggregator().registerAll(qp.getChild().getDateAggregators()); return qp; @@ -70,6 +71,7 @@ public void collectRequiredQueries(Set requiredQueries) { @Override public void resolve(QueryResolveContext context) { resolvedDateAggregationMode = dateAggregationMode; + if (context.getDateAggregationMode() != null) { log.trace("Overriding date aggregation mode ({}) with mode from context ({})", dateAggregationMode, context.getDateAggregationMode()); resolvedDateAggregationMode = context.getDateAggregationMode(); @@ -78,13 +80,16 @@ public void resolve(QueryResolveContext context) { } @Override - public List getResultInfos() { + public List getResultInfos(PrintSettings printSettings) { Preconditions.checkNotNull(resolvedDateAggregationMode); - List resultInfos = new ArrayList<>(); - if (!DateAggregationMode.NONE.equals(resolvedDateAggregationMode)) { - resultInfos.add(ConqueryConstants.DATES_INFO); + + final List resultInfos = new ArrayList<>(); + + if (resolvedDateAggregationMode != DateAggregationMode.NONE) { + resultInfos.add(ResultHeaders.datesInfo(printSettings)); } - resultInfos.addAll(root.getResultInfos()); + + resultInfos.addAll(root.getResultInfos(printSettings)); return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java index cde81b3cc1..483c71a545 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/Query.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.ManagedQuery; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.QueryPlan; @@ -37,7 +38,7 @@ public Set collectRequiredQueries() { } @JsonIgnore - public abstract List getResultInfos(); + public abstract List getResultInfos(PrintSettings printSettings); @Override public ManagedQuery toManagedExecution(User user, Dataset submittedDataset, MetaStorage storage) { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index ec3918246d..85ba3ddc9a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -26,9 +27,7 @@ import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.SecondaryIdQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo; -import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; import com.fasterxml.jackson.annotation.JsonView; import jakarta.validation.constraints.NotNull; import lombok.Getter; @@ -126,12 +125,12 @@ public void resolve(final QueryResolveContext context) { } @Override - public List getResultInfos() { + public List getResultInfos(PrintSettings printSettings) { final List resultInfos = new ArrayList<>(); - resultInfos.add(new SimpleResultInfo(secondaryId.getLabel(), ResultType.StringT.INSTANCE, secondaryId.getDescription(), Set.of(new SemanticType.SecondaryIdT(getSecondaryId())))); + resultInfos.add(new SecondaryIdResultInfo(secondaryId, printSettings)); - resultInfos.addAll(query.getResultInfos()); + resultInfos.addAll(query.getResultInfos(printSettings)); return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 8f6576d08a..c8f88d8dbb 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -3,7 +3,6 @@ import java.time.LocalDate; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; @@ -15,7 +14,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import com.bakdata.conquery.ConqueryConstants; +import com.bakdata.conquery.ResultHeaders; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -32,6 +31,7 @@ import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -41,7 +41,8 @@ import com.bakdata.conquery.models.query.queryplan.TableExportQueryPlan; import com.bakdata.conquery.models.query.resultinfo.ColumnResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; +import com.bakdata.conquery.models.query.resultinfo.printers.SecondaryIdResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonCreator; @@ -102,9 +103,10 @@ public class TableExportQuery extends Query { @JsonView(View.InternalCommunication.class) private Map positions; - @JsonIgnore - private List resultInfos = Collections.emptyList(); + private Set conceptColumns; + @JsonIgnore + private Map secondaryIdPositions; @Override @@ -142,7 +144,7 @@ public void resolve(QueryResolveContext context) { // First is dates, second is source id final AtomicInteger currentPosition = new AtomicInteger(2); - final Map secondaryIdPositions = calculateSecondaryIdPositions(currentPosition); + secondaryIdPositions = calculateSecondaryIdPositions(currentPosition); final Set validityDates = tables.stream() .map(CQConcept::getTables) @@ -152,7 +154,7 @@ public void resolve(QueryResolveContext context) { .collect(Collectors.toSet()); // We need to know if a column is a concept column so we can prioritize it if it is also a SecondaryId - final Set conceptColumns = tables.stream() + conceptColumns = tables.stream() .map(CQConcept::getTables) .flatMap(Collection::stream) .map(CQTable::getConnector) @@ -162,7 +164,12 @@ public void resolve(QueryResolveContext context) { positions = calculateColumnPositions(currentPosition, tables, secondaryIdPositions, conceptColumns, validityDates); - resultInfos = createResultInfos(secondaryIdPositions, conceptColumns); + + } + + @Override + public List getResultInfos(PrintSettings printSettings) { + return createResultInfos(conceptColumns, printSettings); } private Map calculateSecondaryIdPositions(AtomicInteger currentPosition) { @@ -215,36 +222,21 @@ private static Map calculateColumnPositions(AtomicInteger curre return positions; } - private List createResultInfos(Map secondaryIdPositions, Set conceptColumns) { + private List createResultInfos(Set conceptColumns, PrintSettings printSettings) { final int size = positions.values().stream().mapToInt(i -> i).max().getAsInt() + 1; final ResultInfo[] infos = new ResultInfo[size]; - infos[0] = ConqueryConstants.DATES_INFO_HISTORY; - infos[1] = ConqueryConstants.SOURCE_INFO; + infos[0] = ResultHeaders.historyDatesInfo(printSettings); + infos[1] = ResultHeaders.sourceInfo(printSettings); for (Map.Entry e : secondaryIdPositions.entrySet()) { final SecondaryIdDescription desc = e.getKey(); final Integer pos = e.getValue(); - // If mapping is available, values are mapped - final ResultType.StringT resultType = - desc.getMapping() != null - ? new ResultType.StringT((internal, printSettings) -> { - if (internal == null) { - return null; - } - return desc.getMapping().external((String) internal); - }) - : ResultType.StringT.INSTANCE; - - final Set semantics = new HashSet<>(); - - semantics.add(new SemanticType.SecondaryIdT(desc)); - - infos[pos] = new SimpleResultInfo(desc.getLabel(), resultType, desc.getDescription(), semantics); + infos[pos] = new SecondaryIdResultInfo(desc, printSettings); } @@ -267,14 +259,14 @@ private List createResultInfos(Map // SecondaryIds and date columns are pulled to the front, thus already covered. if (column.getSecondaryId() != null && !conceptColumns.contains(column)) { - infos[secondaryIdPositions.get(column.getSecondaryId())].getSemantics() - .add(new SemanticType.ColumnT(column)); + infos[secondaryIdPositions.get(column.getSecondaryId())].addSemantics(new SemanticType.ColumnT(column)); continue; } final Set semantics = new HashSet<>(); ResultType resultType = ResultType.resolveResultType(column.getType()); + ResultPrinters.Printer printer = ResultPrinters.printerFor(resultType, printSettings); if (connectorColumns.containsKey(column)) { // Additionally, Concept Columns are returned as ConceptElementId, when rawConceptColumns is not set. @@ -285,7 +277,8 @@ private List createResultInfos(Map semantics.add(new SemanticType.ConceptColumnT(concept)); if (!isRawConceptValues()) { - resultType = new ResultType.StringT(concept::printConceptLocalId); + resultType = ResultType.Primitive.STRING; + printer = new ResultPrinters.ConceptIdPrinter(concept, printSettings); } } else { @@ -293,7 +286,7 @@ private List createResultInfos(Map semantics.add(new SemanticType.ColumnT(column)); } - infos[position] = new ColumnResultInfo(column, resultType, semantics); + infos[position] = new ColumnResultInfo(column, resultType, semantics, printer, column.getDescription(), printSettings); } return List.of(infos); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java index f598846635..f49ff668f9 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQAnd.java @@ -8,13 +8,14 @@ import java.util.Set; import java.util.function.Consumer; -import c10n.C10N; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.internationalization.CQElementC10n; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.C10nCache; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -25,8 +26,9 @@ import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.queryplan.specific.AndNode; -import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; +import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonView; @@ -35,8 +37,10 @@ import jakarta.validation.constraints.NotEmpty; import lombok.Getter; import lombok.Setter; +import lombok.ToString; @CPSType(id = "AND", base = CQElement.class) +@ToString public class CQAnd extends CQElement implements ExportForm.DefaultSelectSettable { @Getter @@ -65,7 +69,7 @@ public void setDefaultExists() { public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { Preconditions.checkNotNull(dateAction); - QPNode[] nodes = new QPNode[children.size()]; + final QPNode[] nodes = new QPNode[children.size()]; for (int i = 0; i < nodes.length; i++) { nodes[i] = children.get(i).createQueryPlan(context, plan); } @@ -109,16 +113,19 @@ private DateAggregationAction determineDateAction(QueryResolveContext context) { } @Override - public List getResultInfos() { - List resultInfos = new ArrayList<>(); + public List getResultInfos(PrintSettings settings) { + final List resultInfos = new ArrayList<>(); for (CQElement c : children) { - resultInfos.addAll(c.getResultInfos()); + resultInfos.addAll(c.getResultInfos(settings)); } if (createExists()) { - resultInfos.add(new LocalizedDefaultResultInfo(this::getUserOrDefaultLabel, this::defaultLabel, ResultType.BooleanT.INSTANCE, Set.of())); - } + final ResultPrinters.BooleanPrinter printer = new ResultPrinters.BooleanPrinter(settings); + final String userOrDefaultLabel = getUserOrDefaultLabel(settings.getLocale()); + final String defaultLabel = defaultLabel(settings.getLocale()); + resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings, printer)); + } return resultInfos; } @@ -128,13 +135,13 @@ public String getUserOrDefaultLabel(Locale locale) { if (getLabel() != null) { return getLabel(); } - return QueryUtils.createDefaultMultiLabel(children, " " + C10N.get(CQElementC10n.class, locale).and() + " ", locale); + return QueryUtils.createDefaultMultiLabel(children, " " + C10nCache.getLocalized(CQElementC10n.class, locale).and() + " ", locale); } @Override public String defaultLabel(Locale locale) { // This forces the default label on children even if there was a user label - return QueryUtils.createTotalDefaultMultiLabel(children, " " + C10N.get(CQElementC10n.class, locale).and() + " ", locale); + return QueryUtils.createTotalDefaultMultiLabel(children, " " + C10nCache.getLocalized(CQElementC10n.class, locale).and() + " ", locale); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index be0e5814e9..538a987d93 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -24,6 +24,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.models.query.NamespacedIdentifiableHolding; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -285,16 +286,16 @@ private ValidityDate selectValidityDate(CQTable table) { } @Override - public List getResultInfos() { + public List getResultInfos(PrintSettings settings) { final List resultInfos = new ArrayList<>(); for (Select select : selects) { - resultInfos.add(select.getResultInfo(this)); + resultInfos.add(select.getResultInfo(this, settings)); } for (CQTable table : tables) { for (Select sel : table.getSelects()) { - resultInfos.add(sel.getResultInfo(this)); + resultInfos.add(sel.getResultInfo(this, settings)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java index 6f78e0be8f..251f5c779b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQDateRestriction.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -59,8 +60,8 @@ public void resolve(QueryResolveContext context) { } @Override - public List getResultInfos() { - return child.getResultInfos(); + public List getResultInfos(PrintSettings settings) { + return child.getResultInfos(settings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java index 57fbef50f4..2ae44af793 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQNegation.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; @@ -66,8 +67,8 @@ private DateAggregationAction determineDateAction(QueryResolveContext context) { } @Override - public List getResultInfos() { - return child.getResultInfos(); + public List getResultInfos(PrintSettings settings) { + return child.getResultInfos(settings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java index e93562016d..6037200bea 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQOr.java @@ -8,13 +8,14 @@ import java.util.Set; import java.util.function.Consumer; -import c10n.C10N; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.internationalization.CQElementC10n; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.C10nCache; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -25,8 +26,9 @@ import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; import com.bakdata.conquery.models.query.queryplan.specific.OrNode; -import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; +import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JsonView; @@ -37,10 +39,12 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import lombok.ToString; @NoArgsConstructor @AllArgsConstructor @CPSType(id = "OR", base = CQElement.class) +@ToString public class CQOr extends CQElement implements ExportForm.DefaultSelectSettable { @Getter @Setter @@ -114,14 +118,18 @@ private DateAggregationAction determineDateAction(QueryResolveContext context) { } @Override - public List getResultInfos() { + public List getResultInfos(PrintSettings settings) { List resultInfos = new ArrayList<>(); for (CQElement c : children) { - resultInfos.addAll(c.getResultInfos()); + resultInfos.addAll(c.getResultInfos(settings)); } if (createExists()) { - resultInfos.add(new LocalizedDefaultResultInfo(this::getUserOrDefaultLabel, this::defaultLabel, ResultType.BooleanT.INSTANCE, Set.of())); + final ResultPrinters.BooleanPrinter printer = new ResultPrinters.BooleanPrinter(settings); + final String userOrDefaultLabel = getUserOrDefaultLabel(settings.getLocale()); + final String defaultLabel = defaultLabel(settings.getLocale()); + + resultInfos.add(new FixedLabelResultInfo(userOrDefaultLabel, defaultLabel, ResultType.Primitive.BOOLEAN, Set.of(), settings, printer)); } return resultInfos; @@ -133,13 +141,13 @@ public String getUserOrDefaultLabel(Locale locale) { if (getLabel() != null) { return getLabel(); } - return QueryUtils.createDefaultMultiLabel(children, " " + C10N.get(CQElementC10n.class, locale).or() + " ", locale); + return QueryUtils.createDefaultMultiLabel(children, " " + C10nCache.getLocalized(CQElementC10n.class, locale).or() + " ", locale); } @Override public String defaultLabel(Locale locale) { // This forces the default label on children even if there was a user label - return QueryUtils.createTotalDefaultMultiLabel(children, " " + C10N.get(CQElementC10n.class, locale).or() + " ", locale); + return QueryUtils.createTotalDefaultMultiLabel(children, " " + C10nCache.getLocalized(CQElementC10n.class, locale).or() + " ", locale); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java index 496fa51120..7162f8f4fb 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.ManagedQuery; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -92,8 +93,8 @@ public void visit(Consumer visitor) { } @Override - public List getResultInfos() { - return resolvedQuery.getReusableComponents().getResultInfos(); + public List getResultInfos(PrintSettings settings) { + return resolvedQuery.getReusableComponents().getResultInfos(settings); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/ResultInfoDecorator.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/ResultInfoDecorator.java deleted file mode 100644 index 3f7b5454cd..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/ResultInfoDecorator.java +++ /dev/null @@ -1,82 +0,0 @@ -package com.bakdata.conquery.apiv1.query.concept.specific; - -import java.util.List; -import java.util.Set; -import java.util.function.Consumer; - -import com.bakdata.conquery.apiv1.query.CQElement; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.models.query.QueryExecutionContext; -import com.bakdata.conquery.models.query.QueryPlanContext; -import com.bakdata.conquery.models.query.QueryResolveContext; -import com.bakdata.conquery.models.query.RequiredEntities; -import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; -import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.google.common.collect.ClassToInstanceMap; -import com.google.common.collect.MutableClassToInstanceMap; -import jakarta.validation.constraints.NotNull; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -/** - * A wrapper for {@link CQElement}s to provide additional infos to parts of a query. - */ -@Getter -@Setter -@ToString -@NoArgsConstructor -@AllArgsConstructor -@CPSType(id = "RESULT_INFO_DECORATOR", base = CQElement.class) -public class ResultInfoDecorator extends CQElement { - - @NotNull - private ClassToInstanceMap values = MutableClassToInstanceMap.create(); - @NotNull - private CQElement child; - - @SuppressWarnings({"rawtypes", "unchecked"}) - @Override - public List getResultInfos() { - List resultInfos = child.getResultInfos(); - resultInfos.listIterator() - .forEachRemaining(sd -> { - for (Class entry : values.keySet()) { - sd.addAppendix(entry, values.getInstance(entry)); - } - }); - return resultInfos; - } - - @Override - public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { - return child.createQueryPlan(context, plan); - } - - @Override - public void visit(Consumer visitor) { - super.visit(visitor); - child.visit(visitor); - } - - @Override - public void collectRequiredQueries(Set requiredQueries) { - child.collectRequiredQueries(requiredQueries); - } - - @Override - public void resolve(QueryResolveContext context) { - child.resolve(context); - } - - - @Override - public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { - return getChild().collectRequiredEntities(context); - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 2ba0d02a0b..d6d23de78d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.config.IdColumnConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -27,8 +28,8 @@ import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ConstantValueAggregator; import com.bakdata.conquery.models.query.queryplan.specific.ExternalNode; +import com.bakdata.conquery.models.query.resultinfo.ExternalResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.models.query.resultinfo.SimpleResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonView; @@ -214,7 +215,7 @@ public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { } @Override - public List getResultInfos() { + public List getResultInfos(PrintSettings settings) { if (extra == null) { return Collections.emptyList(); } @@ -224,11 +225,10 @@ public List getResultInfos() { continue; } - String column = headers[col]; + final String column = headers[col]; - resultInfos.add(new SimpleResultInfo(column, onlySingles ? - ResultType.StringT.INSTANCE : - new ResultType.ListT(ResultType.StringT.INSTANCE))); + final ResultType type = onlySingles ? ResultType.Primitive.STRING : new ResultType.ListT<>(ResultType.Primitive.STRING); + resultInfos.add(new ExternalResultInfo(column, type, settings)); } return resultInfos; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQAbstractTemporalQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQAbstractTemporalQuery.java index 63e33a0616..33a720c5ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQAbstractTemporalQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/temporal/CQAbstractTemporalQuery.java @@ -8,6 +8,7 @@ import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.DateAggregationMode; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -80,10 +81,10 @@ public void resolve(QueryResolveContext context) { } @Override - public List getResultInfos() { + public List getResultInfos(PrintSettings settings) { List resultInfos = new ArrayList<>(); - resultInfos.addAll(index.getChild().getResultInfos()); - resultInfos.addAll(preceding.getChild().getResultInfos()); + resultInfos.addAll(index.getChild().getResultInfos(settings)); + resultInfos.addAll(preceding.getChild().getResultInfos(settings)); return resultInfos; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java index 826c7a6133..fde3a9ad80 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowRenderer.java @@ -14,8 +14,8 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.models.types.ResultType; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.util.Preconditions; import org.apache.arrow.vector.BitVector; @@ -232,30 +232,28 @@ public static RowConsumer[] generateWriterPipeline(VectorSchemaRoot root, int ve RowConsumer[] builder = new RowConsumer[numVectors]; for ( - int vecI = vectorOffset; - (vecI < root.getFieldVectors().size()) && (vecI < vectorOffset + numVectors); - vecI++ - ) { + int vecI = vectorOffset; + (vecI < root.getFieldVectors().size()) && (vecI < vectorOffset + numVectors); + vecI++ + ) { final int pos = vecI - vectorOffset; final FieldVector vector = root.getVector(vecI); final ResultInfo resultInfo = resultInfos.get(pos); - builder[pos] = - generateVectorFiller(pos, vector, settings, resultInfo.getType()); + builder[pos] = generateVectorFiller(pos, vector, settings, resultInfo.getPrinter()); } return builder; } - private static RowConsumer generateVectorFiller(int pos, ValueVector vector, final PrintSettings settings, ResultType resultType) { - //TODO When Pattern-matching lands, clean this up. (Think Java 12?) - if (vector instanceof IntVector) { - return intVectorFiller((IntVector) vector, (line) -> (Integer) line[pos]); + private static RowConsumer generateVectorFiller(int pos, ValueVector vector, final PrintSettings settings, ResultPrinters.Printer printer) { + if (vector instanceof IntVector intVector) { + return intVectorFiller(intVector, (line) -> (Integer) line[pos]); } - if (vector instanceof VarCharVector) { + if (vector instanceof VarCharVector varCharVector) { return varCharVectorFiller( - (VarCharVector) vector, + varCharVector, (line) -> { // This is a bit clunky at the moment, since this lambda is executed for each textual value // in the result, but it should be okay for now. This code moves as soon shards deliver themselves @@ -265,32 +263,34 @@ private static RowConsumer generateVectorFiller(int pos, ValueVector vector, fin // If there is no value, we don't want to have it displayed as an empty string (see next if) return null; } - return resultType.printNullable(settings, line[pos]); + // We reference the printer directly, + return printer.print(line[pos]); }); } - if (vector instanceof BitVector) { - return bitVectorFiller((BitVector) vector, (line) -> (Boolean) line[pos]); + if (vector instanceof BitVector bitVector) { + return bitVectorFiller(bitVector, (line) -> (Boolean) line[pos]); } - if (vector instanceof Float4Vector) { - return float4VectorFiller((Float4Vector) vector, (line) -> (Number) line[pos]); + if (vector instanceof Float4Vector float4Vector) { + return float4VectorFiller(float4Vector, (line) -> (Number) line[pos]); } - if (vector instanceof Float8Vector) { - return float8VectorFiller((Float8Vector) vector, (line) -> (Number) line[pos]); + if (vector instanceof Float8Vector float8Vector) { + return float8VectorFiller(float8Vector, (line) -> (Number) line[pos]); } - if (vector instanceof DateDayVector) { - return dateDayVectorFiller((DateDayVector) vector, (line) -> (Number) line[pos]); + if (vector instanceof DateDayVector dateDayVector) { + return dateDayVectorFiller(dateDayVector, (line) -> (Number) line[pos]); } if (vector instanceof StructVector structVector) { List nestedVectors = structVector.getPrimitiveVectors(); RowConsumer [] nestedConsumers = new RowConsumer[nestedVectors.size()]; + for (int i = 0; i < nestedVectors.size(); i++) { - nestedConsumers[i] = generateVectorFiller(i, nestedVectors.get(i), settings, resultType); + nestedConsumers[i] = generateVectorFiller(i, nestedVectors.get(i), settings, printer); } return structVectorFiller(structVector, nestedConsumers, (line) -> (List) line[pos]); } @@ -300,7 +300,7 @@ private static RowConsumer generateVectorFiller(int pos, ValueVector vector, fin ValueVector nestedVector = listVector.getDataVector(); // pos = 0 is a workaround for now - return listVectorFiller(listVector, generateVectorFiller(0, nestedVector, settings, ((ResultType.ListT) (resultType)).getElementType()), (line) -> (List) line[pos]); + return listVectorFiller(listVector, generateVectorFiller(0, nestedVector, settings, ((ResultPrinters.ListPrinter) printer).elementPrinter()), (line) -> (List) line[pos]); } throw new IllegalArgumentException("Unsupported vector type " + vector); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java index 12458db1cc..7d2bc1ffcd 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ArrowUtil.java @@ -2,7 +2,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.function.BiFunction; import java.util.stream.Collectors; @@ -22,17 +21,23 @@ @UtilityClass public class ArrowUtil { - public final static RootAllocator ROOT_ALLOCATOR = new RootAllocator(); + public static final RootAllocator ROOT_ALLOCATOR = new RootAllocator(); + + private BiFunction fieldFor(ResultType type) { + if (type instanceof ResultType.ListT) { + return ArrowUtil::listField; + } + + return switch (((ResultType.Primitive) type)) { + case BOOLEAN -> ArrowUtil::boolField; + case INTEGER, MONEY -> ArrowUtil::integerField; + case NUMERIC -> ArrowUtil::floatField; + case DATE -> ArrowUtil::dateField; + case DATE_RANGE -> ArrowUtil::dateRangeField; + case STRING -> ArrowUtil::stringField; + }; + } - private final static Map, BiFunction> FIELD_MAP = Map.of( - ResultType.BooleanT.class, ArrowUtil::boolField, - ResultType.IntegerT.class, ArrowUtil::integerField, - ResultType.NumericT.class, ArrowUtil::floatField, - ResultType.DateT.class, ArrowUtil::dateField, - ResultType.DateRangeT.class, ArrowUtil::dateRangeField, - ResultType.MoneyT.class, ArrowUtil::integerField, - ResultType.ListT.class, ArrowUtil::listField - ); private static Field stringField(ResultInfo info, @NonNull String uniqueName) { return new Field(uniqueName, FieldType.nullable(new ArrowType.Utf8()), null); @@ -70,7 +75,7 @@ private static Field listField(ResultInfo info, @NonNull String uniqueName) { } final ResultType elementType = ((ResultType.ListT) info.getType()).getElementType(); - BiFunction nestedFieldCreator = FIELD_MAP.getOrDefault(elementType.getClass(), ArrowUtil::stringField); + BiFunction nestedFieldCreator = fieldFor(elementType); final Field nestedField = nestedFieldCreator.apply(info, uniqueName); return new Field( uniqueName, @@ -88,7 +93,7 @@ private static Field listField(ResultInfo info, @NonNull String uniqueName) { */ public Field createField(ResultInfo info, UniqueNamer collector) { // Fallback to string field if type is not explicitly registered - BiFunction fieldCreator = FIELD_MAP.getOrDefault(info.getType().getClass(), ArrowUtil::stringField); + BiFunction fieldCreator = fieldFor(info.getType()); return fieldCreator.apply(info, collector.getUniqueName(info)); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 48b8f78ad7..7e2c02ada1 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -95,18 +95,12 @@ public static Response getArrow IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - PrintSettings settings = new PrintSettings( - pretty, - locale, - namespace, - config, - idPrinter::createId - ); + PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); // Collect ResultInfos for id columns and result columns - final List resultInfosId = config.getIdColumns().getIdResultInfos(); - final List resultInfosExec = exec.getResultInfos(); + final List resultInfosId = config.getIdColumns().getIdResultInfos(settings); + final List resultInfosExec = exec.getResultInfos(settings); StreamingOutput out = output -> { try { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java index 3eca172f01..295df2b14a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/CsvRenderer.java @@ -38,16 +38,16 @@ private void createCSVBody(PrintSettings cfg, List infos, Stream res .getValue() .streamValues() - .forEach(result -> printLine(cfg, res.getKey(), infos, result))); + .forEach(result -> printLine(res.getKey(), infos, result))); } - public void printLine(PrintSettings cfg, EntityPrintId entity, List infos, Object[] value) { + public void printLine(EntityPrintId entity, List infos, Object[] value) { // Cast here to Object[] so it is clear to intellij that the varargs call is intended writer.addValues((Object[]) entity.getExternalId()); try { for (int i = 0; i < infos.size(); i++) { - writer.addValue(infos.get(i).getType().printNullable(cfg, value[i])); + writer.addValue(infos.get(i).printNullable(value[i])); } } catch (Exception e) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 928e07aa54..0eb59cca95 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -57,12 +57,12 @@ public Response createResult(Su // Get the locale extracted by the LocaleFilter final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); final StreamingOutput out = os -> { try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(os, charset))) { final CsvRenderer renderer = new CsvRenderer(config.getCsv().createWriter(writer), settings); - renderer.toCSV(config.getIdColumns().getIdResultInfos(), exec.getResultInfos(), exec.streamResults(limit)); + renderer.toCSV(config.getIdColumns().getIdResultInfos(settings), exec.getResultInfos(settings), exec.streamResults(limit)); } catch (EofException e) { log.trace("User canceled download"); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java index b38c9d4735..71e10f3ae1 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ExcelRenderer.java @@ -43,13 +43,22 @@ public class ExcelRenderer { public static final int MAX_LINES = 1_048_576; - private static final Map, TypeWriter> TYPE_WRITER_MAP = Map.of( - ResultType.DateT.class, ExcelRenderer::writeDateCell, - ResultType.IntegerT.class, ExcelRenderer::writeIntegerCell, - ResultType.MoneyT.class, ExcelRenderer::writeMoneyCell, - ResultType.NumericT.class, ExcelRenderer::writeNumericCell, - ResultType.BooleanT.class, ExcelRenderer::writeBooleanCell - ); + private static TypeWriter writer(ResultType type) { + if(!(type instanceof ResultType.Primitive)){ + //Excel cannot handle complex types so we just toString them. + return (info, settings, cell, value, styles) -> writeStringCell(info, cell, value, styles); + } + + return switch (((ResultType.Primitive) type)) { + case BOOLEAN -> (info, settings, cell, value, styles) -> writeBooleanCell(info, cell, value, styles); + case INTEGER -> ExcelRenderer::writeIntegerCell; + case MONEY -> ExcelRenderer::writeMoneyCell; + case NUMERIC -> ExcelRenderer::writeNumericCell; + case DATE -> ExcelRenderer::writeDateCell; + default -> (info, settings, cell, value, styles) -> writeStringCell(info, cell, value, styles); + }; + } + public static final int CHARACTER_WIDTH_DIVISOR = 256; public static final int AUTOFILTER_SPACE_WIDTH = 3; @@ -71,11 +80,9 @@ private interface TypeWriter { void writeCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles); } - public void renderToStream( - List idHeaders, - E exec, - OutputStream outputStream, OptionalLong limit) throws IOException { - final List resultInfosExec = exec.getResultInfos(); + public void renderToStream(List idHeaders, E exec, OutputStream outputStream, OptionalLong limit, PrintSettings printSettings) + throws IOException { + final List resultInfosExec = exec.getResultInfos(printSettings); setMetaData(exec); @@ -258,7 +265,7 @@ private int writeRowsForEntity( } // Fallback to string if type is not explicitly registered - TypeWriter typeWriter = TYPE_WRITER_MAP.getOrDefault(resultInfo.getType().getClass(), ExcelRenderer::writeStringCell); + TypeWriter typeWriter = writer(resultInfo.getType()); typeWriter.writeCell(resultInfo, settings, dataCell, resultValue, styles); } @@ -296,24 +303,23 @@ private void setColumnWidthsAndUntrack(SXSSFSheet sheet) { } // Type specific cell writers - private static void writeStringCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { + private static void writeStringCell(ResultInfo info, Cell cell, Object value, Map styles) { cell.setCellValue( - info.getType().printNullable( - settings, + info.printNullable( value )); } /** - * This writer is only used on Columns with the result type {@link ResultType.BooleanT}, not on complex types such as `LIST[BOOLEAN]`, + * This writer is only used on Columns with the result type {@link ResultType.Primitive#BOOLEAN}, not on complex types such as `LIST[BOOLEAN]`, * because MS Excel can only represent those as strings */ - private static void writeBooleanCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { + private static void writeBooleanCell(ResultInfo info, Cell cell, Object value, Map styles) { if (value instanceof Boolean aBoolean) { cell.setCellValue(aBoolean); return; } - cell.setCellValue(info.getType().printNullable(settings, value)); + cell.setCellValue(info.printNullable(value)); } private static void writeDateCell(ResultInfo info, PrintSettings settings, Cell cell, Object value, Map styles) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index cf726f4a12..89194e75c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -52,12 +52,12 @@ public Response createResult(Su final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, conqueryConfig.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, conqueryConfig, idPrinter::createId, null); final ExcelRenderer excelRenderer = new ExcelRenderer(excelConfig, settings); final StreamingOutput out = output -> { - excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(), exec, output, limit); + excelRenderer.renderToStream(conqueryConfig.getIdColumns().getIdResultInfos(settings), exec, output, limit, settings); log.trace("FINISHED downloading {}", exec.getId()); }; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java index 95053843f3..8dcad2291d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/EntityResultWriteSupport.java @@ -9,9 +9,11 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; import com.bakdata.conquery.models.types.ResultType; +import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.apache.arrow.vector.types.pojo.Schema; @@ -118,15 +120,15 @@ public static MessageType generateSchema( } - @RequiredArgsConstructor + @Data private static class StringTColumnConsumer implements ColumnConsumer { - private final ResultType.StringT resultType; + private final ResultPrinters.Printer printer; private final PrintSettings printSettings; @Override public void accept(RecordConsumer recordConsumer, Object o) { - final String printValue = resultType.printNullable(printSettings, o); + final String printValue = getPrinter().print(o); recordConsumer.addBinary(Binary.fromString(printValue)); } } @@ -190,13 +192,11 @@ public void accept(RecordConsumer recordConsumer, Object o) { @RequiredArgsConstructor private static class ListTColumnConsumer implements ColumnConsumer { - private final ResultType.ListT resultType; + private final ColumnConsumer elementConsumer; private final PrintSettings printSettings; @Override public void accept(RecordConsumer recordConsumer, Object o) { - final ResultType elementType = resultType.getElementType(); - final ColumnConsumer elementConsumer = getForResultType(elementType, printSettings); List list = (List) o; @@ -225,41 +225,27 @@ public void accept(RecordConsumer recordConsumer, Object o) { private static List generateColumnConsumers(List idHeaders, List resultInfos, PrintSettings printSettings) { final List consumers = new ArrayList<>(); for (ResultInfo idHeader : idHeaders) { - consumers.add(getForResultType(idHeader.getType(), printSettings)); + consumers.add(getForResultType(idHeader.getType(), idHeader.getPrinter(), printSettings)); } for (ResultInfo resultInfo : resultInfos) { - consumers.add(getForResultType(resultInfo.getType(), printSettings)); + consumers.add(getForResultType(resultInfo.getType(), resultInfo.getPrinter(), printSettings)); } return consumers; } - private static ColumnConsumer getForResultType(ResultType resultType, PrintSettings printSettings) { - if (resultType instanceof ResultType.StringT) { - return new StringTColumnConsumer((ResultType.StringT) resultType, printSettings); - } - else if (resultType instanceof ResultType.BooleanT) { - return new BooleanTColumnConsumer(); - } - else if (resultType instanceof ResultType.IntegerT) { - return new IntegerTColumnConsumer(); - } - else if (resultType instanceof ResultType.NumericT) { - return new NumericTColumnConsumer(); - } - else if (resultType instanceof ResultType.MoneyT) { - return new IntegerTColumnConsumer(); - } - else if (resultType instanceof ResultType.DateT) { - return new IntegerTColumnConsumer(); - } - else if (resultType instanceof ResultType.DateRangeT) { - return new DateRangeTColumnConsumer(); - } - else if (resultType instanceof ResultType.ListT) { - return new ListTColumnConsumer((ResultType.ListT) resultType, printSettings); + private static ColumnConsumer getForResultType(ResultType resultType, ResultPrinters.Printer printer, PrintSettings printSettings) { + + if (resultType instanceof ResultType.ListT listT) { + return new ListTColumnConsumer(getForResultType(listT.getElementType(), ((ResultPrinters.ListPrinter) printer).elementPrinter(), printSettings), printSettings); } - throw new IllegalArgumentException(String.format("Cannot support ResultType %s", resultType)); + return switch (((ResultType.Primitive) resultType)) { + case BOOLEAN -> new BooleanTColumnConsumer(); + case INTEGER, DATE, MONEY -> new IntegerTColumnConsumer(); + case NUMERIC -> new NumericTColumnConsumer(); + case DATE_RANGE -> new DateRangeTColumnConsumer(); + case STRING -> new StringTColumnConsumer(printer, printSettings); + }; } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index 185aa1157a..6dfb0e61e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -49,24 +49,18 @@ public Response createResultFile(Subject subject, ManagedExecution exec, boolean final Namespace namespace = datasetRegistry.get(dataset.getId()); - IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); + final IdPrinter idPrinter = IdColumnUtil.getIdPrinter(subject, exec, namespace, config.getIdColumns().getIds()); final Locale locale = I18n.LOCALE.get(); - PrintSettings settings = new PrintSettings( - pretty, - locale, - namespace, - config, - idPrinter::createId - ); + final PrintSettings settings = new PrintSettings(pretty, locale, namespace, config, idPrinter::createId, null); - StreamingOutput out = output -> { + final StreamingOutput out = output -> { final SingleTableResult singleTableResult = (SingleTableResult) exec; ParquetRenderer.writeToStream( output, - config.getIdColumns().getIdResultInfos(), - singleTableResult.getResultInfos(), + config.getIdColumns().getIdResultInfos(settings), + singleTableResult.getResultInfos(settings), settings, singleTableResult.streamResults(limit) ); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 5f44e2fb75..79581354d0 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -53,7 +53,7 @@ public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaSto ResultSetProcessor resultSetProcessor = ResultSetProcessorFactory.create(config, sqlDialect); SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, resultSetProcessor); NodeConversions nodeConversions = new NodeConversions(idColumns, sqlDialect, dslContext, databaseConfig, sqlExecutionService); - SqlConverter sqlConverter = new SqlConverter(nodeConversions); + SqlConverter sqlConverter = new SqlConverter(nodeConversions, config); ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage); SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); SqlEntityResolver sqlEntityResolver = new SqlEntityResolver(idColumns, dslContext, sqlDialect, sqlExecutionService); diff --git a/backend/src/main/java/com/bakdata/conquery/models/common/LocalizedToString.java b/backend/src/main/java/com/bakdata/conquery/models/common/LocalizedToString.java new file mode 100644 index 0000000000..e61a29c76c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/common/LocalizedToString.java @@ -0,0 +1,7 @@ +package com.bakdata.conquery.models.common; + +import java.util.Locale; + +public interface LocalizedToString { + String toString(Locale locale); +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 68fcbc2efa..633ffbc9f4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -7,21 +7,22 @@ import java.util.Collections; import java.util.List; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotNull; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.UriBuilder; - import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; import com.bakdata.conquery.io.result.excel.ResultExcelProcessor; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.i18n.I18n; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.resources.api.ResultExcelResource; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.jersey.DropwizardResourceConfig; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.UriBuilder; import lombok.Data; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -72,7 +73,13 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil return Collections.emptyList(); } - final int columnCount = singleExecution.getResultInfos().size() + idColumnsCount; + + final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), exec.getNamespace(), exec.getConfig(), null, null); + + // Save id column count to later check if xlsx dimensions are feasible + idColumnsCount = exec.getConfig().getIdColumns().getIdResultInfos(printSettings).size(); + + final int columnCount = singleExecution.getResultInfos(printSettings).size() + idColumnsCount; final int maxColumnCount = SpreadsheetVersion.EXCEL2007.getMaxColumns(); if (columnCount > maxColumnCount) { @@ -90,8 +97,6 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuil @Override public void registerResultResource(DropwizardResourceConfig environment, ManagerNode manager) { - // Save id column count to later check if xlsx dimensions are feasible - idColumnsCount = manager.getConfig().getIdColumns().getIdResultInfos().size(); // inject required services environment.register(new AbstractBinder() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java index 9c8d1202f6..ebac0b4a46 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IdColumnConfig.java @@ -12,8 +12,10 @@ import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.query.resultinfo.LocalizedDefaultResultInfo; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.FixedLabelResultInfo; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.printers.ResultPrinters; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -121,28 +123,21 @@ public boolean isExactlyOnePseudo() { * @return */ @JsonIgnore - public List getIdResultInfos() { - return ids.stream() - .filter(ColumnConfig::isPrint) - .map(col -> new LocalizedDefaultResultInfo( - locale -> { - final Map label = col.getLabel(); - // Get the label for the locale, - // fall back to any label if there is exactly one defined, - // then fall back to the field name. - return label.getOrDefault( - locale, - // fall backs - label.size() == 1 ? - label.values().stream().collect(MoreCollectors.onlyElement()) : - col.getField() - ); - }, - locale -> col.getField(), - ResultType.StringT.getINSTANCE(), - Set.of(new SemanticType.IdT(col.getName())) - )) - .collect(Collectors.toUnmodifiableList()); + public List getIdResultInfos(PrintSettings printSettings) { + return ids.stream().filter(ColumnConfig::isPrint).map(col -> { + final Map labels = col.getLabel(); + // Get the label for the locale, + // fall back to any label if there is exactly one defined, + // then fall back to the field name. + final String label = Objects.requireNonNullElse(labels.getOrDefault( + printSettings.getLocale(), + // fall backs + labels.size() == 1 ? labels.values().stream().collect(MoreCollectors.onlyElement()) : col.getField() + ), col.getField()); + + //TODO we can now hook our anonymizers into this + return new FixedLabelResultInfo(label, label, ResultType.Primitive.STRING, Set.of(new SemanticType.IdT(col.getName())), printSettings, ResultPrinters.printerFor(ResultType.Primitive.STRING, printSettings)); + }).collect(Collectors.toUnmodifiableList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java index f50f0b2375..4fcb77d2ae 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java @@ -4,7 +4,6 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; -import jakarta.validation.Valid; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; @@ -15,9 +14,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; -import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.queryplan.QPNode; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; @@ -28,6 +25,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonTypeInfo; +import jakarta.validation.Valid; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; @@ -58,14 +56,6 @@ public abstract class Concept extends ConceptElemen @NsIdRef private Dataset dataset; - /** - * rawValue is expected to be an Integer, expressing a localId for {@link TreeConcept#getElementByLocalId(int)}. - * - *

- * If {@link PrintSettings#isPrettyPrint()} is false, {@link ConceptElement#getId()} is used to print. - */ - public abstract String printConceptLocalId(Object rawValue, PrintSettings printSettings); - public List