Merge branch 'develop' into master

.github/workflows/codeql-analysis.yml

@@ -44,10 +44,12 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v2
 
-    - name: set java 17
-      uses: actions/setup-java@v1
+    - name: Set up JDK
+      uses: actions/setup-java@v4
       with:
-        java-version: 17
+        distribution: 'temurin'
+        java-version: '17'
+        overwrite-settings: false
 
       # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/test_cypress.yml

@@ -17,26 +17,28 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18
 
       - name: Set up JDK
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
-          java-version: 17
+          distribution: 'temurin'
+          java-version: '17'
+          overwrite-settings: false
 
       - name: Build Backend
         run: ./scripts/build_backend_no_version.sh
@@ -53,14 +55,16 @@ jobs:
           start: bash ./scripts/run_e2e_all.sh
           wait-on: "http://localhost:8000"
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: cypress-screenshots
           path: cypress/screenshots
+          if-no-files-found: ignore
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: cypress-videos
           path: cypress/videos
+          if-no-files-found: ignore

backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java

@@ -9,7 +9,7 @@
 import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider;
 import com.bakdata.conquery.metrics.ActiveUsersFilter;
 import com.bakdata.conquery.models.auth.basic.JWTokenHandler;
-import com.bakdata.conquery.models.auth.web.DefaultAuthFilter;
+import com.bakdata.conquery.models.auth.web.AuthFilter;
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormConfigProcessor;
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormProcessor;
 import com.bakdata.conquery.resources.ResourcesProvider;
@@ -63,8 +63,8 @@ protected void configure() {
 		 * We use the same instance of the filter for the api servlet and the admin servlet to have a single
 		 * point for authentication.
 		 */
-		jersey.register(DefaultAuthFilter.class);
-		DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, jersey.getResourceConfig());
+		jersey.register(AuthFilter.class);
+		AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, jersey.getResourceConfig());
 
 
 		jersey.register(IdParamConverter.Provider.INSTANCE);

backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java

@@ -2,25 +2,22 @@
 
 import java.net.URL;
 import java.time.LocalDate;
-import java.util.Map;
+import java.util.List;
 
 import com.bakdata.conquery.models.config.FrontendConfig;
 import com.bakdata.conquery.models.config.IdColumnConfig;
 
 /**
  * API Response for the dynamic configuration of the frontend
  *
- * @param version             backend version
- * @param formBackendVersions mapping of form backend ids to their versions (version can be null)
+ * @param versions            mapping of backend and form backend ids to their versions (version can be null)
  * @param currency            currency representation
  * @param queryUpload         identifier specific column configuration for the query upload
  * @param manualUrl           url to a user manual
  * @param contactEmail        typical a mailto-url
  */
 public record FrontendConfiguration(
-		String version,
-
-		Map<String, String> formBackendVersions,
+		List<VersionContainer> versions,
 		FrontendConfig.CurrencyConfig currency,
 		IdColumnConfig queryUpload,
 		URL manualUrl,

backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java

@@ -0,0 +1,12 @@
+package com.bakdata.conquery.apiv1.frontend;
+
+import java.time.ZonedDateTime;
+
+import lombok.NonNull;
+
+public record VersionContainer(
+		@NonNull String name,
+		String version,
+		ZonedDateTime buildTime
+) {
+}

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java

@@ -4,7 +4,6 @@
 import java.math.BigDecimal;
 
 import javax.annotation.Nonnull;
-import jakarta.validation.constraints.NotNull;
 
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterType;
 import com.bakdata.conquery.io.cps.CPSBase;
@@ -32,13 +31,15 @@
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
 import com.fasterxml.jackson.databind.exc.InvalidTypeIdException;
+import jakarta.validation.constraints.NotNull;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import lombok.SneakyThrows;
 import lombok.ToString;
+import org.jooq.Condition;
 
 @Getter
 @Setter
@@ -67,14 +68,19 @@ public FilterNode<?> createNode() {
 	}
 
 	public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, ConceptConversionTables tables) {
-		FilterContext<VALUE> filterContext = new FilterContext<>(ids, value, context, tables);
+		FilterContext<VALUE> filterContext = FilterContext.forConceptConversion(ids, value, context, tables);
 		SqlFilters sqlFilters = filter.convertToSqlFilter(filterContext);
 		if (context.isNegation()) {
 			return new SqlFilters(sqlFilters.getSelects(), sqlFilters.getWhereClauses().negated());
 		}
 		return sqlFilters;
 	}
 
+	public Condition convertForTableExport(SqlIdColumns ids, ConversionContext context) {
+		FilterContext<VALUE> filterContext = FilterContext.forTableExport(ids, value, context);
+		return filter.convertForTableExport(filterContext);
+	}
+
 	@NoArgsConstructor
 	@CPSType(id = FrontendFilterType.Fields.MULTI_SELECT, base = FilterValue.class)
 	@ToString(callSuper = true)

backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java

@@ -29,6 +29,7 @@
 import com.bakdata.conquery.models.messages.network.specific.AddShardNode;
 import com.bakdata.conquery.models.messages.network.specific.RegisterWorker;
 import com.bakdata.conquery.models.messages.network.specific.UpdateJobManagerStatus;
+import com.bakdata.conquery.models.worker.IdResolveContext;
 import com.bakdata.conquery.models.worker.Worker;
 import com.bakdata.conquery.models.worker.WorkerInformation;
 import com.bakdata.conquery.models.worker.Workers;
@@ -51,6 +52,7 @@
 import org.apache.mina.core.session.IoSession;
 import org.apache.mina.filter.FilterEvent;
 import org.apache.mina.transport.socket.nio.NioSocketConnector;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * This node holds a shard of data (in so called {@link Worker}s for the different datasets in conquery.
@@ -64,6 +66,7 @@ public class ShardNode extends ConqueryCommand implements IoHandler, Managed {
 	public static final String DEFAULT_NAME = "shard-node";
 
 	private NioSocketConnector connector;
+	private ConnectFuture future;
 	private JobManager jobManager;
 	private Validator validator;
 	private ConqueryConfig config;
@@ -88,7 +91,6 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 		this.environment = environment;
 		this.config = config;
 
-		connector = new NioSocketConnector();
 
 		jobManager = new JobManager(getName(), config.isFailOnError());
 		environment.lifecycle().manage(this);
@@ -106,6 +108,7 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 				getConfig().getQueries().getSecondaryIdSubPlanRetention()
 		);
 
+
 		final Collection<WorkerStorage> workerStorages = config.getStorage().discoverWorkerStorages();
 
 
@@ -268,6 +271,8 @@ private static void scheduleIdleLogger(ScheduledExecutorService scheduler, IoSes
 	public void sessionClosed(IoSession session) {
 		setLocation(session);
 		log.info("Disconnected from ManagerNode.");
+
+		scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS);
 	}
 
 	@Override
@@ -291,7 +296,9 @@ public void messageSent(IoSession session, Object message) {
 	@Override
 	public void inputClosed(IoSession session) {
 		setLocation(session);
-		log.info("Session closed.");
+		log.info("Input closed.");
+		session.closeNow();
+		scheduler.schedule(this::disconnectFromCluster, 0, TimeUnit.SECONDS);
 	}
 
 	@Override
@@ -306,24 +313,27 @@ public void start() throws Exception {
 			value.getJobManager().addSlowJob(new SimpleJob("Update Bucket Manager", value.getBucketManager()::fullUpdate));
 		}
 
-		ObjectMapper om = createInternalObjectMapper(View.InternalCommunication.class);
+		scheduler.schedule(this::connectToCluster, 0, TimeUnit.MINUTES);
 
-		BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, validator, om);
-		connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om)));
-		connector.setHandler(this);
-		connector.getSessionConfig().setAll(config.getCluster().getMina());
 
+	}
+
+	private void connectToCluster() {
 		InetSocketAddress address = new InetSocketAddress(
 				config.getCluster().getManagerURL().getHostAddress(),
 				config.getCluster().getPort()
 		);
 
+		disconnectFromCluster();
+
+		connector = getClusterConnector(workers);
+
 		while (true) {
 			try {
 				log.info("Trying to connect to {}", address);
 
 				// Try opening a connection (Note: This fails immediately instead of waiting a minute to try and connect)
-				ConnectFuture future = connector.connect(address);
+				future = connector.connect(address);
 
 				future.awaitUninterruptibly();
 
@@ -339,9 +349,24 @@ public void start() throws Exception {
 			catch (RuntimeIoException e) {
 				log.warn("Failed to connect to {}", address, e);
 			}
+			catch (InterruptedException e) {
+				log.warn("Interrupted while trying to connector to cluster, giving up.", e);
+				break;
+			}
 		}
+	}
 
+	@NotNull
+	private NioSocketConnector getClusterConnector(IdResolveContext workers) {
+		ObjectMapper om = createInternalObjectMapper(View.InternalCommunication.class);
+
+		NioSocketConnector connector = new NioSocketConnector();
 
+		BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, validator, om);
+		connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om)));
+		connector.setHandler(this);
+		connector.getSessionConfig().setAll(config.getCluster().getMina());
+		return connector;
 	}
 
 	@Override
@@ -350,12 +375,23 @@ public void stop() throws Exception {
 
 		workers.stop();
 
+		disconnectFromCluster();
+	}
+
+	private void disconnectFromCluster() {
+		if (future != null) {
+			future.cancel();
+		}
+
 		//after the close command was send
 		if (context != null) {
 			context.awaitClose();
 		}
-		log.info("Connection was closed by ManagerNode");
-		connector.dispose();
+
+		if (connector != null) {
+			log.info("Connection was closed by ManagerNode");
+			connector.dispose();
+		}
 	}
 
 	public boolean isBusy() {

backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java

@@ -7,14 +7,6 @@
 import java.util.UUID;
 import java.util.function.Function;
 
-import jakarta.ws.rs.client.Client;
-import jakarta.ws.rs.client.Entity;
-import jakarta.ws.rs.client.Invocation;
-import jakarta.ws.rs.client.WebTarget;
-import jakarta.ws.rs.core.GenericType;
-import jakarta.ws.rs.core.MediaType;
-import jakarta.ws.rs.core.Response;
-
 import com.bakdata.conquery.apiv1.forms.ExternalForm;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.permissions.Ability;
@@ -24,6 +16,13 @@
 import com.codahale.metrics.health.HealthCheck;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.dropwizard.health.check.http.HttpHealthCheck;
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.client.Invocation;
+import jakarta.ws.rs.client.WebTarget;
+import jakarta.ws.rs.core.GenericType;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -129,8 +128,8 @@ public HealthCheck createHealthCheck() {
 		return new HttpHealthCheck(getHealthTarget.getUri().toString(), client);
 	}
 
-	public String getVersion() {
-		return getVersionTarget.request(MediaType.APPLICATION_JSON_TYPE).get(FormBackendVersion.class).getVersion();
+	public FormBackendVersion getVersion() {
+		return getVersionTarget.request(MediaType.APPLICATION_JSON_TYPE).get(FormBackendVersion.class);
 	}
 
 	public ExternalTaskState cancelTask(UUID taskId) {

backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java

@@ -1,8 +1,10 @@
 package com.bakdata.conquery.io.external.form;
 
-import lombok.Data;
+import java.time.ZonedDateTime;
 
-@Data
-public class FormBackendVersion {
-	private String version;
+
+public record FormBackendVersion(
+		String version,
+		ZonedDateTime buildTime
+) {
 }