Merge branch 'develop' into reintegrate-main

ingef · Dec 12, 2024 · 2341241 · 2341241
2 parents 19e6d2e + 18e1f87
commit 2341241
Show file tree

Hide file tree

Showing 10 changed files with 60 additions and 133 deletions.
diff --git a/backend/pom.xml b/backend/pom.xml
@@ -225,13 +225,9 @@
         <dependency>
             <groupId>org.apache.shiro</groupId>
             <artifactId>shiro-core</artifactId>
-            <version>1.13.0</version>
+            <version>2.0.2</version>
             <exclusions>
                 <!-- All these transitive deps are already bundled in shiro core-->
-                <exclusion>
-                    <groupId>org.apache.shiro</groupId>
-                    <artifactId>shiro-cache</artifactId>
-                </exclusion>
                 <exclusion>
                     <groupId>org.apache.shiro</groupId>
                     <artifactId>shiro-config-ogdl</artifactId>
@@ -252,14 +248,6 @@
                     <groupId>org.apache.shiro</groupId>
                     <artifactId>shiro-config-core</artifactId>
                 </exclusion>
-                <exclusion>
-                    <groupId>org.apache.shiro</groupId>
-                    <artifactId>shiro-event</artifactId>
-                </exclusion>
-                <exclusion>
-                    <groupId>org.apache.shiro</groupId>
-                    <artifactId>shiro-lang</artifactId>
-                </exclusion>
             </exclusions>
         </dependency>
         <dependency>

diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java
@@ -3,7 +3,12 @@
 import jakarta.validation.Validator;
 
 import ch.qos.logback.classic.Level;
-import com.bakdata.conquery.commands.*;
+import com.bakdata.conquery.commands.DistributedStandaloneCommand;
+import com.bakdata.conquery.commands.ManagerNode;
+import com.bakdata.conquery.commands.MigrateCommand;
+import com.bakdata.conquery.commands.PreprocessorCommand;
+import com.bakdata.conquery.commands.RecodeStoreCommand;
+import com.bakdata.conquery.commands.ShardCommand;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
 import com.bakdata.conquery.metrics.prometheus.PrometheusBundle;
@@ -21,7 +26,6 @@
 import io.dropwizard.core.setup.Environment;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
-import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.text.StringSubstitutor;
 import org.glassfish.jersey.internal.inject.AbstractBinder;
@@ -32,13 +36,15 @@
 public class Conquery extends Application<ConqueryConfig> {
 
 	private final String name;
-	@Setter
-	private ManagerNode managerNode;
 
 	public Conquery() {
 		this("Conquery");
 	}
 
+	public static void main(String... args) throws Exception {
+		new Conquery().run(args);
+	}
+
 	@Override
 	public void initialize(Bootstrap<ConqueryConfig> bootstrap) {
 		final ObjectMapper confMapper = bootstrap.getObjectMapper();
@@ -49,7 +55,7 @@ public void initialize(Bootstrap<ConqueryConfig> bootstrap) {
 
 		bootstrap.addCommand(new ShardCommand());
 		bootstrap.addCommand(new PreprocessorCommand());
-		bootstrap.addCommand(new DistributedStandaloneCommand(this));
+		bootstrap.addCommand(new DistributedStandaloneCommand());
 		bootstrap.addCommand(new RecodeStoreCommand());
 		bootstrap.addCommand(new MigrateCommand());
 
@@ -92,17 +98,10 @@ protected Level bootstrapLogLevel() {
 	public void run(ConqueryConfig configuration, Environment environment) throws Exception {
 		ManagerProvider provider = configuration.getSqlConnectorConfig().isEnabled() ?
 								   new LocalManagerProvider() : new ClusterManagerProvider();
-		run(provider.provideManager(configuration, environment));
-	}
+		Manager manager = provider.provideManager(configuration, environment);
 
-	public void run(Manager manager) throws InterruptedException {
-		if (managerNode == null) {
-			managerNode = new ManagerNode();
-		}
-		managerNode.run(manager);
-	}
+		ManagerNode managerNode = new ManagerNode();
 
-	public static void main(String... args) throws Exception {
-		new Conquery().run(args);
+		managerNode.run(manager);
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/DistributedStandaloneCommand.java
@@ -4,14 +4,13 @@
 import java.util.List;
 import java.util.Vector;
 
-import com.bakdata.conquery.Conquery;
 import com.bakdata.conquery.mode.cluster.ClusterManager;
 import com.bakdata.conquery.mode.cluster.ClusterManagerProvider;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.XodusStoreFactory;
+import com.bakdata.conquery.util.commands.NoOpConquery;
 import com.bakdata.conquery.util.io.ConqueryMDC;
 import io.dropwizard.core.cli.ServerCommand;
-import io.dropwizard.core.setup.Bootstrap;
 import io.dropwizard.core.setup.Environment;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -21,68 +20,42 @@
 @Getter
 public class DistributedStandaloneCommand extends ServerCommand<ConqueryConfig> implements StandaloneCommand {
 
-	private final Conquery conquery;
-	private ClusterManager manager;
 	private final ManagerNode managerNode = new ManagerNode();
 	private final List<ShardNode> shardNodes = new Vector<>();
+	private ClusterManager manager;
 
-	// TODO clean up the command structure, so we can use the Environment from EnvironmentCommand
-	private Environment environment;
-
-	public DistributedStandaloneCommand(Conquery conquery) {
-		super(conquery, "standalone", "starts a server and a client at the same time.");
-		this.conquery = conquery;
+	public DistributedStandaloneCommand() {
+		super(new NoOpConquery(), "standalone", "starts a manager node and shard node(s) at the same time in a single JVM.");
 	}
 
-	// this must be overridden so that
 	@Override
-	public void run(Bootstrap<ConqueryConfig> bootstrap, Namespace namespace, ConqueryConfig configuration) throws Exception {
-		environment = new Environment(
-				bootstrap.getApplication().getName(),
-				bootstrap.getObjectMapper(),
-				bootstrap.getValidatorFactory(),
-				bootstrap.getMetricRegistry(),
-				bootstrap.getClassLoader(),
-				bootstrap.getHealthCheckRegistry(),
-				configuration
-		);
-		configuration.getMetricsFactory().configure(environment.lifecycle(), bootstrap.getMetricRegistry());
-		configuration.getServerFactory().configure(environment);
-
-		bootstrap.run(configuration, environment);
-		startStandalone(environment, namespace, configuration);
-	}
+	protected void run(Environment environment, Namespace namespace, ConqueryConfig configuration) throws Exception {
 
-	public void startStandalone(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception {
-		// start ManagerNode
-		ConqueryMDC.setLocation("ManagerNode");
-		log.debug("Starting ManagerNode");
 
-		ConqueryConfig managerConfig = config;
+		ConqueryConfig managerConfig = configuration;
 
-		if (config.getStorage() instanceof XodusStoreFactory) {
-			final Path managerDir = ((XodusStoreFactory) config.getStorage()).getDirectory().resolve("manager");
-			managerConfig = config.withStorage(((XodusStoreFactory) config.getStorage()).withDirectory(managerDir));
+		if (configuration.getStorage() instanceof XodusStoreFactory) {
+			final Path managerDir = ((XodusStoreFactory) configuration.getStorage()).getDirectory().resolve("manager");
+			managerConfig = configuration.withStorage(((XodusStoreFactory) configuration.getStorage()).withDirectory(managerDir));
 		}
 
 		manager = new ClusterManagerProvider().provideManager(managerConfig, environment);
 
-		conquery.setManagerNode(managerNode);
-		conquery.run(manager);
+		managerNode.run(manager);
 
-		for (int id = 0; id < config.getStandalone().getNumberOfShardNodes(); id++) {
+		for (int id = 0; id < configuration.getStandalone().getNumberOfShardNodes(); id++) {
 
 			ShardNode sc = new ShardNode(ShardNode.DEFAULT_NAME + id);
 
 			shardNodes.add(sc);
 
 			ConqueryMDC.setLocation(sc.getName());
 
-			ConqueryConfig clone = config;
+			ConqueryConfig clone = configuration;
 
-			if (config.getStorage() instanceof XodusStoreFactory) {
-				final Path managerDir = ((XodusStoreFactory) config.getStorage()).getDirectory().resolve("shard-node" + id);
-				clone = config.withStorage(((XodusStoreFactory) config.getStorage()).withDirectory(managerDir));
+			if (configuration.getStorage() instanceof XodusStoreFactory) {
+				final Path managerDir = ((XodusStoreFactory) configuration.getStorage()).getDirectory().resolve("shard-node" + id);
+				clone = configuration.withStorage(((XodusStoreFactory) configuration.getStorage()).withDirectory(managerDir));
 			}
 
 			sc.run(clone, environment);
@@ -93,6 +66,6 @@ public void startStandalone(Environment environment, Namespace namespace, Conque
 		ConqueryMDC.setLocation("ManagerNode");
 		log.debug("Starting REST Server");
 		ConqueryMDC.setLocation(null);
-		super.run(environment, namespace, config);
+		super.run(environment, namespace, configuration);
 	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java
@@ -2,25 +2,15 @@
 
 import java.util.List;
 
-import com.bakdata.conquery.Conquery;
 import com.bakdata.conquery.mode.Manager;
-import com.bakdata.conquery.models.config.ConqueryConfig;
-import io.dropwizard.core.setup.Bootstrap;
 import io.dropwizard.core.setup.Environment;
-import net.sourceforge.argparse4j.inf.Namespace;
 
 public interface StandaloneCommand {
 
-	void startStandalone(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception;
-
 	Manager getManager();
 
 	List<ShardNode> getShardNodes();
 
-	void run(Bootstrap<ConqueryConfig> bootstrap, Namespace namespace, ConqueryConfig configuration) throws Exception;
-
-	Conquery getConquery();
-
 	ManagerNode getManagerNode();
 
 	Environment getEnvironment();

diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java
@@ -34,11 +34,12 @@
 import org.apache.shiro.SecurityUtils;
 import org.apache.shiro.authc.pam.FirstSuccessfulStrategy;
 import org.apache.shiro.authc.pam.ModularRealmAuthenticator;
+import org.apache.shiro.lang.util.LifecycleUtils;
 import org.apache.shiro.mgt.DefaultSecurityManager;
 import org.apache.shiro.realm.AuthorizingRealm;
 import org.apache.shiro.realm.Realm;
-import org.apache.shiro.util.LifecycleUtils;
 import org.glassfish.hk2.utilities.binding.AbstractBinder;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * The central class for the initialization of authorization and authentication.
@@ -78,7 +79,7 @@ public final class AuthorizationController implements Managed {
 	@Getter
 	private DropwizardResourceConfig unprotectedAuthAdmin;
 
-	public AuthorizationController(MetaStorage storage, ConqueryConfig config, Environment environment, AdminServlet adminServlet) {
+	public AuthorizationController(@NotNull MetaStorage storage, @NotNull ConqueryConfig config, @NotNull Environment environment, AdminServlet adminServlet) {
 		this.storage = storage;
 		this.config = config;
 		this.environment = environment;

diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java
@@ -41,8 +41,8 @@
 import org.apache.shiro.authc.AuthenticationToken;
 import org.apache.shiro.authc.CredentialsException;
 import org.apache.shiro.authc.IncorrectCredentialsException;
+import org.apache.shiro.lang.util.Destroyable;
 import org.apache.shiro.realm.AuthenticatingRealm;
-import org.apache.shiro.util.Destroyable;
 
 /**
  * This realm stores credentials in a local database ({@link XodusStore}). Upon
@@ -52,7 +52,7 @@
  * authorization related user information that is saved in the
  * {@link MetaStorage}. So adding or removing a user in this realm does
  * not change the {@link MetaStorage}. {@link Conquery} interacts with
- * this realm using the Shiro frame work. However, endusers can interface it
+ * this realm using the Shiro framework. However, endusers can interface it
  * through specific endpoints that are registerd by this realm.
  */
 @Slf4j

diff --git a/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java b/backend/src/main/java/com/bakdata/conquery/tasks/PermissionCleanupTask.java
@@ -133,7 +133,7 @@ public static <E extends IdentifiableImpl<ID> & Owned, ID extends Id<E>> int del
 				if (wpermission.getInstances().size() != 1) {
 					log.trace("Skipping permission {} because it refers to multiple instances.", wpermission);
 				}
-				ID executionId = null;
+				ID executionId;
 				try {
 					executionId = idParser.parse(wpermission.getInstances().iterator().next());
 				}
@@ -144,14 +144,17 @@ public static <E extends IdentifiableImpl<ID> & Owned, ID extends Id<E>> int del
 
 				E execution = instanceStorageExtractor.apply(executionId);
 				if (execution == null) {
-					log.trace("The execution referenced in permission {} does not exist. Skipping permission");
+					log.trace("The execution referenced in permission {} does not exist. Skipping permission", wpermission);
 					continue;
 				}
 
 				if (!user.isOwner(execution)) {
-					log.trace("The user is not owner of the instance. Keeping the permission. User: {}, Owner: {}, Instance: {}, Permission: {}", user.getId(), execution
-																																										.getOwner(), execution
-																																															 .getId(), wpermission);
+					log.trace("The user is not owner of the instance. Keeping the permission. User: {}, Owner: {}, Instance: {}, Permission: {}",
+							  user.getId(),
+							  execution.getOwner(),
+							  execution.getId(),
+							  wpermission
+					);
 					continue;
 				}
 

diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java
@@ -3,7 +3,6 @@
 import java.util.Collections;
 import java.util.List;
 
-import com.bakdata.conquery.Conquery;
 import com.bakdata.conquery.commands.ManagerNode;
 import com.bakdata.conquery.commands.ShardNode;
 import com.bakdata.conquery.commands.StandaloneCommand;
@@ -12,9 +11,9 @@
 import com.bakdata.conquery.mode.local.LocalManagerProvider;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.worker.LocalNamespace;
+import com.bakdata.conquery.util.commands.NoOpConquery;
 import com.bakdata.conquery.util.io.ConqueryMDC;
 import io.dropwizard.core.cli.ServerCommand;
-import io.dropwizard.core.setup.Bootstrap;
 import io.dropwizard.core.setup.Environment;
 import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
@@ -24,27 +23,11 @@
 @Getter
 public class SqlStandaloneCommand extends ServerCommand<ConqueryConfig> implements StandaloneCommand {
 
-	private final Conquery conquery;
-	private ManagerNode managerNode = new ManagerNode();
+	private final ManagerNode managerNode = new ManagerNode();
 	private DelegateManager<LocalNamespace> manager;
-	private Environment environment;
 
-	public SqlStandaloneCommand(Conquery conquery) {
-		super(conquery, "standalone", "starts a sql server and a client at the same time.");
-		this.conquery = conquery;
-	}
-
-	@Override
-	public void startStandalone(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception {
-		ConqueryMDC.setLocation("ManagerNode");
-		log.debug("Starting ManagerNode");
-		this.manager = new LocalManagerProvider(new TestSqlDialectFactory()).provideManager(config, environment);
-		this.conquery.setManagerNode(managerNode);
-		this.conquery.run(manager);
-		// starts the Jersey Server
-		log.debug("Starting REST Server");
-		ConqueryMDC.setLocation(null);
-		super.run(environment, namespace, config);
+	public SqlStandaloneCommand() {
+		super(new NoOpConquery(), "standalone", "starts a sql server and a client at the same time.");
 	}
 
 	@Override
@@ -53,23 +36,14 @@ public List<ShardNode> getShardNodes() {
 	}
 
 	@Override
-	public void run(Bootstrap<ConqueryConfig> bootstrap, Namespace namespace, ConqueryConfig configuration) throws Exception {
-		environment = new Environment(
-				bootstrap.getApplication().getName(),
-				bootstrap.getObjectMapper(),
-				bootstrap.getValidatorFactory(),
-				bootstrap.getMetricRegistry(),
-				bootstrap.getClassLoader(),
-				bootstrap.getHealthCheckRegistry(),
-				configuration
-		);
-		configuration.getMetricsFactory().configure(environment.lifecycle(), bootstrap.getMetricRegistry());
-		configuration.getServerFactory().configure(environment);
-
-		bootstrap.run(configuration, environment);
-		startStandalone(environment, namespace, configuration);
+	protected void run(Environment environment, Namespace namespace, ConqueryConfig configuration) throws Exception {
+		ConqueryMDC.setLocation("ManagerNode");
+		log.debug("Starting ManagerNode");
+		this.manager = new LocalManagerProvider(new TestSqlDialectFactory()).provideManager(configuration, environment);
+		managerNode.run(manager);
+		// starts the Jersey Server
+		log.debug("Starting REST Server");
+		ConqueryMDC.setLocation(null);
+		super.run(environment, namespace, configuration);
 	}
-
-
-
 }
diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/LocalAuthRealmTest.java
@@ -21,7 +21,7 @@
 import org.apache.shiro.authc.BearerToken;
 import org.apache.shiro.authc.CredentialsException;
 import org.apache.shiro.authc.IncorrectCredentialsException;
-import org.apache.shiro.util.LifecycleUtils;
+import org.apache.shiro.lang.util.LifecycleUtils;
 import org.junit.jupiter.api.AfterAll;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeAll;