Split out DB migration command, service and CLI fixes #1153 #1186

pom.xml

@@ -51,7 +51,7 @@
         <quartz.version>2.3.2</quartz.version>
         <cronutils.version>9.2.1</cronutils.version>
         <httpcomponents.version>4.5.13</httpcomponents.version>
-        <nzymeplugin.version>2.0.21</nzymeplugin.version>
+        <nzymeplugin.version>2.0.24</nzymeplugin.version>
         <bouncycastle.version>1.75</bouncycastle.version>
         <oshi.version>6.4.0</oshi.version>
         <commonsnet.version>3.9.0</commonsnet.version>
@@ -567,7 +567,7 @@
                                     <type>file</type>
                                     <mapper>
                                         <type>perm</type>
-                                        <prefix>/usr/share/nzyme/bin</prefix>
+                                        <prefix>/usr/local/bin</prefix>
                                         <filemode>740</filemode>
                                         <user>root</user>
                                         <group>root</group>
@@ -607,7 +607,18 @@
                                 </data>
 
                                 <data>
-                                    <src>src/main/resources/log4j2-debian.xml</src>
+                                    <src>src/main/resources/log4j2-production.xml</src>
+                                    <type>file</type>
+                                    <mapper>
+                                        <type>perm</type>
+                                        <prefix>/etc/nzyme</prefix>
+                                        <user>root</user>
+                                        <group>root</group>
+                                    </mapper>
+                                </data>
+
+                                <data>
+                                    <src>src/main/resources/log4j2-production-foreground.xml</src>
                                     <type>file</type>
                                     <mapper>
                                         <type>perm</type>

src/deb/bin/nzyme

@@ -3,7 +3,37 @@
 set -e
 
 if [ -f "/etc/default/nzyme" ]; then
-    . "/etc/default/nzyme"
+  . "/etc/default/nzyme"
 fi
 
-${JAVA:=/usr/bin/java} $NZYME_JAVA_OPTS -jar -Dlog4j.configurationFile=file:///etc/nzyme/log4j2-debian.xml /usr/share/nzyme/nzyme.jar -c /etc/nzyme/nzyme.conf
+# Default to running the main application if no specific command is provided.
+COMMAND="main"
+ARGS=""
+
+# Parse arguments to detect commands and store non-command flags in ARGS.
+for arg in "$@"; do
+  case $arg in
+    --migrate-database)
+      COMMAND="migrate-database"
+      ;;
+    --bootstrap-test)
+      COMMAND="bootstrap-test"
+      ;;
+    *)
+      ARGS="$ARGS $arg"
+      ;;
+  esac
+done
+
+# Execute the appropriate command with all arguments passed through.
+case $COMMAND in
+  migrate-database)
+    ${JAVA:=/usr/bin/java} $NZYME_JAVA_OPTS -jar -Dlog4j.configurationFile=file:///etc/nzyme/log4j2-production-foreground.xml /usr/share/nzyme/nzyme.jar -c /etc/nzyme/nzyme.conf --migrate-database $ARGS
+    ;;
+  bootstrap-test)
+    ${JAVA:=/usr/bin/java} $NZYME_JAVA_OPTS -jar -Dlog4j.configurationFile=file:///etc/nzyme/log4j2-production-foreground.xml /usr/share/nzyme/nzyme.jar -c /etc/nzyme/nzyme.conf --bootstrap-test $ARGS
+    ;;
+  main)
+    ${JAVA:=/usr/bin/java} $NZYME_JAVA_OPTS -jar -Dlog4j.configurationFile=file:///etc/nzyme/log4j2-production.xml /usr/share/nzyme/nzyme.jar -c /etc/nzyme/nzyme.conf $ARGS
+    ;;
+esac

src/deb/systemd/nzyme.service

@@ -1,6 +1,6 @@
 [Unit]
-Description=Nzyme
-Documentation=https://github.com/lennartkoopmann/nzyme
+Description=nzyme
+Documentation=https://github.com/nzymedefense/nzyme
 Wants=network-online.target
 After=network-online.target
 
@@ -12,7 +12,7 @@ User=root
 Group=root
 LimitNOFILE=64000
 
-ExecStart=/usr/share/nzyme/bin/nzyme
+ExecStart=/usr/local/bin/nzyme
 
 # When a JVM receives a SIGTERM signal it exits with 143.
 SuccessExitStatus=143

src/main/java/app/nzyme/core/Main.java

@@ -111,18 +111,22 @@ public static void main(String[] argv) {
             LOG.error("Could not read configuration file.", e);
             System.exit(FAILURE);
         }
-
-        LOG.info("Performance Configuration: {}", nodeConfiguration.performance());
 
         // Database.
         DatabaseImpl database = new DatabaseImpl(nodeConfiguration);
         try {
-            database.initializeAndMigrate();
+            if (cliArguments.isMigrateDatabaseMode()) {
+                LOG.info("Database migration requested.");
+                database.migrate();
+                System.exit(SUCCESS);
+            }
+            database.initialize();
         } catch (LiquibaseException e) {
-            LOG.fatal("Error during database initialization and migration.", e);
+            LOG.fatal("Database error.", e);
             System.exit(FAILURE);
         }
 
+        LOG.info("Performance Configuration: {}", nodeConfiguration.performance());
         NzymeNode nzyme = new NzymeNodeImpl(baseConfiguration, nodeConfiguration, database);
 
         try {

src/main/java/app/nzyme/core/MockNzyme.java

@@ -128,7 +128,8 @@ public MockNzyme(int taskAndMessagePollInterval, TimeUnit taskAndMessagePollInte
         this.database = new DatabaseImpl(configuration);
 
         try {
-            this.database.initializeAndMigrate();
+            this.database.migrate();
+            this.database.initialize();
         } catch (LiquibaseException e) {
             throw new RuntimeException(e);
         }

src/main/java/app/nzyme/core/configuration/CLIArguments.java

@@ -24,6 +24,9 @@ public class CLIArguments {
     @Parameter(names={"--config-file", "-c"}, required = true)
     private String configFilePath;
 
+    @Parameter(names={"--migrate-database", "-m"})
+    private boolean migrateDatabaseMode;
+
     @Parameter(names={"--debug", "-d"})
     private boolean debugMode;
 
@@ -40,6 +43,10 @@ public String getConfigFilePath() {
         return configFilePath;
     }
 
+    public boolean isMigrateDatabaseMode() {
+        return migrateDatabaseMode;
+    }
+
     public boolean isDebugMode() {
         return debugMode;
     }

src/main/java/app/nzyme/core/database/DatabaseImpl.java

@@ -81,7 +81,7 @@ public DatabaseImpl(NodeConfiguration configuration) {
         this.configuration = configuration;
     }
 
-    public void initializeAndMigrate() throws LiquibaseException {
+    public void initialize() throws LiquibaseException {
         // TODO use reflection here at some point.
         this.jdbi = Jdbi.create("jdbc:" + configuration.databasePath())
                 .installPlugin(new PostgresPlugin())
@@ -180,10 +180,8 @@ public void logAfterExecution(StatementContext context) {
             });
         }
 
-        // Run migrations against underlying JDBC connection.
-        JdbcConnection connection;
-
         // Try to establish connection, retry if connection fails.
+        JdbcConnection connection;
         while (true) {
             try {
                 connection = new JdbcConnection(jdbi.open().getConnection());
@@ -203,7 +201,10 @@ public void logAfterExecution(StatementContext context) {
         try {
             liquibase.database.Database database = DatabaseFactory.getInstance().findCorrectDatabaseImplementation(connection);
             liquibase = new liquibase.Liquibase("db/migrations.xml", new ClassLoaderResourceAccessor(), database);
-            liquibase.update(new Contexts(), new LabelExpression());
+
+            if (!liquibase.listUnrunChangeSets(new Contexts(), new LabelExpression()).isEmpty()) {
+                throw new RuntimeException("There are un-run database changesets. Please run migrations.");
+            }
         } finally {
             if (liquibase != null) {
                 liquibase.close();
@@ -214,6 +215,48 @@ public void logAfterExecution(StatementContext context) {
         }
     }
 
+    public void migrate() throws LiquibaseException {
+        Jdbi migrationJdbi = Jdbi.create("jdbc:" + configuration.databasePath())
+                .installPlugin(new PostgresPlugin());
+
+        JdbcConnection migrationConnection;
+
+        // Try to establish connection, retry if connection fails.
+        while (true) {
+            try {
+                migrationConnection = new JdbcConnection(migrationJdbi.open().getConnection());
+                break;
+            } catch (ConnectionException e) {
+                LOG.warn("Could not connect to PostgreSQL. Retrying.", e);
+
+                try {
+                    Thread.sleep(5000);
+                } catch (InterruptedException ex) {
+                    throw new RuntimeException(ex);
+                }
+            }
+        }
+
+        Liquibase liquibase = null;
+        try {
+            LOG.info("Running database migrations.");
+            liquibase.database.Database database = DatabaseFactory.getInstance()
+                    .findCorrectDatabaseImplementation(migrationConnection);
+            liquibase = new liquibase.Liquibase(
+                    "db/migrations.xml", new ClassLoaderResourceAccessor(), database
+            );
+            liquibase.update(new Contexts(), new LabelExpression());
+            LOG.info("All database migrations complete.");
+        } finally {
+            if (liquibase != null) {
+                liquibase.close();
+            }
+            if (!migrationConnection.isClosed()) {
+                migrationConnection.close();
+            }
+        }
+    }
+
     public long getTotalSize() {
         return withHandle(handle ->
                 handle.createQuery("SELECT pg_database_size(current_database())")

src/main/resources/log4j2-production-foreground.xml

@@ -0,0 +1,33 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<Configuration status="WARN">
+    <Appenders>
+        <Console name="Console" target="SYSTEM_OUT">
+            <PatternLayout pattern="%d{yyyy-MM-dd'T'HH:mm:ss.SSSZ} [%t] %-5level %logger{36} - %msg%n" />
+        </Console>
+    </Appenders>
+    <Loggers>
+        <Root level="info">
+            <AppenderRef ref="Console" />
+        </Root>
+
+        <Logger name="org.pcap4j" level="error">
+            <AppenderRef ref="Console" />
+        </Logger>
+
+        <Logger name="liquibase" level="warn">
+            <AppenderRef ref="Console" />
+        </Logger>
+
+        <Logger name="org.quartz" level="warn">
+            <AppenderRef ref="Console" />
+        </Logger>
+
+        <Logger name="com.mchange" level="warn">
+            <AppenderRef ref="Console" />
+        </Logger>
+
+        <Logger name="org.glassfish.grizzly" level="error">
+            <AppenderRef ref="Console" />
+        </Logger>
+    </Loggers>
+</Configuration>