OZ-631: Fix Ozone Flink Jobs tests following Superset refactoring in Ozone Analytics

.github/workflows/ci.yml

@@ -13,17 +13,20 @@ jobs:
     uses: mekomsolutions/shared-github-workflow/.github/workflows/maven-build-test.yml@main
     with:
       java-version: 11
-      maven-args: -DskipTests
+      use-secrets: true
+      login-docker: true
     secrets:
       NEXUS_USERNAME: ${{ secrets.NEXUS_USERNAME }}
       NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }}
+      DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }}
+      DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }}
 
   spotless:
     needs: validate
     uses: mekomsolutions/shared-github-workflow/.github/workflows/maven-spotless-check.yml@main
     with:
       java-version: 11
-      maven-args: -Pspotless -DskipTests
+      maven-args: -Pspotless
 
   docker-ozone-flink-parquet-export:
     if: ${{ (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'release' }}

pom.xml

@@ -18,6 +18,18 @@
       <url>https://www.mekomsolutions.com</url>
     </developer>
   </developers>
+  <distributionManagement>
+    <repository>
+      <id>mks-nexus-public-releases</id>
+      <name>Mekom Solutions Nexus repo</name>
+      <url>https://nexus.mekomsolutions.net/repository/maven-private-releases/</url>
+    </repository>
+    <snapshotRepository>
+      <id>mks-nexus-private-snapshots</id>
+      <name>Mekom Solutions Nexus repo for snapshots</name>
+      <url>https://nexus.mekomsolutions.net/repository/maven-private-snapshots</url>
+    </snapshotRepository>
+  </distributionManagement>
 
   <properties>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -37,6 +49,7 @@
     <testcontainers.version>1.19.7</testcontainers.version>
     <analytics.queries.version>1.3.0-SNAPSHOT</analytics.queries.version>
     <ozone.version>1.0.0-SNAPSHOT</ozone.version>
+    <ozone.analytics.version>2.5.0-SNAPSHOT</ozone.analytics.version>
     <spring.version>5.3.23</spring.version>
     <mockitoJunitVersion>5.7.0</mockitoJunitVersion>
     <powerMockVersion>2.0.9</powerMockVersion>
@@ -233,6 +246,13 @@
       <type>zip</type>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>com.ozonehis</groupId>
+      <artifactId>ozone-analytics</artifactId>
+      <version>${ozone.analytics.version}</version>
+      <type>zip</type>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.springframework</groupId>
       <artifactId>spring-jdbc</artifactId>
@@ -270,6 +290,11 @@
       <name>Mekom Solutions Nexus repo for snapshots</name>
       <url>https://nexus.mekomsolutions.net/repository/maven-snapshots</url>
     </repository>
+    <repository>
+      <id>mks-nexus-private</id>
+      <name>Mekom Solutions private Nexus repo</name>
+      <url>https://nexus.mekomsolutions.net/repository/maven-private</url>
+    </repository>
   </repositories>
   <pluginRepositories>
     <pluginRepository>
@@ -351,12 +376,15 @@
               <phase>generate-test-resources</phase>
               <configuration>
                 <includeGroupIds>com.ozonehis</includeGroupIds>
-                <includeArtifactIds>analytics-queries,ozone</includeArtifactIds>
+                <includeArtifactIds>analytics-queries,ozone,ozone-analytics</includeArtifactIds>
                 <includeTypes>zip</includeTypes>
-                <includes>run/docker/docker-compose-common.yml,
-                    run/docker/docker-compose-superset.yml,
-                    run/docker/docker-compose-openmrs.yml,
-                    run/docker/docker-compose-odoo.yml,
+                <includes>docker-compose-db.yaml,
+                    docker-compose-migration.yaml,
+                    docker-compose-streaming-common.yaml,
+                    docker-compose-superset.yaml,
+                    /run/docker/docker-compose-common.yml,
+                    /run/docker/docker-compose-openmrs.yml,
+                    /run/docker/docker-compose-odoo.yml,
                     distro/configs/superset/**,
                     distro/data/**,
                     distro/configs/openmrs/properties/**,

src/test/java/com/ozonehis/data/pipelines/BaseJobTest.java

@@ -92,7 +92,9 @@ public abstract class BaseJobTest {
 
     protected static String exportDir;
 
-    protected ComposeContainer composeContainer;
+    protected ComposeContainer ozoneCompose;
+
+    protected ComposeContainer analyticsCompose;
 
     private MiniCluster cluster;
 
@@ -101,6 +103,7 @@ public static void beforeAllSuper() {
         try {
             testDir = Files.createTempDirectory(TEST_DIR).toFile().getAbsolutePath();
             exportDir = testDir + "/" + EXPORT_DIR_NAME;
+            TestUtils.pullDockerImage("mekomsolutions/superset-pro", testDir);
             TestUtils.createNetworkIfNecessary("web", testDir);
         } catch (IOException e) {
             throw new RuntimeException(e);
@@ -116,78 +119,91 @@ public static void afterAllSuper() throws IOException {
 
     @BeforeEach
     public void beforeSuper() throws Exception {
-        Map<String, String> envs = new HashMap<>();
-        envs.put("SQL_SCRIPTS_PATH", getResourcePath("distro/data"));
-        envs.put("DISTRO_PATH", getResourcePath("distro"));
-
-        envs.put("POSTGRES_PASSWORD", "password");
-        envs.put("POSTGRES_USER", "postgres");
-        envs.put("ANALYTICS_DB_NAME", DB_NAME_ANALYTICS);
-        envs.put("ANALYTICS_DB_USER", USER_ANALYTICS_DB);
-        envs.put("ANALYTICS_DB_PASSWORD", PASSWORD_ANALYTICS_DB);
-
-        envs.put("MYSQL_ROOT_PASSWORD", PASSWORD_OPENMRS_DB);
-        envs.put("OPENMRS_DB_NAME", DB_NAME_OPENMRS);
-        envs.put("OPENMRS_DB_USER", USER_OPENMRS_DB);
-        envs.put("OPENMRS_DB_PASSWORD", "password");
-        envs.put("OPENMRS_CONFIG_PATH", getResourcePath("distro/configs/openmrs/initializer_config"));
-        envs.put("OPENMRS_PROPERTIES_PATH", getResourcePath("distro/configs/openmrs/properties"));
-        envs.put("OPENMRS_FRONTEND_BINARY_PATH", getResourcePath("distro/binaries/openmrs/frontend"));
-        envs.put("OPENMRS_FRONTEND_CONFIG_PATH", getResourcePath("distro/configs/openmrs/frontend_config"));
-
-        envs.put("ODOO_CONFIG_FILE_PATH", getResourcePath("distro/configs/odoo/config/odoo.conf"));
-        envs.put("POSTGRES_DB_HOST", "postgresql");
-        envs.put("ODOO_DB_NAME", DB_NAME_ODOO);
-        envs.put("ODOO_DB_USER", USER_ODOO_DB);
-        envs.put("ODOO_DB_PASSWORD", PASSWORD_ODOO_DB);
-        envs.put("EIP_ODOO_OPENMRS_ROUTES_PATH", testDir);
-
-        envs.put("SUPERSET_CONFIG_PATH", getResourcePath("distro/configs/superset"));
-        envs.put("SUPERSET_DB", "superset");
-        envs.put("SUPERSET_DB_USER", "superset");
-        envs.put("SUPERSET_DB_PASSWORD", "superset");
-        List<File> dockerComposeFiles = new ArrayList<>();
-        dockerComposeFiles.add(new File(getResourcePath("run/docker/docker-compose-common.yml")));
-        dockerComposeFiles.add(new File(getResourcePath("run/docker/docker-compose-superset.yml")));
-        List<String> services = new ArrayList<>();
-        services.add("env-substitution");
-        services.add("postgresql");
-        services.add("superset");
+        Map<String, String> commonEnvs = new HashMap<>();
+        commonEnvs.put("DISTRO_PATH", getResourcePath("distro"));
+        commonEnvs.put("SQL_SCRIPTS_PATH", getResourcePath("distro/data"));
+        commonEnvs.put("POSTGRES_PASSWORD", "password");
+        commonEnvs.put("POSTGRES_USER", "postgres");
+        commonEnvs.put("POSTGRES_DB_HOST", "postgresql");
+        Map<String, String> ozoneEnvs = new HashMap<>(commonEnvs);
+
+        ozoneEnvs.put("MYSQL_ROOT_PASSWORD", PASSWORD_OPENMRS_DB);
+        ozoneEnvs.put("OPENMRS_DB_NAME", DB_NAME_OPENMRS);
+        ozoneEnvs.put("OPENMRS_DB_USER", USER_OPENMRS_DB);
+        ozoneEnvs.put("OPENMRS_DB_PASSWORD", "password");
+        ozoneEnvs.put("OPENMRS_CONFIG_PATH", getResourcePath("distro/configs/openmrs/initializer_config"));
+        ozoneEnvs.put("OPENMRS_PROPERTIES_PATH", getResourcePath("distro/configs/openmrs/properties"));
+        ozoneEnvs.put("OPENMRS_FRONTEND_BINARY_PATH", getResourcePath("distro/binaries/openmrs/frontend"));
+        ozoneEnvs.put("OPENMRS_FRONTEND_CONFIG_PATH", getResourcePath("distro/configs/openmrs/frontend_config"));
+
+        ozoneEnvs.put("ODOO_CONFIG_FILE_PATH", getResourcePath("distro/configs/odoo/config/odoo.conf"));
+        ozoneEnvs.put("ODOO_DB_NAME", DB_NAME_ODOO);
+        ozoneEnvs.put("ODOO_DB_USER", USER_ODOO_DB);
+        ozoneEnvs.put("ODOO_DB_PASSWORD", PASSWORD_ODOO_DB);
+        ozoneEnvs.put("EIP_ODOO_OPENMRS_ROUTES_PATH", testDir);
+
+        List<File> ozoneComposeFiles = new ArrayList<>();
+        ozoneComposeFiles.add(new File(getResourcePath("run/docker/docker-compose-common.yml")));
+        List<String> ozoneServices = new ArrayList<>();
+        ozoneServices.add("env-substitution");
+        ozoneServices.add("postgresql");
         if (requiresSourceDb()) {
-            dockerComposeFiles.add(new File(getResourcePath("run/docker/" + getDockerComposeFile())));
-            services.add(getSourceDbServiceName());
-            services.add(getSourceSystemName());
+            ozoneComposeFiles.add(new File(getResourcePath("run/docker/" + getDockerComposeFile())));
+            ozoneServices.add(getSourceDbServiceName());
+            ozoneServices.add(getSourceSystemName());
             if ("odoo".equalsIgnoreCase(getSourceSystemName())) {
-                dockerComposeFiles.add(new File(getResourcePath("run/docker/docker-compose-openmrs.yml")));
+                ozoneComposeFiles.add(new File(getResourcePath("run/docker/docker-compose-openmrs.yml")));
             }
         }
 
-        composeContainer = new ComposeContainer(dockerComposeFiles)
-                .withEnv(envs)
-                .withServices(services.toArray(new String[] {}))
-                .withExposedService("postgresql", 5432, Wait.forListeningPort());
+        ozoneCompose = new ComposeContainer(ozoneComposeFiles)
+                .withEnv(ozoneEnvs)
+                .withServices(ozoneServices.toArray(String[]::new));
         if (requiresSourceDb()) {
-            composeContainer.withExposedService(
+            ozoneCompose.withExposedService(
                     getSourceDbServiceName(), getSourceDbExposedPort(), Wait.forListeningPort());
             if ("openmrs".equals(getSourceSystemName())) {
-                composeContainer.waitingFor(
+                ozoneCompose.waitingFor(
                         "openmrs", Wait.forHealthcheck().withStartupTimeout(Duration.of(WAIT, SECONDS)));
             } else if ("odoo".equals(getSourceSystemName())) {
-                composeContainer.waitingFor(
+                ozoneCompose.waitingFor(
                         "odoo",
                         Wait.forLogMessage(".*odoo\\.modules\\.loading: Modules loaded.*", 1)
                                 .withStartupTimeout(Duration.of(120, SECONDS)));
             }
         }
+        ozoneCompose.withStartupTimeout(Duration.of(WAIT, SECONDS));
+
+        Map<String, String> analyticsEnvs = new HashMap<>(commonEnvs);
+        analyticsEnvs.put("ANALYTICS_DB_NAME", DB_NAME_ANALYTICS);
+        analyticsEnvs.put("ANALYTICS_DB_USER", USER_ANALYTICS_DB);
+        analyticsEnvs.put("ANALYTICS_DB_PASSWORD", PASSWORD_ANALYTICS_DB);
+        analyticsEnvs.put("SUPERSET_CONFIG_PATH", getResourcePath("distro/configs/superset"));
+        analyticsEnvs.put("SUPERSET_DASHBOARDS_PATH", getResourcePath("distro/configs/superset/assets"));
+        analyticsEnvs.put("SUPERSET_DB", "superset");
+        analyticsEnvs.put("SUPERSET_DB_USER", "superset");
+        analyticsEnvs.put("SUPERSET_DB_PASSWORD", "password");
+        List<File> analyticsComposeFiles = new ArrayList<>();
+        analyticsComposeFiles.add(new File(getResourcePath("docker-compose-db.yaml")));
+        analyticsComposeFiles.add(new File(getResourcePath("docker-compose-superset.yaml")));
+        List<String> analyticsServices = new ArrayList<>();
+        analyticsServices.add("postgresql");
+        analyticsServices.add("superset");
+        analyticsCompose = new ComposeContainer(analyticsComposeFiles)
+                .withEnv(analyticsEnvs)
+                .withServices(analyticsServices.toArray(String[]::new))
+                .withExposedService("postgresql", 5432, Wait.forListeningPort());
+        analyticsCompose.withStartupTimeout(Duration.of(WAIT, SECONDS));
 
-        composeContainer.withStartupTimeout(Duration.of(WAIT, SECONDS));
         long start = System.currentTimeMillis();
-        composeContainer.start();
+        ozoneCompose.start();
+        analyticsCompose.start();
         long duration = (System.currentTimeMillis() - start) / 1000;
-        System.out.println("Compose container startup took: " + duration + "secs");
-        analyticsDb = composeContainer.getContainerByServiceName("postgresql").get();
+        System.out.println("Compose containers startup took: " + duration + "secs");
+
+        analyticsDb = analyticsCompose.getContainerByServiceName("postgresql").get();
         if (requiresSourceDb()) {
-            sourceDb = composeContainer
+            sourceDb = ozoneCompose
                     .getContainerByServiceName(getSourceDbServiceName())
                     .get();
         }
@@ -226,7 +242,8 @@ public void setup() throws Exception {
             }
         }
 
-        composeContainer.stop();
+        ozoneCompose.stop();
+        analyticsCompose.stop();
     }
 
     protected void initJobAndStartCluster(BaseJob job) throws Exception {

src/test/java/com/ozonehis/data/pipelines/TestUtils.java

@@ -81,11 +81,19 @@ public static void createNetworkIfNecessary(String name, String workingDir) {
             execute("docker network inspect " + name, workingDir);
         } catch (RuntimeException t) {
             // Ignore, network does not exist
+            System.out.println("Creating network " + name);
             execute("docker network create -d bridge " + name, workingDir);
+            System.out.println("Network created " + name);
         }
     }
 
-    private static void execute(String cmd, String workingDir) {
+    public static void pullDockerImage(String name, String workingDir) {
+        System.out.println("Pulling docker image " + name);
+        execute("docker pull " + name, workingDir);
+        System.out.println("Done pulling image " + name);
+    }
+
+    public static void execute(String cmd, String workingDir) {
         ProcessBuilder builder = new ProcessBuilder();
         builder.command("sh", "-c", cmd).directory(new File(workingDir));
         ExecutorService executor = Executors.newFixedThreadPool(2);