From c1d297004b607b757fd29bdd6ce6c8194db21887 Mon Sep 17 00:00:00 2001 From: Kiran Godishala <53332225+kirangodishala@users.noreply.github.com> Date: Tue, 11 Jun 2024 07:49:24 +0530 Subject: [PATCH] fix(liquibase): fix checkSum errors occurring with spinnaker upgrade (#4727) * fix(tests): Add new integration test for postgres migration - PostgresMigrationContainerTest * fix(liquibase): Fix validChecksum errors occurred due to liquibase upgrade (cherry picked from commit c5e7aad963913b70dc53b733e44aa2f48672b7f4) --- .../spinnaker/orca/BaseContainerTest.java | 2 +- .../orca/PostgresMigrationContainerTest.java | 154 ++++++++++++++++++ .../20180510-add-legacy-id-fields.yml | 1 + .../20180515-execution-canceled-column.yml | 1 + .../db/changelog/20180521-status-enum.yml | 1 + .../db/changelog/20180724-partitions.yml | 1 + .../db/changelog/20181016-add-start-time.yml | 1 + .../20200327-deleted-executions-table.yml | 4 +- 8 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 orca-integration/src/test/java/com/netflix/spinnaker/orca/PostgresMigrationContainerTest.java diff --git a/orca-integration/src/test/java/com/netflix/spinnaker/orca/BaseContainerTest.java b/orca-integration/src/test/java/com/netflix/spinnaker/orca/BaseContainerTest.java index 0f8664c961..a0643f65e2 100644 --- a/orca-integration/src/test/java/com/netflix/spinnaker/orca/BaseContainerTest.java +++ b/orca-integration/src/test/java/com/netflix/spinnaker/orca/BaseContainerTest.java @@ -41,7 +41,7 @@ class BaseContainerTest { protected final Network network = Network.newNetwork(); - private static final int ORCA_PORT = 8083; + protected static final int ORCA_PORT = 8083; protected GenericContainer orcaContainer; diff --git a/orca-integration/src/test/java/com/netflix/spinnaker/orca/PostgresMigrationContainerTest.java b/orca-integration/src/test/java/com/netflix/spinnaker/orca/PostgresMigrationContainerTest.java new file mode 100644 index 0000000000..536eafe62b --- /dev/null +++ b/orca-integration/src/test/java/com/netflix/spinnaker/orca/PostgresMigrationContainerTest.java @@ -0,0 +1,154 @@ +/* + * Copyright 2024 OpsMx, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.netflix.spinnaker.orca; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import java.time.Duration; +import java.util.Map; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Testcontainers +public class PostgresMigrationContainerTest extends BaseContainerTest { + + private static final Logger logger = + LoggerFactory.getLogger(PostgresMigrationContainerTest.class); + + private static final String POSTGRES_NETWORK_ALIAS = "postgresHost"; + + private static final int POSTGRES_PORT = 5432; + + private PostgreSQLContainer postgres; + + private GenericContainer orcaInitialContainer; + + // this is the latest image that is still running on liquibase 3.10.3 which create the conditions + // similar to real scenario so that test identifies when validChecksums are not added in the later + // version of orca where higher liquibase versions are used + private static final DockerImageName previousDockerImageName = + DockerImageName.parse( + "us-docker.pkg.dev/spinnaker-community/docker/orca:8.36.3-dev-release-1.32.x-3f8965d03-202406101625-unvalidated"); + + private String jdbcUrl = ""; + + @BeforeEach + void setup() throws Exception { + postgres = + new PostgreSQLContainer<>("postgres:15") + .withDatabaseName("orca") + .withUsername("postgres") + .withPassword("postgres") + .withNetwork(network) + .withNetworkAliases(POSTGRES_NETWORK_ALIAS) + .withInitScript("postgres_init.sql"); + postgres.start(); + jdbcUrl = String.format("jdbc:postgresql://%s:%d/orca", POSTGRES_NETWORK_ALIAS, POSTGRES_PORT); + + // Start the first orca(from previous release) container so that all the db changelog + // sets are executed + orcaInitialContainer = + new GenericContainer(previousDockerImageName) + .withNetwork(network) + .withExposedPorts(ORCA_PORT) + .waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(120))) + .dependsOn(postgres) + .withEnv("SPRING_APPLICATION_JSON", getSpringApplicationJson()); + orcaInitialContainer.start(); + Slf4jLogConsumer logConsumer = new Slf4jLogConsumer(logger); + orcaInitialContainer.followOutput(logConsumer); + orcaInitialContainer.stop(); + + orcaContainer + .dependsOn(postgres) + .withEnv("SPRING_APPLICATION_JSON", getSpringApplicationJson()) + .start(); + + orcaContainer.followOutput(logConsumer); + } + + private String getSpringApplicationJson() throws JsonProcessingException { + logger.info("--------- jdbcUrl: '{}'", jdbcUrl); + Map connectionPool = + Map.of( + "dialect", + "POSTGRES", + "jdbcUrl", + jdbcUrl, + "user", + "orca_service", + "password", + "0rcaPassw0rd"); + Map migration = + Map.of("jdbcUrl", jdbcUrl, "user", "orca_migrate", "password", "0rcaPassw0rd"); + Map sql = Map.of("enabled", true); + Map redis = Map.of("enabled", false); + Map pendingExecutionService = Map.of("sql", sql, "redis", redis); + Map executionRepository = Map.of("sql", sql, "redis", redis); + Map keiko = Map.of("sql", sql, "redis", redis); + + Map properties = + Map.of( + "sql.enabled", + "true", + "services.fiat.baseUrl", + "http://nowhere", + "sql.connectionPool", + connectionPool, + "redis.enabled", + "false", + "sql.migration", + migration, + "executionRepository", + executionRepository, + "keiko.queue", + keiko, + "queue.pendingExecutionService", + pendingExecutionService, + "monitor.activeExecutions.redis", + "false"); + ObjectMapper mapper = new ObjectMapper(); + return mapper.writeValueAsString(properties); + } + + @AfterAll + void cleanupOnce() { + if (orcaContainer != null) { + orcaContainer.stop(); + } + + if (postgres != null) { + postgres.stop(); + } + } + + @Test + void testHealthCheckWithPostgres() throws Exception { + super.testHealthCheck(); + } +} diff --git a/orca-sql/src/main/resources/db/changelog/20180510-add-legacy-id-fields.yml b/orca-sql/src/main/resources/db/changelog/20180510-add-legacy-id-fields.yml index c5d70c4488..386c03cd16 100644 --- a/orca-sql/src/main/resources/db/changelog/20180510-add-legacy-id-fields.yml +++ b/orca-sql/src/main/resources/db/changelog/20180510-add-legacy-id-fields.yml @@ -4,6 +4,7 @@ databaseChangeLog: dbms: postgresql remove: afterColumn - changeSet: + validCheckSum: 8:f97b552b426f284932461b028e9d6d9c id: add-legacy-id-fields author: cthielen diff --git a/orca-sql/src/main/resources/db/changelog/20180515-execution-canceled-column.yml b/orca-sql/src/main/resources/db/changelog/20180515-execution-canceled-column.yml index d6bd0ec791..51694a65a9 100644 --- a/orca-sql/src/main/resources/db/changelog/20180515-execution-canceled-column.yml +++ b/orca-sql/src/main/resources/db/changelog/20180515-execution-canceled-column.yml @@ -4,6 +4,7 @@ databaseChangeLog: dbms: postgresql remove: afterColumn - changeSet: + validCheckSum: 8:4c214f01b403163ac014e78ef3603c71 id: add-canceled-column author: robzienert changes: diff --git a/orca-sql/src/main/resources/db/changelog/20180521-status-enum.yml b/orca-sql/src/main/resources/db/changelog/20180521-status-enum.yml index d72e9b4a8a..3d4035ef1a 100644 --- a/orca-sql/src/main/resources/db/changelog/20180521-status-enum.yml +++ b/orca-sql/src/main/resources/db/changelog/20180521-status-enum.yml @@ -1,5 +1,6 @@ databaseChangeLog: - changeSet: + validCheckSum: 8:a927aca379a0eecf2a6eecdee281be9f id: modify-status-column-enum author: afeldman changes: diff --git a/orca-sql/src/main/resources/db/changelog/20180724-partitions.yml b/orca-sql/src/main/resources/db/changelog/20180724-partitions.yml index ea84766f8a..39c185b206 100644 --- a/orca-sql/src/main/resources/db/changelog/20180724-partitions.yml +++ b/orca-sql/src/main/resources/db/changelog/20180724-partitions.yml @@ -4,6 +4,7 @@ databaseChangeLog: dbms: postgresql remove: afterColumn - changeSet: + validCheckSum: 8:26bce67df849ed409cdd5dffc335a95b id: partition-updated-executions author: rzienert changes: diff --git a/orca-sql/src/main/resources/db/changelog/20181016-add-start-time.yml b/orca-sql/src/main/resources/db/changelog/20181016-add-start-time.yml index bccb0f20de..d10e32e151 100644 --- a/orca-sql/src/main/resources/db/changelog/20181016-add-start-time.yml +++ b/orca-sql/src/main/resources/db/changelog/20181016-add-start-time.yml @@ -4,6 +4,7 @@ databaseChangeLog: dbms: postgresql remove: afterColumn - changeSet: + validCheckSum: 8:576429ab611ed541e5f99ab282de4177 id: 20181016-add-start-time author: robzienert changes: diff --git a/orca-sql/src/main/resources/db/changelog/20200327-deleted-executions-table.yml b/orca-sql/src/main/resources/db/changelog/20200327-deleted-executions-table.yml index 9da222bd61..b629de1d2b 100644 --- a/orca-sql/src/main/resources/db/changelog/20200327-deleted-executions-table.yml +++ b/orca-sql/src/main/resources/db/changelog/20200327-deleted-executions-table.yml @@ -1,10 +1,8 @@ databaseChangeLog: - changeSet: + validCheckSum: 8:10687930e669629d8c370cd5a2348585 id: create-deleted-executions-table author: mvulfson - validCheckSum: - # Original changeset checksum - - 8:847dd443c55dcb0a47a9576b891ad5ef changes: - createTable: tableName: deleted_executions