diff --git a/.github/workflows/aurora_performance.yml b/.github/workflows/aurora_performance.yml index 3f41f3b1..49ae6e74 100644 --- a/.github/workflows/aurora_performance.yml +++ b/.github/workflows/aurora_performance.yml @@ -61,6 +61,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ env.TEMP_AWS_SECRET_ACCESS_KEY }} AWS_SESSION_TOKEN: ${{ env.TEMP_AWS_SESSION_TOKEN }} NUM_INSTANCES: 5 + AURORA_MYSQL_DB_ENGINE_VERSION: "default" + AURORA_PG_DB_ENGINE_VERSION: "default" - name: "Get Github Action IP" if: always() diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index d6ff0ccf..abcc0a50 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -1,4 +1,4 @@ -name: Integration Tests +name: Integration Tests Default on: workflow_dispatch: @@ -16,7 +16,7 @@ on: jobs: run-integration-tests: - name: Run Integration Tests + name: Run Integration Tests with Default runs-on: ubuntu-latest steps: @@ -68,6 +68,8 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ env.TEMP_AWS_SECRET_ACCESS_KEY }} AWS_SESSION_TOKEN: ${{ env.TEMP_AWS_SESSION_TOKEN }} NUM_INSTANCES: 5 + AURORA_MYSQL_DB_ENGINE_VERSION: "default" + AURORA_PG_DB_ENGINE_VERSION: "default" - name: "Get Github Action IP" if: always() @@ -88,6 +90,6 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: integration-report + name: integration-report-default path: ./tests/integration/container/reports retention-days: 5 diff --git a/.github/workflows/integration_tests_latest.yml b/.github/workflows/integration_tests_latest.yml new file mode 100644 index 00000000..cd64c03d --- /dev/null +++ b/.github/workflows/integration_tests_latest.yml @@ -0,0 +1,95 @@ +name: Integration Tests Latest + +on: + workflow_dispatch: + push: + branches: + - main + paths-ignore: + - "**/*.md" + - "**/*.jpg" + - "**/README.txt" + - "**/LICENSE.txt" + - "docs/**" + - "ISSUE_TEMPLATE/**" + - "**/remove-old-artifacts.yml" + +jobs: + run-integration-tests: + name: Run Integration Tests with Latest + runs-on: ubuntu-latest + + steps: + - name: Clone repository + uses: actions/checkout@v4 + with: + fetch-depth: 50 + + - name: "Set up JDK 8" + uses: actions/setup-java@v3 + with: + distribution: "corretto" + java-version: 8 + - name: Set up Node.js + uses: actions/setup-node@v3 + with: + node-version: "20.x" + - name: Install dependencies + run: npm install --no-save + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_DEFAULT_REGION }} + + - name: Set up Temp AWS Credentials + run: | + creds=($(aws sts get-session-token \ + --duration-seconds 21600 \ + --query 'Credentials.[AccessKeyId, SecretAccessKey, SessionToken]' \ + --output text \ + | xargs)); + echo "::add-mask::${creds[0]}" + echo "::add-mask::${creds[1]}" + echo "::add-mask::${creds[2]}" + echo "TEMP_AWS_ACCESS_KEY_ID=${creds[0]}" >> $GITHUB_ENV + echo "TEMP_AWS_SECRET_ACCESS_KEY=${creds[1]}" >> $GITHUB_ENV + echo "TEMP_AWS_SESSION_TOKEN=${creds[2]}" >> $GITHUB_ENV + + - name: Run Integration Tests + run: | + ./gradlew --no-parallel --no-daemon test-aurora --info + env: + AURORA_CLUSTER_DOMAIN: ${{ secrets.DB_CONN_SUFFIX }} + AURORA_DB_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_ACCESS_KEY_ID: ${{ env.TEMP_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ env.TEMP_AWS_SECRET_ACCESS_KEY }} + AWS_SESSION_TOKEN: ${{ env.TEMP_AWS_SESSION_TOKEN }} + NUM_INSTANCES: 5 + AURORA_MYSQL_DB_ENGINE_VERSION: "latest" + AURORA_PG_DB_ENGINE_VERSION: "latest" + + - name: "Get Github Action IP" + if: always() + id: ip + uses: haythem/public-ip@v1.3 + + - name: "Remove Github Action IP" + if: always() + run: | + aws ec2 revoke-security-group-ingress \ + --group-name default \ + --protocol -1 \ + --port -1 \ + --cidr ${{ steps.ip.outputs.ipv4 }}/32 \ + 2>&1 > /dev/null; + + - name: Archive results + if: always() + uses: actions/upload-artifact@v4 + with: + name: integration-report-latest + path: ./tests/integration/container/reports + retention-days: 5 diff --git a/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java b/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java index 284b4021..59ac6968 100644 --- a/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java +++ b/tests/integration/host/src/test/java/integration/host/TestEnvironmentConfig.java @@ -35,6 +35,11 @@ public class TestEnvironmentConfig implements AutoCloseable { private String auroraClusterName; // "cluster-mysql" private String auroraClusterDomain; // "XYZ.us-west-2.rds.amazonaws.com" + // Expected values: "latest", "default", or engine version, for example, "15.4" + // If left as empty, will use default version + public String auroraMySqlDbEngineVersion; + public String auroraPgDbEngineVersion; + private String awsAccessKeyId; private String awsSecretAccessKey; private String awsSessionToken; @@ -202,6 +207,10 @@ private static void createAuroraDbCluster(TestEnvironmentConfig env, int numOfIn env.auroraClusterName = System.getenv("AURORA_CLUSTER_NAME"); // "cluster-mysql" env.auroraClusterDomain = System.getenv("AURORA_CLUSTER_DOMAIN"); // "XYZ.us-west-2.rds.amazonaws.com" + env.auroraMySqlDbEngineVersion = + System.getenv("AURORA_MYSQL_DB_ENGINE_VERSION"); // "latest", "default" + env.auroraPgDbEngineVersion = + System.getenv("AURORA_PG_ENGINE_VERSION"); if (StringUtils.isNullOrEmpty(env.auroraClusterDomain)) { throw new RuntimeException("Environment variable AURORA_CLUSTER_DOMAIN is required."); @@ -251,7 +260,13 @@ private static void createAuroraDbCluster(TestEnvironmentConfig env, int numOfIn try { String engine = getAuroraDbEngine(env.info.getRequest()); - String engineVersion = getAuroraDbEngineVersion(env.info.getRequest()); + + String engineVersion = getAuroraDbEngineVersion(env); + if (StringUtils.isNullOrEmpty(engineVersion)) { + throw new RuntimeException("Failed to get engine version."); + } + LOGGER.finer("Using " + engine + " " + engineVersion); + String instanceClass = getAuroraInstanceClass(env.info.getRequest()); env.auroraClusterDomain = @@ -328,15 +343,37 @@ private static String getAuroraDbEngine(TestEnvironmentRequest request) { } } - private static String getAuroraDbEngineVersion(TestEnvironmentRequest request) { + private static String getAuroraDbEngineVersion(TestEnvironmentConfig env) { + String engineName; + String systemPropertyVersion; + TestEnvironmentRequest request = env.info.getRequest(); switch (request.getDatabaseEngine()) { case MYSQL: - return "8.0.mysql_aurora.3.04.0"; + engineName = "aurora-mysql"; + systemPropertyVersion = env.auroraMySqlDbEngineVersion; + break; case PG: - return "15.2"; + engineName = "aurora-postgresql"; + systemPropertyVersion = env.auroraPgDbEngineVersion; + break; default: throw new NotImplementedException(request.getDatabaseEngine().toString()); } + return findAuroraDbEngineVersion(env, engineName, systemPropertyVersion.toLowerCase()); + } + + private static String findAuroraDbEngineVersion(TestEnvironmentConfig env, String engineName, String systemPropertyVersion) { + if (systemPropertyVersion == null) { + return env.auroraUtil.getDefaultVersion(engineName); + } + switch (systemPropertyVersion) { + case "default": + return env.auroraUtil.getDefaultVersion(engineName); + case "latest": + return env.auroraUtil.getLatestVersion(engineName); + default: + return systemPropertyVersion; + } } private static String getAuroraInstanceClass(TestEnvironmentRequest request) { diff --git a/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java b/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java index d8bfad5c..3dcde58f 100644 --- a/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java +++ b/tests/integration/host/src/test/java/integration/host/util/AuroraTestUtility.java @@ -37,6 +37,7 @@ import java.sql.Statement; import java.time.Duration; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.Optional; import java.util.Properties; @@ -63,12 +64,15 @@ import software.amazon.awssdk.services.rds.model.CreateDbInstanceRequest; import software.amazon.awssdk.services.rds.model.DBCluster; import software.amazon.awssdk.services.rds.model.DBClusterMember; +import software.amazon.awssdk.services.rds.model.DBEngineVersion; import software.amazon.awssdk.services.rds.model.DBInstance; import software.amazon.awssdk.services.rds.model.DbClusterNotFoundException; import software.amazon.awssdk.services.rds.model.DeleteDbClusterResponse; import software.amazon.awssdk.services.rds.model.DeleteDbInstanceRequest; import software.amazon.awssdk.services.rds.model.DescribeDbClustersRequest; import software.amazon.awssdk.services.rds.model.DescribeDbClustersResponse; +import software.amazon.awssdk.services.rds.model.DescribeDbEngineVersionsRequest; +import software.amazon.awssdk.services.rds.model.DescribeDbEngineVersionsResponse; import software.amazon.awssdk.services.rds.model.DescribeDbInstancesResponse; import software.amazon.awssdk.services.rds.model.FailoverDbClusterResponse; import software.amazon.awssdk.services.rds.model.Filter; @@ -476,4 +480,35 @@ public void addAuroraAwsIamUser( } } } + + public String getDefaultVersion(String engine) { + final DescribeDbEngineVersionsResponse versions = rdsClient.describeDBEngineVersions( + DescribeDbEngineVersionsRequest.builder() + .defaultOnly(true) + .engine(engine) + .build() + ); + if (!versions.dbEngineVersions().isEmpty()) { + return versions.dbEngineVersions().get(0).engineVersion(); + } + throw new RuntimeException("Failed to find default version"); + } + + public String getLatestVersion(String engine) { + return getEngineVersions(engine).stream() + .sorted(Comparator.reverseOrder()) + .findFirst() + .orElse(null); + } + + private List getEngineVersions(String engine) { + final List res = new ArrayList<>(); + final DescribeDbEngineVersionsResponse versions = rdsClient.describeDBEngineVersions( + DescribeDbEngineVersionsRequest.builder().engine(engine).build() + ); + for (DBEngineVersion version : versions.dbEngineVersions()) { + res.add(version.engineVersion()); + } + return res; + } }