Integration test setup

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java

@@ -59,6 +59,7 @@
 
 import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
 import com.expediagroup.beekeeper.core.model.PeriodDuration;
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
 import com.expediagroup.beekeeper.integration.model.AddPartitionSqsMessage;
 import com.expediagroup.beekeeper.integration.model.AlterPartitionSqsMessage;
 import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage;
@@ -213,6 +214,17 @@ public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLExcepti
     assertExpiredMetadata(expiredMetadata.get(1), LOCATION_B, PARTITION_B_NAME);
   }
 
+  @Test
+  public void addEventToHistoryTable() throws SQLException, IOException, URISyntaxException {
+    CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true);
+    amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString()));
+
+    await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1);
+
+    List<BeekeeperHistory> beekeeperHistory = getBeekeeperHistory(EXPIRED);
+    System.out.println(beekeeperHistory);
+  }
+
   @Test
   public void healthCheck() {
     CloseableHttpClient client = HttpClientBuilder.create().build();

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java

@@ -30,6 +30,8 @@
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.CREATION_TIMESTAMP_VALUE;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_FIELD;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE;
+import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_DETAILS_FIELD;
+import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_TIMESTAMP_FIELD;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.HOUSEKEEPING_STATUS_FIELD;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.ID_FIELD;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.LIFECYCLE_TYPE_FIELD;
@@ -39,6 +41,7 @@
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.SHORT_CLEANUP_DELAY_VALUE;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_FIELD;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE;
+import static com.expediagroup.beekeeper.integration.utils.ResultSetToBeekeeperHistoryMapper.mapToBeekeeperHistory;
 import static com.expediagroup.beekeeper.integration.utils.ResultSetToHousekeepingEntityMapper.mapToHousekeepingMetadata;
 import static com.expediagroup.beekeeper.integration.utils.ResultSetToHousekeepingEntityMapper.mapToHousekeepingPath;
 
@@ -61,8 +64,10 @@
 
 import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
 import com.expediagroup.beekeeper.core.model.HousekeepingPath;
+import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
 import com.expediagroup.beekeeper.core.model.LifecycleEventType;
 import com.expediagroup.beekeeper.core.model.PeriodDuration;
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
 import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils;
 import com.expediagroup.beekeeper.integration.utils.MySqlTestUtils;
 
@@ -98,6 +103,8 @@ public abstract class BeekeeperIntegrationTestBase {
       .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, PARTITION_NAME_FIELD,
           HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD,
           CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD);
+  private static final String BEEKEEPER_HISTORY_FIELDS = String.join(",", ID_FIELD, EVENT_TIMESTAMP_FIELD,
+      DATABASE_NAME_FIELD, TABLE_NAME_FIELD, LIFECYCLE_TYPE_FIELD, HOUSEKEEPING_STATUS_FIELD, EVENT_DETAILS_FIELD);
   private static final String LIFE_CYCLE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD;
   private static final String LIFE_CYCLE_AND_UPDATE_FILTER = "WHERE "
       + LIFECYCLE_TYPE_FIELD
@@ -178,7 +185,7 @@ protected void insertExpiredMetadata(String path, String partitionName) throws S
   }
 
   protected void insertExpiredMetadata(String tableName, String path, String partitionName, String cleanupDelay)
-    throws SQLException {
+      throws SQLException {
     HousekeepingMetadata metadata = createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay);
     insertExpiredMetadata(metadata);
   }
@@ -198,6 +205,16 @@ protected void insertExpiredMetadata(HousekeepingMetadata metadata) throws SQLEx
             values);
   }
 
+  protected void insertBeekeeperHistory(BeekeeperHistory beekeeperHistory) throws SQLException {
+    String values = Stream.of(beekeeperHistory.getId().toString(), beekeeperHistory.getEventTimestamp(),
+            beekeeperHistory.getDatabaseName(), beekeeperHistory.getTableName(), beekeeperHistory.getLifecycleType(),
+            beekeeperHistory.getHousekeepingStatus(), beekeeperHistory.getEventDetails())
+        .map(s -> s == null ? null : "\"" + s + "\"")
+        .collect(Collectors.joining(", "));
+
+    mySQLTestUtils.insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, BEEKEEPER_HISTORY_FIELDS, values);
+  }
+
   protected int getUnreferencedPathsRowCount() throws SQLException {
     return mySQLTestUtils
         .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME,
@@ -216,6 +233,13 @@ protected int getUpdatedExpiredMetadataRowCount() throws SQLException {
             format(LIFE_CYCLE_AND_UPDATE_FILTER, EXPIRED));
   }
 
+  protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) throws SQLException {
+    String filter  = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'";
+
+    return mySQLTestUtils.getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME,
+        format(filter, lifecycleEventType));
+  }
+
   protected List<HousekeepingPath> getUnreferencedPaths() throws SQLException {
     List<HousekeepingPath> paths = new ArrayList<>();
     ResultSet resultSet = mySQLTestUtils
@@ -242,6 +266,19 @@ protected List<HousekeepingMetadata> getExpiredMetadata() throws SQLException {
     return metadata;
   }
 
+  protected List<BeekeeperHistory> getBeekeeperHistory(LifecycleEventType lifecycleEventType) throws SQLException {
+    String filter  = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'";
+    List<BeekeeperHistory> history = new ArrayList<>();
+    ResultSet resultSet = mySQLTestUtils.getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME,
+        format(filter, lifecycleEventType));
+
+    while (resultSet.next()) {
+      history.add(mapToBeekeeperHistory(resultSet));
+    }
+
+    return history;
+  }
+
   private HousekeepingPath createHousekeepingPath(String path, LifecycleEventType lifecycleEventType) {
     return HousekeepingPath
         .builder()
@@ -282,4 +319,20 @@ private HousekeepingMetadata createHousekeepingMetadata(
         .build();
   }
 
+  private BeekeeperHistory createBeekeeperHistory(
+      String tableName,
+      LifecycleEventType lifecycleEventType,
+      HousekeepingStatus housekeepingStatus,
+      String eventDetails
+  ) {
+    return BeekeeperHistory.builder()
+        .id(id++)
+        .eventTimestamp(CREATION_TIMESTAMP_VALUE)
+        .databaseName(DATABASE_NAME_VALUE)
+        .tableName(tableName)
+        .lifecycleType(lifecycleEventType.toString())
+        .housekeepingStatus(housekeepingStatus.name())
+        .eventDetails(eventDetails)
+        .build();
+  }
 }

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java

@@ -20,6 +20,7 @@
 import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SQS;
 
 import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED;
+import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED;
 import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION;
 import static com.expediagroup.beekeeper.integration.CommonTestVariables.CLEANUP_ATTEMPTS_VALUE;
@@ -59,6 +60,7 @@
 
 import com.expediagroup.beekeeper.core.model.HousekeepingPath;
 import com.expediagroup.beekeeper.core.model.PeriodDuration;
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
 import com.expediagroup.beekeeper.integration.model.AlterPartitionSqsMessage;
 import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage;
 import com.expediagroup.beekeeper.integration.model.DropPartitionSqsMessage;
@@ -196,6 +198,18 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy
     assertUnreferencedPath(unreferencedPaths.get(1), "s3://bucket/tableLocation2");
   }
 
+  @Test
+  public void addBeekeeperHistoryEvent() throws IOException, URISyntaxException, SQLException {
+    AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation",
+        "s3://bucket/oldTableLocation", true, true);
+    amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString()));
+
+    await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1);
+
+    List<BeekeeperHistory> beekeeperHistory = getBeekeeperHistory(EXPIRED);
+    System.out.println(beekeeperHistory);
+  }
+
   @Test
   public void healthCheck() {
     CloseableHttpClient client = HttpClientBuilder.create().build();

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java

@@ -41,6 +41,8 @@ private CommonTestVariables() {}
   public static final String CLEANUP_ATTEMPTS_FIELD = "cleanup_attempts";
   public static final String CLIENT_ID_FIELD = "client_id";
   public static final String LIFECYCLE_TYPE_FIELD = "lifecycle_type";
+  public static final String EVENT_DETAILS_FIELD = "event_details";
+  public static final String EVENT_TIMESTAMP_FIELD = "event_timestamp";
 
   // HOUSEKEEPINGENTITY DEFAULT VALUES
   public static final String DATABASE_NAME_VALUE = "some_database";

beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java

@@ -0,0 +1,27 @@
+package com.expediagroup.beekeeper.integration.utils;
+
+import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_FIELD;
+import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_DETAILS_FIELD;
+import static com.expediagroup.beekeeper.integration.CommonTestVariables.HOUSEKEEPING_STATUS_FIELD;
+import static com.expediagroup.beekeeper.integration.CommonTestVariables.ID_FIELD;
+import static com.expediagroup.beekeeper.integration.CommonTestVariables.LIFECYCLE_TYPE_FIELD;
+import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_FIELD;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
+
+public class ResultSetToBeekeeperHistoryMapper {
+
+  public static BeekeeperHistory mapToBeekeeperHistory(ResultSet resultSet) throws SQLException {
+    return BeekeeperHistory.builder()
+        .id(resultSet.getLong(ID_FIELD))
+        .databaseName(resultSet.getString(DATABASE_NAME_FIELD))
+        .tableName(resultSet.getString(TABLE_NAME_FIELD))
+        .lifecycleType(resultSet.getString(LIFECYCLE_TYPE_FIELD))
+        .housekeepingStatus(resultSet.getString(HOUSEKEEPING_STATUS_FIELD))
+        .eventDetails(resultSet.getString(EVENT_DETAILS_FIELD))
+        .build();
+  }
+}