diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index ccbf19c6..a5bb23c1 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -59,6 +59,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.model.AddPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage; @@ -213,6 +214,17 @@ public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLExcepti assertExpiredMetadata(expiredMetadata.get(1), LOCATION_B, PARTITION_B_NAME); } + @Test + public void addEventToHistoryTable() throws SQLException, IOException, URISyntaxException { + CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true); + amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1); + + List beekeeperHistory = getBeekeeperHistory(EXPIRED); + System.out.println(beekeeperHistory); + } + @Test public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 1a48ed94..0bbefc49 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -30,6 +30,8 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.CREATION_TIMESTAMP_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_DETAILS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_TIMESTAMP_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.HOUSEKEEPING_STATUS_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.ID_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.LIFECYCLE_TYPE_FIELD; @@ -39,6 +41,7 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.SHORT_CLEANUP_DELAY_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; +import static com.expediagroup.beekeeper.integration.utils.ResultSetToBeekeeperHistoryMapper.mapToBeekeeperHistory; import static com.expediagroup.beekeeper.integration.utils.ResultSetToHousekeepingEntityMapper.mapToHousekeepingMetadata; import static com.expediagroup.beekeeper.integration.utils.ResultSetToHousekeepingEntityMapper.mapToHousekeepingPath; @@ -61,8 +64,10 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.integration.utils.MySqlTestUtils; @@ -98,6 +103,8 @@ public abstract class BeekeeperIntegrationTestBase { .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, PARTITION_NAME_FIELD, HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); + private static final String BEEKEEPER_HISTORY_FIELDS = String.join(",", ID_FIELD, EVENT_TIMESTAMP_FIELD, + DATABASE_NAME_FIELD, TABLE_NAME_FIELD, LIFECYCLE_TYPE_FIELD, HOUSEKEEPING_STATUS_FIELD, EVENT_DETAILS_FIELD); private static final String LIFE_CYCLE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD; private static final String LIFE_CYCLE_AND_UPDATE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD @@ -178,7 +185,7 @@ protected void insertExpiredMetadata(String path, String partitionName) throws S } protected void insertExpiredMetadata(String tableName, String path, String partitionName, String cleanupDelay) - throws SQLException { + throws SQLException { HousekeepingMetadata metadata = createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay); insertExpiredMetadata(metadata); } @@ -198,6 +205,16 @@ protected void insertExpiredMetadata(HousekeepingMetadata metadata) throws SQLEx values); } + protected void insertBeekeeperHistory(BeekeeperHistory beekeeperHistory) throws SQLException { + String values = Stream.of(beekeeperHistory.getId().toString(), beekeeperHistory.getEventTimestamp(), + beekeeperHistory.getDatabaseName(), beekeeperHistory.getTableName(), beekeeperHistory.getLifecycleType(), + beekeeperHistory.getHousekeepingStatus(), beekeeperHistory.getEventDetails()) + .map(s -> s == null ? null : "\"" + s + "\"") + .collect(Collectors.joining(", ")); + + mySQLTestUtils.insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, BEEKEEPER_HISTORY_FIELDS, values); + } + protected int getUnreferencedPathsRowCount() throws SQLException { return mySQLTestUtils .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, @@ -216,6 +233,13 @@ protected int getUpdatedExpiredMetadataRowCount() throws SQLException { format(LIFE_CYCLE_AND_UPDATE_FILTER, EXPIRED)); } + protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) throws SQLException { + String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; + + return mySQLTestUtils.getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, + format(filter, lifecycleEventType)); + } + protected List getUnreferencedPaths() throws SQLException { List paths = new ArrayList<>(); ResultSet resultSet = mySQLTestUtils @@ -242,6 +266,19 @@ protected List getExpiredMetadata() throws SQLException { return metadata; } + protected List getBeekeeperHistory(LifecycleEventType lifecycleEventType) throws SQLException { + String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; + List history = new ArrayList<>(); + ResultSet resultSet = mySQLTestUtils.getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, + format(filter, lifecycleEventType)); + + while (resultSet.next()) { + history.add(mapToBeekeeperHistory(resultSet)); + } + + return history; + } + private HousekeepingPath createHousekeepingPath(String path, LifecycleEventType lifecycleEventType) { return HousekeepingPath .builder() @@ -282,4 +319,20 @@ private HousekeepingMetadata createHousekeepingMetadata( .build(); } + private BeekeeperHistory createBeekeeperHistory( + String tableName, + LifecycleEventType lifecycleEventType, + HousekeepingStatus housekeepingStatus, + String eventDetails + ) { + return BeekeeperHistory.builder() + .id(id++) + .eventTimestamp(CREATION_TIMESTAMP_VALUE) + .databaseName(DATABASE_NAME_VALUE) + .tableName(tableName) + .lifecycleType(lifecycleEventType.toString()) + .housekeepingStatus(housekeepingStatus.name()) + .eventDetails(eventDetails) + .build(); + } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 92a3d0c0..d43ab5dc 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -20,6 +20,7 @@ import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SQS; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.CLEANUP_ATTEMPTS_VALUE; @@ -59,6 +60,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.model.AlterPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage; import com.expediagroup.beekeeper.integration.model.DropPartitionSqsMessage; @@ -196,6 +198,18 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy assertUnreferencedPath(unreferencedPaths.get(1), "s3://bucket/tableLocation2"); } + @Test + public void addBeekeeperHistoryEvent() throws IOException, URISyntaxException, SQLException { + AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", + "s3://bucket/oldTableLocation", true, true); + amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + + List beekeeperHistory = getBeekeeperHistory(EXPIRED); + System.out.println(beekeeperHistory); + } + @Test public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java index 5f263448..c3d56db8 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java @@ -41,6 +41,8 @@ private CommonTestVariables() {} public static final String CLEANUP_ATTEMPTS_FIELD = "cleanup_attempts"; public static final String CLIENT_ID_FIELD = "client_id"; public static final String LIFECYCLE_TYPE_FIELD = "lifecycle_type"; + public static final String EVENT_DETAILS_FIELD = "event_details"; + public static final String EVENT_TIMESTAMP_FIELD = "event_timestamp"; // HOUSEKEEPINGENTITY DEFAULT VALUES public static final String DATABASE_NAME_VALUE = "some_database"; diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java new file mode 100644 index 00000000..46bec68b --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java @@ -0,0 +1,27 @@ +package com.expediagroup.beekeeper.integration.utils; + +import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_DETAILS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.HOUSEKEEPING_STATUS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.ID_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.LIFECYCLE_TYPE_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_FIELD; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; + +public class ResultSetToBeekeeperHistoryMapper { + + public static BeekeeperHistory mapToBeekeeperHistory(ResultSet resultSet) throws SQLException { + return BeekeeperHistory.builder() + .id(resultSet.getLong(ID_FIELD)) + .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) + .tableName(resultSet.getString(TABLE_NAME_FIELD)) + .lifecycleType(resultSet.getString(LIFECYCLE_TYPE_FIELD)) + .housekeepingStatus(resultSet.getString(HOUSEKEEPING_STATUS_FIELD)) + .eventDetails(resultSet.getString(EVENT_DETAILS_FIELD)) + .build(); + } +}