diff --git a/CHANGELOG.md b/CHANGELOG.md index 883a71dc..1ecc9bdf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.6.0] - 2024-11-29 +## Added +- Added a db migration file and implementation of `beekeeper-history` table to track beekeeper activity. + ## [3.5.8] - 2024-11-28 ### Added - Added `IcebergTableListenerEventFilter` filter for Iceberg tables in `beekeeper-scheduler-apiary` to prevent scheduling paths and metadata for deletion. diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 4496df03..ffd944b1 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -44,6 +44,27 @@ 27.1-jre + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + io.micrometer diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java index cd9c22a7..f945b54a 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java @@ -18,6 +18,8 @@ public enum HousekeepingStatus { SCHEDULED, FAILED, + FAILED_TO_DELETE, + FAILED_TO_SCHEDULE, DELETED, DISABLED, SKIPPED diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java new file mode 100644 index 00000000..a598d7e9 --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java @@ -0,0 +1,73 @@ +package com.expediagroup.beekeeper.core.model.history; + +import java.time.LocalDateTime; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import com.expediagroup.beekeeper.core.monitoring.MetricTag; +import com.expediagroup.beekeeper.core.monitoring.Taggable; + +@Data +@NoArgsConstructor +@Entity +@Table(name = "beekeeper_history") +public class BeekeeperHistory implements Taggable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @EqualsAndHashCode.Exclude + @Column(name = "event_timestamp", nullable = false, updatable = false) + private LocalDateTime eventTimestamp; + + @Column(name = "database_name", nullable = false) + private String databaseName; + + @Column(name = "table_name", nullable = false) + private String tableName; + + @Column(name = "lifecycle_type", nullable = false) + private String lifecycleType; + + @Column(name = "housekeeping_status", nullable = false) + private String housekeepingStatus; + + @Column(name = "event_details", columnDefinition = "TEXT") + private String eventDetails; + + @Builder + public BeekeeperHistory( + Long id, + LocalDateTime eventTimestamp, + String databaseName, + String tableName, + String lifecycleType, + String housekeepingStatus, + String eventDetails + ) { + this.id = id; + this.eventTimestamp = eventTimestamp; + this.databaseName = databaseName; + this.tableName = tableName; + this.lifecycleType = lifecycleType; + this.housekeepingStatus = housekeepingStatus; + this.eventDetails = eventDetails; + } + + @Override + public MetricTag getMetricTag() { + return new MetricTag("table", String.join(".", databaseName, tableName)); + } + +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java new file mode 100644 index 00000000..55c59e6b --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java @@ -0,0 +1,19 @@ +package com.expediagroup.beekeeper.core.repository; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.data.repository.query.Param; + +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; + +public interface BeekeeperHistoryRepository extends PagingAndSortingRepository, + JpaSpecificationExecutor { + + @Query(value = "from BeekeeperHistory t where t.lifecycleType = :lifecycle") + Slice findRecordsByLifecycleType( + @Param("lifecycle") String lifecycle, + Pageable pageable); +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java new file mode 100644 index 00000000..dc4b7427 --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -0,0 +1,37 @@ +package com.expediagroup.beekeeper.core.service; + +import java.time.LocalDateTime; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; + +public class BeekeeperHistoryService { + + private static final Logger log = LoggerFactory.getLogger(BeekeeperHistoryService.class); + + private final BeekeeperHistoryRepository beekeeperHistoryRepository; + + public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + this.beekeeperHistoryRepository = beekeeperHistoryRepository; + } + + public void saveHistory(HousekeepingEntity housekeepingEntity, HousekeepingStatus status) { + BeekeeperHistory event = BeekeeperHistory.builder() + .id(housekeepingEntity.getId()) + .eventTimestamp(LocalDateTime.now()) + .databaseName(housekeepingEntity.getDatabaseName()) + .tableName(housekeepingEntity.getTableName()) + .lifecycleType(housekeepingEntity.getLifecycleType()) + .housekeepingStatus(status.name()) + .eventDetails(housekeepingEntity.toString()) + .build(); + + log.info("Saving activity in Beekeeper History table; {}", event); + beekeeperHistoryRepository.save(event); + } +} diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java new file mode 100644 index 00000000..2c2993cd --- /dev/null +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java @@ -0,0 +1,141 @@ +package com.expediagroup.beekeeper.core.repository; + +import static org.assertj.core.api.Assertions.assertThat; + +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; + +import org.assertj.core.util.Lists; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import com.expediagroup.beekeeper.core.TestApplication; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; +import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; + +@ExtendWith(SpringExtension.class) +@TestPropertySource(properties = { + "hibernate.data-source.driver-class-name=org.h2.Driver", + "hibernate.dialect=org.hibernate.dialect.H2Dialect", + "hibernate.hbm2ddl.auto=create", + "spring.jpa.show-sql=true", + "spring.datasource.url=jdbc:h2:mem:beekeeper;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE;MODE=MySQL" }) +@ContextConfiguration(classes = { TestApplication.class }, loader = AnnotationConfigContextLoader.class) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +public class BeekeeperHistoryRepositoryTest { + + protected static final String DATABASE_NAME = "database"; + protected static final String TABLE_NAME = "table"; + protected static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); + protected static final LocalDateTime COLUMN_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + protected static final LocalDateTime EVENT_TIMESTAMP = COLUMN_TIMESTAMP.plus(CLEANUP_DELAY); + + private static final int PAGE = 0; + private static final int PAGE_SIZE = 500; + + @Autowired + private BeekeeperHistoryRepository repository; + + @BeforeEach + public void setupDb() { + repository.deleteAll(); + } + + @Test + public void typicalSave() { + BeekeeperHistory expiredEntry = createExpiredEvent(SCHEDULED); + BeekeeperHistory unreferencedEntry = createUnreferencedEvent(SCHEDULED); + + repository.save(expiredEntry); + repository.save(unreferencedEntry); + + List historyList = Lists.newArrayList( + repository.findRecordsByLifecycleType("EXPIRED", PageRequest.of(PAGE, PAGE_SIZE))); + assertThat(historyList.size()).isEqualTo(1); + + historyList = Lists.newArrayList( + repository.findRecordsByLifecycleType("UNREFERENCED", PageRequest.of(PAGE, PAGE_SIZE))); + assertThat(historyList.size()).isEqualTo(1); + } + + @Test + public void expired_multipleStatuses() { + BeekeeperHistory scheduledEntry = createExpiredEvent(SCHEDULED); + BeekeeperHistory deletedEntry = createExpiredEvent(DELETED); + BeekeeperHistory failedEntry = createExpiredEvent(FAILED); + + repository.save(scheduledEntry); + repository.save(deletedEntry); + repository.save(failedEntry); + + List historyList = Lists.newArrayList( + repository.findRecordsByLifecycleType("EXPIRED", PageRequest.of(PAGE, PAGE_SIZE))); + assertThat(historyList.size()).isEqualTo(3); + } + + @Test + public void unreferenced_multipleStatuses() { + BeekeeperHistory scheduledEntry = createUnreferencedEvent(SCHEDULED); + BeekeeperHistory deletedEntry = createUnreferencedEvent(DELETED); + BeekeeperHistory failedEntry = createUnreferencedEvent(FAILED); + + repository.save(scheduledEntry); + repository.save(deletedEntry); + repository.save(failedEntry); + + List historyList = Lists.newArrayList( + repository.findRecordsByLifecycleType("UNREFERENCED", PageRequest.of(PAGE, PAGE_SIZE))); + assertThat(historyList.size()).isEqualTo(3); + } + + protected BeekeeperHistory createExpiredEvent(HousekeepingStatus status) { + HousekeepingMetadata entity = HousekeepingMetadata.builder() + .cleanupAttempts(3) + .cleanupDelay(PeriodDuration.parse("P1D")) + .partitionName("event_date") + .creationTimestamp(COLUMN_TIMESTAMP) + .modifiedTimestamp(COLUMN_TIMESTAMP) + .build(); + + return createHistoryEntry("EXPIRED", status, entity.toString()); + } + + protected BeekeeperHistory createUnreferencedEvent(HousekeepingStatus status) { + HousekeepingPath entity = HousekeepingPath.builder() + .cleanupAttempts(3) + .cleanupDelay(PeriodDuration.parse("P1D")) + .creationTimestamp(COLUMN_TIMESTAMP) + .modifiedTimestamp(COLUMN_TIMESTAMP) + .build(); + + return createHistoryEntry("UNREFERENCED", status, entity.toString()); + } + + protected BeekeeperHistory createHistoryEntry(String lifecycleType, HousekeepingStatus status, + String eventDetails) { + return BeekeeperHistory.builder() + .eventTimestamp(EVENT_TIMESTAMP) + .databaseName(DATABASE_NAME) + .tableName(TABLE_NAME) + .lifecycleType(lifecycleType) + .housekeepingStatus(status.name()) + .eventDetails(eventDetails) + .build(); + } +} diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java new file mode 100644 index 00000000..4037bd36 --- /dev/null +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java @@ -0,0 +1,110 @@ +package com.expediagroup.beekeeper.core.service; + +import static org.mockito.Mockito.verify; + +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.fasterxml.jackson.core.JsonProcessingException; + +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; + +@ExtendWith(MockitoExtension.class) +public class BeekeeperHistoryServiceTest { + + private BeekeeperHistoryService beekeeperHistoryService; + + private @Mock BeekeeperHistoryRepository repository; + + private static final String DATABASE = "database"; + private static final String TABLE_NAME = "tableName"; + private static final String VALID_TABLE_PATH = "s3://bucket/table"; + private static final String VALID_PARTITION_PATH = "s3://bucket/table/partition"; + private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; + private static final LocalDateTime CLEANUP_INSTANCE = LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC); + private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); + + @BeforeEach + public void setup() { + beekeeperHistoryService = new BeekeeperHistoryService(repository); + } + + @Test + void expiredHistory() throws JsonProcessingException { + HousekeepingMetadata metadata = createHousekeepingMetadata(); + String details = createEventDetails(metadata); + BeekeeperHistory history = createHistoryEvent(metadata, details, "DELETED"); + + beekeeperHistoryService.saveHistory(metadata, DELETED); + verify(repository).save(history); + } + + @Test + void unreferencedHistory() throws JsonProcessingException { + HousekeepingPath path = createHousekeepingPath(); + String details = createEventDetails(path); + BeekeeperHistory history = createHistoryEvent(path, details, "SCHEDULED"); + + beekeeperHistoryService.saveHistory(path, SCHEDULED); + verify(repository).save(history); + } + + private BeekeeperHistory createHistoryEvent(HousekeepingEntity entity, String eventDetails, String status) { + return BeekeeperHistory.builder() + .id(entity.getId()) + .databaseName(entity.getDatabaseName()) + .tableName(entity.getTableName()) + .lifecycleType(entity.getLifecycleType()) + .housekeepingStatus(status) + .eventDetails(eventDetails) + .build(); + } + + private HousekeepingMetadata createHousekeepingMetadata() { + return HousekeepingMetadata.builder() + .path(VALID_TABLE_PATH) + .databaseName(DATABASE) + .tableName(TABLE_NAME) + .partitionName(PARTITION_NAME) + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CLEANUP_INSTANCE) + .cleanupDelay(CLEANUP_DELAY) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.name()) + .build(); + } + + private HousekeepingPath createHousekeepingPath() { + return HousekeepingPath.builder() + .path(VALID_PARTITION_PATH) + .databaseName(DATABASE) + .tableName(TABLE_NAME) + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CLEANUP_INSTANCE) + .cleanupDelay(CLEANUP_DELAY) + .cleanupAttempts(0) + .lifecycleType(UNREFERENCED.name()) + .build(); + } + + private String createEventDetails(HousekeepingEntity housekeepingEntity) { + return housekeepingEntity.toString(); + } +} diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index 66382669..c1be07e1 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -59,6 +59,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.model.AddPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage; @@ -224,6 +225,21 @@ public void expiredMetadataCreateIcebergTableEvent() throws SQLException, IOExce assertThat(expiredMetadata.size()).isEqualTo(0); } + @Test + public void testEventAddedToHistoryTable() throws SQLException, IOException, URISyntaxException { + CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true); + amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1); + + List beekeeperHistory = getBeekeeperHistory(EXPIRED); + BeekeeperHistory history = beekeeperHistory.get(0); + assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); + assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); + assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString()); + assertThat(history.getHousekeepingStatus()).isEqualTo(SCHEDULED.name()); + } + @Test public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 6967cbb4..6060f5cf 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -30,6 +30,8 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.CREATION_TIMESTAMP_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_DETAILS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_TIMESTAMP_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.HOUSEKEEPING_STATUS_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.ID_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.LIFECYCLE_TYPE_FIELD; @@ -39,6 +41,7 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.SHORT_CLEANUP_DELAY_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; +import static com.expediagroup.beekeeper.integration.utils.ResultSetToBeekeeperHistoryMapper.mapToBeekeeperHistory; import static com.expediagroup.beekeeper.integration.utils.ResultSetToHousekeepingEntityMapper.mapToHousekeepingMetadata; import static com.expediagroup.beekeeper.integration.utils.ResultSetToHousekeepingEntityMapper.mapToHousekeepingPath; @@ -63,6 +66,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.integration.utils.MySqlTestUtils; @@ -86,6 +90,7 @@ public abstract class BeekeeperIntegrationTestBase { private static final String BEEKEEPER_FLYWAY_TABLE = "flyway_schema_history"; private static final String BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME = "housekeeping_path"; private static final String BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME = "housekeeping_metadata"; + private static final String BEEKEEPER_HISTORY_TABLE_NAME = "beekeeper_history"; // FIELDS TO INSERT INTO BEEKEEPER TABLES private Long id = 1L; @@ -97,6 +102,8 @@ public abstract class BeekeeperIntegrationTestBase { .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, PARTITION_NAME_FIELD, HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); + private static final String BEEKEEPER_HISTORY_FIELDS = String.join(",", ID_FIELD, EVENT_TIMESTAMP_FIELD, + DATABASE_NAME_FIELD, TABLE_NAME_FIELD, LIFECYCLE_TYPE_FIELD, HOUSEKEEPING_STATUS_FIELD, EVENT_DETAILS_FIELD); private static final String LIFE_CYCLE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD; private static final String LIFE_CYCLE_AND_UPDATE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD @@ -148,6 +155,7 @@ public void dropMySQLTables() throws SQLException { mySQLTestUtils.dropTable(BEEKEEPER_DB_NAME, BEEKEEPER_FLYWAY_TABLE); mySQLTestUtils.dropTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME); mySQLTestUtils.dropTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME); + mySQLTestUtils.dropTable(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME); } protected void insertUnreferencedPath(String path) throws SQLException { @@ -176,7 +184,7 @@ protected void insertExpiredMetadata(String path, String partitionName) throws S } protected void insertExpiredMetadata(String tableName, String path, String partitionName, String cleanupDelay) - throws SQLException { + throws SQLException { HousekeepingMetadata metadata = createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay); insertExpiredMetadata(metadata); } @@ -214,6 +222,13 @@ protected int getUpdatedExpiredMetadataRowCount() throws SQLException { format(LIFE_CYCLE_AND_UPDATE_FILTER, EXPIRED)); } + protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) throws SQLException { + String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; + + return mySQLTestUtils.getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, + format(filter, lifecycleEventType)); + } + protected List getUnreferencedPaths() throws SQLException { List paths = new ArrayList<>(); ResultSet resultSet = mySQLTestUtils @@ -240,6 +255,19 @@ protected List getExpiredMetadata() throws SQLException { return metadata; } + protected List getBeekeeperHistory(LifecycleEventType lifecycleEventType) throws SQLException { + String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; + List history = new ArrayList<>(); + ResultSet resultSet = mySQLTestUtils.getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, + format(filter, lifecycleEventType)); + + while (resultSet.next()) { + history.add(mapToBeekeeperHistory(resultSet)); + } + + return history; + } + private HousekeepingPath createHousekeepingPath(String path, LifecycleEventType lifecycleEventType) { return HousekeepingPath .builder() @@ -279,5 +307,4 @@ private HousekeepingMetadata createHousekeepingMetadata( .clientId(CLIENT_ID_FIELD) .build(); } - } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java index e45ee0b9..fdaff3f4 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java @@ -27,6 +27,7 @@ import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DISABLED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.LONG_CLEANUP_DELAY_VALUE; @@ -69,6 +70,8 @@ import com.google.common.collect.ImmutableMap; import com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter; +import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.integration.utils.HiveTestUtils; import com.expediagroup.beekeeper.metadata.cleanup.BeekeeperMetadataCleanup; @@ -389,6 +392,27 @@ public void onlyCleanupLocationWhenPartitionExists() throws TException, SQLExcep assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_OBJECT_KEY)).isTrue(); } + @Test + public void testEventAddedToHistoryTable() throws TException, SQLException { + hiveTestUtils.createTable(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false); + amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); + + insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> getBeekeeperHistoryRowCount(LifecycleEventType.EXPIRED) == 1); + + assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isFalse(); + assertThat(amazonS3.doesObjectExist(BUCKET, UNPARTITIONED_OBJECT_KEY)).isFalse(); + + List beekeeperHistory = getBeekeeperHistory(EXPIRED); + BeekeeperHistory history = beekeeperHistory.get(0); + assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); + assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); + assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString()); + assertThat(history.getHousekeepingStatus()).isEqualTo(DELETED.name()); + } + @Test public void metrics() throws Exception { Table table = hiveTestUtils.createTable(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java index 0b8c29b0..8f4c4c17 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java @@ -21,6 +21,7 @@ import static com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter.METRIC_NAME; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; @@ -52,6 +53,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.path.cleanup.BeekeeperPathCleanup; @@ -244,6 +246,30 @@ public void cleanupSentinelForNonEmptyParent() throws SQLException { assertThat(amazonS3.doesObjectExist(BUCKET, tableSentinel)).isTrue(); } + @Test + public void testEventAddedToHistoryTable() throws SQLException { + amazonS3.putObject(BUCKET, OBJECT_KEY1, CONTENT); + amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER, CONTENT); + amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, ""); + + String path = "s3://" + BUCKET + "/" + OBJECT_KEY1; + insertUnreferencedPath(path); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + + assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); + // deleting a file shouldn't delete a folder sentinel + assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY_SENTINEL)).isTrue(); + assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY_OTHER)).isTrue(); + + List beekeeperHistory = getBeekeeperHistory(UNREFERENCED); + BeekeeperHistory history = beekeeperHistory.get(0); + assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); + assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); + assertThat(history.getLifecycleType()).isEqualTo(UNREFERENCED.toString()); + assertThat(history.getHousekeepingStatus()).isEqualTo(DELETED.name()); + } + @Test public void metrics() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY1, CONTENT); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 1c3fd3a4..9d3678d1 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -59,6 +59,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.model.AlterPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage; import com.expediagroup.beekeeper.integration.model.DropPartitionSqsMessage; @@ -196,6 +197,22 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy assertUnreferencedPath(unreferencedPaths.get(1), "s3://bucket/tableLocation2"); } + @Test + public void testEventAddedToHistoryTable() throws IOException, URISyntaxException, SQLException { + AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", + "s3://bucket/oldTableLocation", true, true); + amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + + List beekeeperHistory = getBeekeeperHistory(UNREFERENCED); + BeekeeperHistory history = beekeeperHistory.get(0); + assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); + assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); + assertThat(history.getLifecycleType()).isEqualTo(UNREFERENCED.toString()); + assertThat(history.getHousekeepingStatus()).isEqualTo(SCHEDULED.name()); + } + @Test public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java index 5f263448..c3d56db8 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java @@ -41,6 +41,8 @@ private CommonTestVariables() {} public static final String CLEANUP_ATTEMPTS_FIELD = "cleanup_attempts"; public static final String CLIENT_ID_FIELD = "client_id"; public static final String LIFECYCLE_TYPE_FIELD = "lifecycle_type"; + public static final String EVENT_DETAILS_FIELD = "event_details"; + public static final String EVENT_TIMESTAMP_FIELD = "event_timestamp"; // HOUSEKEEPINGENTITY DEFAULT VALUES public static final String DATABASE_NAME_VALUE = "some_database"; diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java new file mode 100644 index 00000000..46bec68b --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java @@ -0,0 +1,27 @@ +package com.expediagroup.beekeeper.integration.utils; + +import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_DETAILS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.HOUSEKEEPING_STATUS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.ID_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.LIFECYCLE_TYPE_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_FIELD; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; + +public class ResultSetToBeekeeperHistoryMapper { + + public static BeekeeperHistory mapToBeekeeperHistory(ResultSet resultSet) throws SQLException { + return BeekeeperHistory.builder() + .id(resultSet.getLong(ID_FIELD)) + .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) + .tableName(resultSet.getString(TABLE_NAME_FIELD)) + .lifecycleType(resultSet.getString(LIFECYCLE_TYPE_FIELD)) + .housekeepingStatus(resultSet.getString(HOUSEKEEPING_STATUS_FIELD)) + .eventDetails(resultSet.getString(EVENT_DETAILS_FIELD)) + .build(); + } +} diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java index 60518cd0..60edb667 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java @@ -49,7 +49,9 @@ import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; import com.expediagroup.beekeeper.cleanup.validation.IcebergValidator; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.metadata.cleanup.handler.ExpiredMetadataHandler; import com.expediagroup.beekeeper.metadata.cleanup.handler.MetadataHandler; import com.expediagroup.beekeeper.metadata.cleanup.service.MetadataDisableTablesService; @@ -153,9 +155,15 @@ public ExpiredMetadataHandler expiredMetadataHandler( @Qualifier("hiveClientFactory") CleanerClientFactory cleanerClientFactory, HousekeepingMetadataRepository housekeepingMetadataRepository, @Qualifier("hiveTableCleaner") MetadataCleaner metadataCleaner, - @Qualifier("s3PathCleaner") PathCleaner pathCleaner) { + @Qualifier("s3PathCleaner") PathCleaner pathCleaner, + BeekeeperHistoryService beekeeperHistoryService) { return new ExpiredMetadataHandler(cleanerClientFactory, housekeepingMetadataRepository, metadataCleaner, - pathCleaner); + pathCleaner, beekeeperHistoryService); + } + + @Bean + BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + return new BeekeeperHistoryService(beekeeperHistoryRepository); } @Bean diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java index 39fe7d5e..c202cf86 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java @@ -19,6 +19,7 @@ import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_DELETE; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; import java.time.LocalDateTime; @@ -36,6 +37,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.core.validation.S3PathValidator; public class ExpiredMetadataHandler implements MetadataHandler { @@ -46,16 +48,19 @@ public class ExpiredMetadataHandler implements MetadataHandler { private final HousekeepingMetadataRepository housekeepingMetadataRepository; private final MetadataCleaner metadataCleaner; private final PathCleaner pathCleaner; + private final BeekeeperHistoryService historyService; public ExpiredMetadataHandler( CleanerClientFactory cleanerClientFactory, HousekeepingMetadataRepository housekeepingMetadataRepository, MetadataCleaner metadataCleaner, - PathCleaner pathCleaner) { + PathCleaner pathCleaner, + BeekeeperHistoryService historyService) { this.cleanerClientFactory = cleanerClientFactory; this.housekeepingMetadataRepository = housekeepingMetadataRepository; this.metadataCleaner = metadataCleaner; this.pathCleaner = pathCleaner; + this.historyService = historyService; } @Override @@ -77,6 +82,7 @@ public void cleanupMetadata(HousekeepingMetadata housekeepingMetadata, LocalDate boolean deleted = cleanup(client, housekeepingMetadata, instant, dryRunEnabled); if (deleted && !dryRunEnabled) { updateAttemptsAndStatus(housekeepingMetadata, DELETED); + saveHistory(housekeepingMetadata, DELETED, dryRunEnabled); } } catch (BeekeeperIcebergException e) { updateAttemptsAndStatus(housekeepingMetadata, SKIPPED); @@ -90,6 +96,10 @@ public void cleanupMetadata(HousekeepingMetadata housekeepingMetadata, LocalDate housekeepingMetadata.getDatabaseName(), housekeepingMetadata.getTableName()); log.info(logMessage); log.debug(logMessage, e); + saveHistory(housekeepingMetadata, FAILED_TO_DELETE, dryRunEnabled); + log + .warn("Unexpected exception when deleting metadata for table \"{}.{}\"", + housekeepingMetadata.getDatabaseName(), housekeepingMetadata.getTableName(), e); } } @@ -115,6 +125,7 @@ private boolean cleanUpTable(CleanerClient client, HousekeepingMetadata housekee if (!S3PathValidator.validTablePath(housekeepingMetadata.getPath())) { log.warn("Will not clean up table path \"{}\" because it is not valid.", housekeepingMetadata.getPath()); updateStatus(housekeepingMetadata, SKIPPED, dryRunEnabled); + saveHistory(housekeepingMetadata, SKIPPED, dryRunEnabled); return false; } String databaseName = housekeepingMetadata.getDatabaseName(); @@ -136,6 +147,7 @@ private boolean cleanupPartition( if (!S3PathValidator.validPartitionPath(housekeepingMetadata.getPath())) { log.warn("Will not clean up partition path \"{}\" because it is not valid.", housekeepingMetadata.getPath()); updateStatus(housekeepingMetadata, SKIPPED, dryRunEnabled); + saveHistory(housekeepingMetadata, SKIPPED, dryRunEnabled); return false; } String databaseName = housekeepingMetadata.getDatabaseName(); @@ -183,4 +195,12 @@ private Long countPartitionsForDatabaseAndTable( return housekeepingMetadataRepository .countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(databaseName, tableName); } + + private void saveHistory(HousekeepingMetadata metadata, HousekeepingStatus housekeepingStatus, + boolean dryRunEnabled) { + if (dryRunEnabled) { + return; + } + historyService.saveHistory(metadata, housekeepingStatus); + } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java index e2323868..5f5ec9e6 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java @@ -50,7 +50,9 @@ import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; import com.expediagroup.beekeeper.cleanup.validation.IcebergValidator; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.metadata.cleanup.handler.ExpiredMetadataHandler; import com.expediagroup.beekeeper.metadata.cleanup.service.MetadataDisableTablesService; import com.expediagroup.beekeeper.metadata.cleanup.service.MetadataRepositoryCleanupService; @@ -78,6 +80,8 @@ public class CommonBeansTest { private @Mock MeterRegistry meterRegistry; private @Mock HiveClientFactory hiveClientFactory; private @Mock IcebergValidator icebergValidator; + private @Mock BeekeeperHistoryService beekeeperHistoryService; + private @Mock BeekeeperHistoryRepository beekeeperHistoryRepository; @BeforeEach public void awsSetUp() { @@ -161,7 +165,7 @@ void verifyS3pathCleaner() { @Test public void verifyExpiredMetadataHandler() { ExpiredMetadataHandler expiredMetadataHandler = commonBeans.expiredMetadataHandler(hiveClientFactory, - metadataRepository, metadataCleaner, pathCleaner); + metadataRepository, metadataCleaner, pathCleaner, beekeeperHistoryService); assertThat(expiredMetadataHandler).isInstanceOf(ExpiredMetadataHandler.class); } @@ -170,7 +174,8 @@ public void verifyCleanupService() { HiveClientFactory hiveClientFactory = Mockito.mock(HiveClientFactory.class); CleanupService cleanupService = commonBeans.cleanupService( List.of( - commonBeans.expiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner)), 2, + commonBeans.expiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner, + beekeeperHistoryService)), 2, false); assertThat(cleanupService).isInstanceOf(PagingMetadataCleanupService.class); } @@ -187,4 +192,10 @@ public void verifyDisableTablesService() { metadataRepository, hiveClientFactory, false); assertThat(disableTablesService).isInstanceOf(MetadataDisableTablesService.class); } + + @Test + public void verifyBeekeeperHistoryService(){ + BeekeeperHistoryService beekeeperHistoryService = commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); + assertThat(beekeeperHistoryService).isInstanceOf(BeekeeperHistoryService.class); + } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java index 1a4a1391..1790d4c5 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java @@ -16,6 +16,8 @@ package com.expediagroup.beekeeper.metadata.cleanup.handler; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -23,6 +25,7 @@ import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_DELETE; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; @@ -46,6 +49,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class ExpiredMetadataHandlerTest { @@ -56,6 +60,7 @@ public class ExpiredMetadataHandlerTest { private @Mock HiveMetadataCleaner hiveMetadataCleaner; private @Mock S3PathCleaner s3PathCleaner; private @Mock HousekeepingMetadata housekeepingMetadata; + private @Mock BeekeeperHistoryService beekeeperHistoryService; private static final LifecycleEventType lifecycleEventType = EXPIRED; private static final String DATABASE = "database"; @@ -71,8 +76,7 @@ public class ExpiredMetadataHandlerTest { @BeforeEach public void init() { expiredMetadataHandler = new ExpiredMetadataHandler(hiveClientFactory, housekeepingMetadataRepository, - hiveMetadataCleaner, - s3PathCleaner); + hiveMetadataCleaner, s3PathCleaner, beekeeperHistoryService); } @Test @@ -119,6 +123,27 @@ public void typicalRunDroppingTable() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); + } + + @Test + public void typicalDroppingTable_DryRun() { + boolean dryRunEnabled = true; + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); + when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); + when(housekeepingMetadata.getPartitionName()).thenReturn(null); + when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); + when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); + + expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, dryRunEnabled); + verify(hiveMetadataCleaner).dropTable(housekeepingMetadata, hiveClient); + verify(s3PathCleaner).cleanupPath(housekeepingMetadata); + verify(hiveMetadataCleaner, never()).dropPartition(housekeepingMetadata, hiveClient); + verify(housekeepingMetadata, never()).setCleanupAttempts(1); + verify(housekeepingMetadata, never()).setHousekeepingStatus(DELETED); + verify(housekeepingMetadataRepository, never()).save(housekeepingMetadata); + verify(beekeeperHistoryService, never()).saveHistory(any(), any()); } @Test @@ -129,6 +154,7 @@ public void typicalRunDroppingPartition() { when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); + when(hiveMetadataCleaner.dropPartition(Mockito.any(), Mockito.any())).thenReturn(true); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); @@ -138,6 +164,7 @@ public void typicalRunDroppingPartition() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -158,6 +185,26 @@ public void dontDropTableWithInvalidPath() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED)); + } + + @Test + public void dontDropTableWithInvalidPath_DryRun() { + boolean dryRunEnabled = true; + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); + when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); + when(housekeepingMetadata.getPartitionName()).thenReturn(null); + when(housekeepingMetadata.getPath()).thenReturn(INVALID_PATH); + + expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, dryRunEnabled); + verify(hiveMetadataCleaner, never()).dropTable(housekeepingMetadata, hiveClient); + verify(s3PathCleaner, never()).cleanupPath(housekeepingMetadata); + verify(hiveMetadataCleaner, never()).dropPartition(housekeepingMetadata, hiveClient); + verify(housekeepingMetadata, never()).setCleanupAttempts(1); + verify(housekeepingMetadata, never()).setHousekeepingStatus(SKIPPED); + verify(housekeepingMetadataRepository, never()).save(housekeepingMetadata); + verify(beekeeperHistoryService, never()).saveHistory(any(), any()); } @Test @@ -200,6 +247,7 @@ public void dontDropTableOrPathWhenTableDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -215,6 +263,7 @@ public void dontDropPartitionWithInvalidPartitionPath() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED)); } @Test @@ -234,6 +283,7 @@ public void dontDropPartitionWhenTableDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -252,6 +302,7 @@ public void dontDropPathWhenPartitionDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -260,7 +311,7 @@ public void expectedTableDropFailure() { when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); when(housekeepingMetadata.getPartitionName()).thenReturn(null); - when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH);; + when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); when( housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, @@ -273,6 +324,7 @@ public void expectedTableDropFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE)); } @Test @@ -290,6 +342,7 @@ public void expectedPathDeleteFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE)); } @Test @@ -307,5 +360,6 @@ public void expectedPartitionDropFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE)); } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java index 00f8a762..e0b2ac27 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java @@ -46,6 +46,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Captor; +import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; @@ -66,6 +67,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.metadata.cleanup.TestApplication; import com.expediagroup.beekeeper.metadata.cleanup.handler.ExpiredMetadataHandler; import com.expediagroup.beekeeper.metadata.cleanup.handler.MetadataHandler; @@ -89,6 +91,7 @@ public class PagingMetadataCleanupServiceTest { private @MockBean PathCleaner pathCleaner; private @MockBean HiveClientFactory hiveClientFactory; private @MockBean HiveClient hiveClient; + private @Mock BeekeeperHistoryService beekeeperHistoryService; private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; @@ -103,7 +106,8 @@ public void init() { properties.put(UNREFERENCED.getTableParameterName(), "true"); when(hiveClient.getTableProperties(Mockito.any(), Mockito.any())).thenReturn(properties); when(hiveClientFactory.newInstance()).thenReturn(hiveClient); - handler = new ExpiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner); + handler = new ExpiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner, + beekeeperHistoryService); handlers = List.of(handler); pagingCleanupService = new PagingMetadataCleanupService(handlers, 2, false); } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java index 0b118243..605ddf9f 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java @@ -40,7 +40,9 @@ import com.expediagroup.beekeeper.cleanup.service.CleanupService; import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.path.cleanup.handler.GenericPathHandler; import com.expediagroup.beekeeper.path.cleanup.service.PagingPathCleanupService; import com.expediagroup.beekeeper.path.cleanup.service.PathRepositoryCleanupService; @@ -108,4 +110,9 @@ RepositoryCleanupService repositoryCleanupService( DisableTablesService disableTablesService() { return () -> {}; } + + @Bean + BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + return new BeekeeperHistoryService(beekeeperHistoryRepository); + } } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java index 30442c22..e9a440d4 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java @@ -15,6 +15,10 @@ */ package com.expediagroup.beekeeper.path.cleanup.handler; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_DELETE; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; + import java.time.LocalDateTime; import java.util.List; @@ -27,6 +31,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.core.validation.S3PathValidator; public abstract class GenericPathHandler { @@ -35,10 +40,13 @@ public abstract class GenericPathHandler { private final HousekeepingPathRepository housekeepingPathRepository; private final PathCleaner pathCleaner; + private final BeekeeperHistoryService beekeeperHistoryService; - public GenericPathHandler(HousekeepingPathRepository housekeepingPathRepository, PathCleaner pathCleaner) { + public GenericPathHandler(HousekeepingPathRepository housekeepingPathRepository, PathCleaner pathCleaner, + BeekeeperHistoryService beekeeperHistoryService) { this.housekeepingPathRepository = housekeepingPathRepository; this.pathCleaner = pathCleaner; + this.beekeeperHistoryService = beekeeperHistoryService; } public abstract Slice findRecordsToClean(LocalDateTime instant, Pageable pageable); @@ -49,11 +57,11 @@ public GenericPathHandler(HousekeepingPathRepository housekeepingPathRepository, * @param pageable Pageable to iterate through for dryRun * @param page Page to get content from * @param dryRunEnabled Dry Run boolean flag + * @return Pageable to pass to query. In the case of dry runs, this is the next page. * @implNote This parent handler expects the child's cleanupPath call to update & remove the record from this call - * such that subsequent DB queries will not return the record. Hence why we only call next during dryRuns - * where no updates occur. + * such that subsequent DB queries will not return the record. Hence why we only call next during dryRuns + * where no updates occur. * @implNote Note that we only expect pageable.next to be called during a dry run. - * @return Pageable to pass to query. In the case of dry runs, this is the next page. */ public Pageable processPage(Pageable pageable, Slice page, boolean dryRunEnabled) { List pageContent = page.getContent(); @@ -79,12 +87,14 @@ private void cleanupContent(HousekeepingPath housekeepingPath) { try { log.info("Cleaning up path \"{}\"", housekeepingPath.getPath()); if (cleanUpPath(housekeepingPath)) { - updateAttemptsAndStatus(housekeepingPath, HousekeepingStatus.DELETED); + updateAttemptsAndStatus(housekeepingPath, DELETED); + saveHistory(housekeepingPath, DELETED); } else { - updateStatus(housekeepingPath, HousekeepingStatus.SKIPPED); + updateStatus(housekeepingPath, SKIPPED); } } catch (Exception e) { updateAttemptsAndStatus(housekeepingPath, HousekeepingStatus.FAILED); + saveHistory(housekeepingPath, FAILED_TO_DELETE); log.warn("Unexpected exception deleting \"{}\"", housekeepingPath.getPath(), e); } } @@ -98,5 +108,10 @@ private void updateAttemptsAndStatus(HousekeepingPath housekeepingPath, Housekee private void updateStatus(HousekeepingPath housekeepingPath, HousekeepingStatus status) { housekeepingPath.setHousekeepingStatus(status); housekeepingPathRepository.save(housekeepingPath); + saveHistory(housekeepingPath, status); + } + + private void saveHistory(HousekeepingPath housekeepingPath, HousekeepingStatus status) { + beekeeperHistoryService.saveHistory(housekeepingPath, status); } } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandler.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandler.java index 82d68b69..8242e977 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandler.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandler.java @@ -26,6 +26,7 @@ import com.expediagroup.beekeeper.cleanup.path.PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @Component public class UnreferencedPathHandler extends GenericPathHandler { @@ -35,8 +36,9 @@ public class UnreferencedPathHandler extends GenericPathHandler { @Autowired public UnreferencedPathHandler( HousekeepingPathRepository housekeepingPathRepository, - @Qualifier("s3PathCleaner") PathCleaner pathCleaner) { - super(housekeepingPathRepository, pathCleaner); + @Qualifier("s3PathCleaner") PathCleaner pathCleaner, + BeekeeperHistoryService beekeeperHistoryService) { + super(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); this.housekeepingPathRepository = housekeepingPathRepository; } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeansTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeansTest.java index a6667b73..1a76b1c5 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeansTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeansTest.java @@ -40,7 +40,9 @@ import com.expediagroup.beekeeper.cleanup.service.CleanupService; import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.path.cleanup.service.PagingPathCleanupService; import com.expediagroup.beekeeper.path.cleanup.service.PathRepositoryCleanupService; @@ -59,6 +61,7 @@ class CommonBeansTest { private final CommonBeans commonBeans = new CommonBeans(); private @Mock HousekeepingPathRepository repository; private @Mock BytesDeletedReporter bytesDeletedReporter; + private @Mock BeekeeperHistoryRepository beekeeperHistoryRepository; @BeforeEach void setUp() { @@ -121,4 +124,10 @@ public void verifyDisableTablesService() { DisableTablesService disableTablesService = commonBeans.disableTablesService(); assertThat(disableTablesService).isNotNull(); } + + @Test + public void verifyBeekeeperHistoryService(){ + BeekeeperHistoryService beekeeperHistoryService = commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); + assertThat(beekeeperHistoryService).isInstanceOf(BeekeeperHistoryService.class); + } } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java index aa66d192..b9110c74 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java @@ -16,6 +16,8 @@ package com.expediagroup.beekeeper.path.cleanup.handler; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -23,6 +25,7 @@ import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_DELETE; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; import java.util.List; @@ -38,6 +41,7 @@ import com.expediagroup.beekeeper.cleanup.aws.S3PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class GenericPathHandlerTest { @@ -47,6 +51,8 @@ public class GenericPathHandlerTest { @Mock private S3PathCleaner pathCleaner; @Mock + private BeekeeperHistoryService beekeeperHistoryService; + @Mock private HousekeepingPath mockPath; @Mock private Pageable mockPageable; @@ -60,7 +66,7 @@ public class GenericPathHandlerTest { @BeforeEach public void initTest() { - handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); when(mockPath.getPath()).thenReturn(VALID_TABLE_PATH); } @@ -83,6 +89,7 @@ public void typicalProcessPage() { verify(mockPath).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(DELETED); verify(housekeepingPathRepository).save(mockPath); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); assertThat(pageable).isEqualTo(pageable); } @@ -96,6 +103,7 @@ public void processPageFails() { verify(mockPath).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(FAILED); verify(housekeepingPathRepository).save(mockPath); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE)); assertThat(pageable).isEqualTo(pageable); } @@ -109,6 +117,7 @@ public void processPageInvalidPath() { verify(mockPath, never()).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(SKIPPED); verify(housekeepingPathRepository).save(mockPath); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED)); assertThat(pageable).isEqualTo(pageable); } } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandlerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandlerTest.java index c00e9ca6..e911cc78 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandlerTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandlerTest.java @@ -33,6 +33,7 @@ import com.expediagroup.beekeeper.cleanup.aws.S3PathCleaner; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class UnreferencedPathHandlerTest { @@ -41,13 +42,16 @@ public class UnreferencedPathHandlerTest { private HousekeepingPathRepository housekeepingPathRepository; @Mock private S3PathCleaner s3PathCleaner; + @Mock + private BeekeeperHistoryService beekeeperHistoryService; + private LifecycleEventType lifecycleEventType = UNREFERENCED; private UnreferencedPathHandler handler; @BeforeEach public void initTest() { - handler = new UnreferencedPathHandler(housekeepingPathRepository, s3PathCleaner); + handler = new UnreferencedPathHandler(housekeepingPathRepository, s3PathCleaner, beekeeperHistoryService); } @Test diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/PagingCleanupServiceTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/PagingCleanupServiceTest.java index c75b0d53..86030693 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/PagingCleanupServiceTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/PagingCleanupServiceTest.java @@ -54,6 +54,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.path.cleanup.TestApplication; import com.expediagroup.beekeeper.path.cleanup.handler.UnreferencedPathHandler; @@ -73,10 +74,11 @@ public class PagingCleanupServiceTest { private @Captor ArgumentCaptor pathCaptor; private @Autowired HousekeepingPathRepository housekeepingPathRepository; private @MockBean PathCleaner pathCleaner; + private @MockBean BeekeeperHistoryService beekeeperHistoryService; @Test public void typicalWithPaging() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 2, false); List paths = List.of("s3://bucket/some_foo", "s3://bucket/some_bar", "s3://bucket/some_foobar"); @@ -97,7 +99,7 @@ public void typicalWithPaging() { @Test public void mixOfScheduledAndFailedPaths() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 2, false); List paths = List .of(createEntityHousekeepingPath("s3://bucket/some_foo", SCHEDULED), @@ -113,7 +115,7 @@ public void mixOfScheduledAndFailedPaths() { @Test public void mixOfAllPaths() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 2, false); List paths = List .of(createEntityHousekeepingPath("s3://bucket/some_foo", SCHEDULED), @@ -130,7 +132,7 @@ public void mixOfAllPaths() { @Test void pathCleanerException() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 2, false); doThrow(new RuntimeException("Error")).doNothing().when(pathCleaner).cleanupPath(any(HousekeepingPath.class)); @@ -158,7 +160,7 @@ void pathCleanerException() { @Test @Timeout(value = 10) void doNotInfiniteLoopOnRepeatedFailures() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 1, false); List paths = List .of(createEntityHousekeepingPath("s3://bucket/some_foo", FAILED), @@ -186,7 +188,7 @@ void doNotInfiniteLoopOnRepeatedFailures() { @Test @Timeout(value = 10) void doNotInfiniteLoopOnDryRunCleanup() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 1, true); List paths = List .of(createEntityHousekeepingPath("s3://bucket/some_foo", SCHEDULED), diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java index 8bfbae44..35dd5247 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java @@ -38,6 +38,8 @@ import com.expedia.apiary.extensions.receiver.sqs.messaging.SqsMessageReader; import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.scheduler.apiary.filter.EventTypeListenerEventFilter; import com.expediagroup.beekeeper.scheduler.apiary.filter.IcebergTableListenerEventFilter; import com.expediagroup.beekeeper.scheduler.apiary.filter.ListenerEventFilter; @@ -142,4 +144,9 @@ public BeekeeperEventReader eventReader( return new MessageReaderAdapter(messageReader, handlers); } + + @Bean + BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + return new BeekeeperHistoryService(beekeeperHistoryRepository); + } } diff --git a/beekeeper-scheduler-apiary/src/main/resources/db/migration/V2_4__Create_history_table.sql b/beekeeper-scheduler-apiary/src/main/resources/db/migration/V2_4__Create_history_table.sql new file mode 100644 index 00000000..c8f2f977 --- /dev/null +++ b/beekeeper-scheduler-apiary/src/main/resources/db/migration/V2_4__Create_history_table.sql @@ -0,0 +1,15 @@ +USE beekeeper; + +CREATE TABLE IF NOT EXISTS beekeeper_history ( + id BIGINT(20) AUTO_INCREMENT, + event_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + database_name VARCHAR(512), + table_name VARCHAR(512), + lifecycle_type VARCHAR(255) NOT NULL, + housekeeping_status VARCHAR(50) NOT NULL, + event_details TEXT, + PRIMARY KEY (id) +); + +ALTER TABLE beekeeper_history ADD INDEX `beekeeper_history_index_table_name_upper` ((upper(table_name))); +ALTER TABLE beekeeper_history ADD INDEX `beekeeper_history_index_status` (`housekeeping_status`); diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java index cd1f776a..ec7e4cec 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java @@ -33,6 +33,8 @@ import com.expedia.apiary.extensions.receiver.sqs.messaging.SqsMessageReader; import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.scheduler.apiary.filter.IcebergTableListenerEventFilter; import com.expediagroup.beekeeper.scheduler.apiary.filter.ListenerEventFilter; import com.expediagroup.beekeeper.scheduler.apiary.generator.ExpiredHousekeepingMetadataGenerator; @@ -49,14 +51,12 @@ public class CommonBeansTest { private static final String AWS_S3_ENDPOINT_PROPERTY = "aws.s3.endpoint"; private static final String AWS_REGION_PROPERTY = "aws.region"; private static final String REGION = "us-west-2"; - private static final String AWS_ENDPOINT = String.join(".", "s3", REGION, "amazonaws.com"); private static final String ENDPOINT = "endpoint"; - private static final String BUCKET = "bucket"; - private static final String KEY = "key"; private final CommonBeans commonBeans = new CommonBeans(); - @Mock private MessageReader messageReader; - @Mock private UnreferencedHousekeepingPathGenerator unreferencedHousekeepingPathGenerator; - @Mock private ExpiredHousekeepingMetadataGenerator expiredHousekeepingMetadataGenerator; + private @Mock MessageReader messageReader; + private @Mock UnreferencedHousekeepingPathGenerator unreferencedHousekeepingPathGenerator; + private @Mock ExpiredHousekeepingMetadataGenerator expiredHousekeepingMetadataGenerator; + private @Mock BeekeeperHistoryRepository beekeeperHistoryRepository; @AfterAll static void tearDown() { @@ -123,15 +123,23 @@ public void validatePathEventReader() { @Test public void validateUnreferencedHousekeepingPathMessageEventHandlerIncludesIcebergFilter() { - MessageEventHandler handler = commonBeans.unreferencedHousekeepingPathMessageEventHandler(unreferencedHousekeepingPathGenerator); + MessageEventHandler handler = commonBeans.unreferencedHousekeepingPathMessageEventHandler( + unreferencedHousekeepingPathGenerator); List filters = handler.getFilters(); assertThat(filters).hasAtLeastOneElementOfType(IcebergTableListenerEventFilter.class); } @Test public void validateExpiredHousekeepingMetadataMessageEventHandlerIncludesIcebergFilter() { - MessageEventHandler handler = commonBeans.expiredHousekeepingMetadataMessageEventHandler(expiredHousekeepingMetadataGenerator); + MessageEventHandler handler = commonBeans.expiredHousekeepingMetadataMessageEventHandler( + expiredHousekeepingMetadataGenerator); List filters = handler.getFilters(); assertThat(filters).hasAtLeastOneElementOfType(IcebergTableListenerEventFilter.class); } + + @Test + public void verifyBeekeeperHistoryService() { + BeekeeperHistoryService beekeeperHistoryService = commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); + assertThat(beekeeperHistoryService).isInstanceOf(BeekeeperHistoryService.class); + } } diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java index cff64747..768d804c 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java @@ -17,6 +17,8 @@ import static java.lang.String.format; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_SCHEDULE; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import java.time.LocalDateTime; @@ -30,9 +32,11 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.monitoring.TimedTaggable; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @Service public class ExpiredHousekeepingMetadataSchedulerService implements SchedulerService { @@ -41,10 +45,13 @@ public class ExpiredHousekeepingMetadataSchedulerService implements SchedulerSer private static final LifecycleEventType LIFECYCLE_EVENT_TYPE = EXPIRED; private final HousekeepingMetadataRepository housekeepingMetadataRepository; + private final BeekeeperHistoryService beekeeperHistoryService; @Autowired - public ExpiredHousekeepingMetadataSchedulerService(HousekeepingMetadataRepository housekeepingMetadataRepository) { + public ExpiredHousekeepingMetadataSchedulerService(HousekeepingMetadataRepository housekeepingMetadataRepository, + BeekeeperHistoryService beekeeperHistoryService) { this.housekeepingMetadataRepository = housekeepingMetadataRepository; + this.beekeeperHistoryService = beekeeperHistoryService; } @Override @@ -60,7 +67,9 @@ public void scheduleForHousekeeping(HousekeepingEntity housekeepingEntity) { try { housekeepingMetadataRepository.save(housekeepingMetadata); log.info(format("Successfully scheduled %s", housekeepingMetadata)); + saveHistory(housekeepingMetadata, SCHEDULED); } catch (Exception e) { + saveHistory(housekeepingMetadata, FAILED_TO_SCHEDULE); throw new BeekeeperException(format("Unable to schedule %s", housekeepingMetadata), e); } } @@ -140,4 +149,8 @@ private boolean isPartitionedTable(HousekeepingMetadata housekeepingMetadata) { return numPartitions > 0 && housekeepingMetadata.getPartitionName() == null; } + + private void saveHistory(HousekeepingMetadata housekeepingMetadata, HousekeepingStatus status) { + beekeeperHistoryService.saveHistory(housekeepingMetadata, status); + } } diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java index 01879195..6125944a 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java @@ -17,6 +17,8 @@ import static java.lang.String.format; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_SCHEDULE; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import org.slf4j.Logger; @@ -27,9 +29,11 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.monitoring.TimedTaggable; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @Service public class UnreferencedHousekeepingPathSchedulerService implements SchedulerService { @@ -38,10 +42,13 @@ public class UnreferencedHousekeepingPathSchedulerService implements SchedulerSe private static final LifecycleEventType LIFECYCLE_EVENT_TYPE = UNREFERENCED; private final HousekeepingPathRepository housekeepingPathRepository; + private final BeekeeperHistoryService beekeeperHistoryService; @Autowired - public UnreferencedHousekeepingPathSchedulerService(HousekeepingPathRepository housekeepingPathRepository) { + public UnreferencedHousekeepingPathSchedulerService(HousekeepingPathRepository housekeepingPathRepository, + BeekeeperHistoryService beekeeperHistoryService) { this.housekeepingPathRepository = housekeepingPathRepository; + this.beekeeperHistoryService = beekeeperHistoryService; } @Override @@ -56,8 +63,14 @@ public void scheduleForHousekeeping(HousekeepingEntity housekeepingEntity) { try { housekeepingPathRepository.save(housekeepingPath); log.info(format("Successfully scheduled %s", housekeepingPath)); + saveHistory(housekeepingPath, SCHEDULED); } catch (Exception e) { + saveHistory(housekeepingPath, FAILED_TO_SCHEDULE); throw new BeekeeperException(format("Unable to schedule %s", housekeepingPath), e); } } + + private void saveHistory(HousekeepingPath housekeepingPath, HousekeepingStatus status) { + beekeeperHistoryService.saveHistory(housekeepingPath, status); + } } diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java index 61f588bf..1e95ee09 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java @@ -19,10 +19,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_SCHEDULE; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; @@ -41,6 +44,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class ExpiredHousekeepingMetadataSchedulerServiceTest { @@ -54,6 +58,9 @@ public class ExpiredHousekeepingMetadataSchedulerServiceTest { @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; + @Mock + private BeekeeperHistoryService beekeeperHistoryService; + @InjectMocks private ExpiredHousekeepingMetadataSchedulerService expiredHousekeepingMetadataSchedulerService; @@ -67,6 +74,7 @@ public void typicalCreateScheduleForHousekeeping() { expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); verify(housekeepingMetadataRepository).save(metadata); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED); } @Test @@ -76,13 +84,14 @@ public void typicalCreatePartitionScheduleForHousekeeping() { when(housekeepingMetadataRepository .findRecordForCleanupByDbTableAndPartitionName(DATABASE_NAME, TABLE_NAME, PARTITION_NAME)) - .thenReturn(Optional.empty()); + .thenReturn(Optional.empty()); when(housekeepingMetadataRepository.findRecordForCleanupByDbTableAndPartitionName(DATABASE_NAME, TABLE_NAME, null)) .thenReturn(Optional.of(tableMetadata)); expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); verify(housekeepingMetadataRepository).save(metadata); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED); } @Test @@ -101,6 +110,7 @@ public void typicalUpdateScheduleForHousekeepingWhenChangingCleanupDelay() { verify(existingTable).setClientId(metadata.getClientId()); verify(existingTable).setCleanupDelay(metadata.getCleanupDelay()); verify(housekeepingMetadataRepository).save(existingTable); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED); } @Test @@ -126,6 +136,7 @@ public void typicalUpdatePartitionedTableWithShorterCleanupDelay() { verify(existingTable).setCleanupTimestamp(CREATION_TIMESTAMP.plus(Duration.parse("P30D"))); verify(housekeepingMetadataRepository).save(existingTable); + verify(beekeeperHistoryService).saveHistory(any(), eq(SCHEDULED)); } @Test @@ -143,6 +154,7 @@ public void scheduleFails() { .isThrownBy(() -> expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata)) .withMessage(format("Unable to schedule %s", metadata)); verify(housekeepingMetadataRepository).save(metadata); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_SCHEDULE)); } private HousekeepingMetadata createHousekeepingMetadataPartition() { diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java index bda98a65..27173b23 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java @@ -19,9 +19,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_SCHEDULE; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import java.time.LocalDateTime; @@ -36,6 +40,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class UnreferencedHousekeepingPathSchedulerServiceTest { @@ -43,6 +48,9 @@ public class UnreferencedHousekeepingPathSchedulerServiceTest { @Mock private HousekeepingPathRepository housekeepingPathRepository; + @Mock + private BeekeeperHistoryService beekeeperHistoryService; + @InjectMocks private UnreferencedHousekeepingPathSchedulerService unreferencedHousekeepingPathSchedulerService; @@ -54,7 +62,10 @@ public void typicalScheduleForHousekeeping() { .cleanupDelay(PeriodDuration.parse("P3D")) .build(); unreferencedHousekeepingPathSchedulerService.scheduleForHousekeeping(path); + verify(housekeepingPathRepository).save(path); + + verify(beekeeperHistoryService).saveHistory(path, SCHEDULED); } @Test @@ -77,5 +88,6 @@ public void scheduleFails() { .isThrownBy(() -> unreferencedHousekeepingPathSchedulerService.scheduleForHousekeeping(path)) .withMessage(format("Unable to schedule %s", path)); verify(housekeepingPathRepository).save(path); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_SCHEDULE)); } } diff --git a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java index fb89c8b9..7009ccfa 100644 --- a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java +++ b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java @@ -26,8 +26,10 @@ import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.google.common.base.Supplier; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.scheduler.service.SchedulerService; import com.expediagroup.beekeeper.scheduler.service.UnreferencedHousekeepingPathSchedulerService; +import com.expediagroup.beekeeper.vacuum.repository.BeekeeperEventsHistoryRepository; import com.expediagroup.beekeeper.vacuum.repository.BeekeeperRepository; import com.hotels.hcommon.hive.metastore.client.api.CloseableMetaStoreClient; @@ -75,7 +77,13 @@ Supplier metaStoreClientSupplier( } @Bean - public SchedulerService schedulerService(BeekeeperRepository beekeeperRepository) { - return new UnreferencedHousekeepingPathSchedulerService(beekeeperRepository); + public BeekeeperHistoryService beekeeperHistoryService(BeekeeperEventsHistoryRepository repository){ + return new BeekeeperHistoryService(repository); } + + @Bean + public SchedulerService schedulerService(BeekeeperRepository beekeeperRepository, BeekeeperHistoryService beekeeperHistoryService) { + return new UnreferencedHousekeepingPathSchedulerService(beekeeperRepository, beekeeperHistoryService); + } + } diff --git a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperEventsHistoryRepository.java b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperEventsHistoryRepository.java new file mode 100644 index 00000000..a9d265dd --- /dev/null +++ b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperEventsHistoryRepository.java @@ -0,0 +1,19 @@ +package com.expediagroup.beekeeper.vacuum.repository; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; + +@Repository +public interface BeekeeperEventsHistoryRepository extends BeekeeperHistoryRepository { + + @Query(value = "from BeekeeperHistory t where t.lifecycleType = :lifecycle") + Slice findRecordsByLifecycleType( + @Param("lifecycle") String lifecycle, + Pageable pageable); +} diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/CommonBeansTest.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/CommonBeansTest.java index 7a6aaf69..bfe1e928 100644 --- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/CommonBeansTest.java +++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/CommonBeansTest.java @@ -29,6 +29,7 @@ import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.google.common.base.Supplier; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.scheduler.service.SchedulerService; import com.expediagroup.beekeeper.scheduler.service.UnreferencedHousekeepingPathSchedulerService; import com.expediagroup.beekeeper.vacuum.repository.BeekeeperRepository; @@ -48,6 +49,7 @@ class CommonBeansTest { private final CommonBeans commonBeans = new CommonBeans(); private final String metastoreUri = "thrift://localhost:1234"; private @Mock BeekeeperRepository repository; + private @Mock BeekeeperHistoryService historyService; @AfterAll static void teardown() { @@ -90,7 +92,8 @@ void metaStoreClientSupplier() { @Test void schedulerService() { - SchedulerService schedulerService = new UnreferencedHousekeepingPathSchedulerService(repository); - assertThat(schedulerService).isEqualToComparingFieldByField(commonBeans.schedulerService(repository)); + SchedulerService schedulerService = new UnreferencedHousekeepingPathSchedulerService(repository, historyService); + assertThat(schedulerService).isEqualToComparingFieldByField( + commonBeans.schedulerService(repository, historyService)); } } diff --git a/pom.xml b/pom.xml index 2c313ae2..2a6227a9 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ 5.3.25 1.17.6 11-slim + 2.17.2