From b0efb27af081fcba55ae0e68e3df5265199c83e7 Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Thu, 21 Nov 2024 18:51:35 +0000 Subject: [PATCH 01/13] Add table to track history --- beekeeper-core/pom.xml | 21 +++ .../core/model/HousekeepingStatus.java | 2 + .../core/model/history/BeekeeperHistory.java | 73 +++++++++ .../model/history/ExpiredEventDetails.java | 63 ++++++++ .../model/history/HistoryEventDetails.java | 21 +++ .../history/UnreferencedEventDetails.java | 56 +++++++ .../BeekeeperHistoryRepository.java | 19 +++ .../core/service/BeekeeperHistoryService.java | 36 +++++ .../BeekeeperHistoryRepositoryTest.java | 147 ++++++++++++++++++ .../service/BeekeeperHistoryServiceTest.java | 105 +++++++++++++ .../metadata/cleanup/context/CommonBeans.java | 12 +- .../handler/ExpiredMetadataHandler.java | 30 +++- .../cleanup/context/CommonBeansTest.java | 8 +- .../handler/ExpiredMetadataHandlerTest.java | 76 ++++++++- .../PagingMetadataCleanupServiceTest.java | 6 +- .../cleanup/handler/GenericPathHandler.java | 29 +++- .../handler/UnreferencedPathHandler.java | 6 +- .../handler/GenericPathHandlerTest.java | 17 +- .../handler/UnreferencedPathHandlerTest.java | 6 +- .../service/PagingCleanupServiceTest.java | 14 +- .../migration/V3_1__Create_history_table.sql | 12 ++ ...dHousekeepingMetadataSchedulerService.java | 17 +- ...encedHousekeepingPathSchedulerService.java | 17 +- ...sekeepingMetadataSchedulerServiceTest.java | 18 ++- ...dHousekeepingPathSchedulerServiceTest.java | 14 ++ .../beekeeper/vacuum/CommonBeans.java | 12 +- .../BeekeeperEventsHistoryRepository.java | 19 +++ .../beekeeper/vacuum/CommonBeansTest.java | 7 +- pom.xml | 1 + 29 files changed, 829 insertions(+), 35 deletions(-) create mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java create mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java create mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/HistoryEventDetails.java create mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/UnreferencedEventDetails.java create mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java create mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java create mode 100644 beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java create mode 100644 beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java create mode 100644 beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql create mode 100644 beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperEventsHistoryRepository.java diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 58e3760d..dae50a34 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -44,6 +44,27 @@ 27.1-jre + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-annotations + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-core + ${jackson.version} + + + com.fasterxml.jackson.datatype + jackson-datatype-jsr310 + ${jackson.version} + + io.micrometer diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java index cd9c22a7..f945b54a 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/HousekeepingStatus.java @@ -18,6 +18,8 @@ public enum HousekeepingStatus { SCHEDULED, FAILED, + FAILED_TO_DELETE, + FAILED_TO_SCHEDULE, DELETED, DISABLED, SKIPPED diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java new file mode 100644 index 00000000..a598d7e9 --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/BeekeeperHistory.java @@ -0,0 +1,73 @@ +package com.expediagroup.beekeeper.core.model.history; + +import java.time.LocalDateTime; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.GenerationType; +import javax.persistence.Id; +import javax.persistence.Table; + +import lombok.Builder; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import com.expediagroup.beekeeper.core.monitoring.MetricTag; +import com.expediagroup.beekeeper.core.monitoring.Taggable; + +@Data +@NoArgsConstructor +@Entity +@Table(name = "beekeeper_history") +public class BeekeeperHistory implements Taggable { + + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + + @EqualsAndHashCode.Exclude + @Column(name = "event_timestamp", nullable = false, updatable = false) + private LocalDateTime eventTimestamp; + + @Column(name = "database_name", nullable = false) + private String databaseName; + + @Column(name = "table_name", nullable = false) + private String tableName; + + @Column(name = "lifecycle_type", nullable = false) + private String lifecycleType; + + @Column(name = "housekeeping_status", nullable = false) + private String housekeepingStatus; + + @Column(name = "event_details", columnDefinition = "TEXT") + private String eventDetails; + + @Builder + public BeekeeperHistory( + Long id, + LocalDateTime eventTimestamp, + String databaseName, + String tableName, + String lifecycleType, + String housekeepingStatus, + String eventDetails + ) { + this.id = id; + this.eventTimestamp = eventTimestamp; + this.databaseName = databaseName; + this.tableName = tableName; + this.lifecycleType = lifecycleType; + this.housekeepingStatus = housekeepingStatus; + this.eventDetails = eventDetails; + } + + @Override + public MetricTag getMetricTag() { + return new MetricTag("table", String.join(".", databaseName, tableName)); + } + +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java new file mode 100644 index 00000000..58498193 --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java @@ -0,0 +1,63 @@ +package com.expediagroup.beekeeper.core.model.history; + +import java.time.LocalDateTime; +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import lombok.Builder; +import lombok.Data; +import lombok.extern.jackson.Jacksonized; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.PeriodDuration; + +@Builder +@Data +@Jacksonized +public class ExpiredEventDetails { + + private static Logger log = LoggerFactory.getLogger(ExpiredEventDetails.class); + + String partitionName; + String cleanupDelay; + String creationTimestamp; + String modifiedTimestamp; + String cleanupTimestamp; + int cleanupAttempts; + + public static ExpiredEventDetails fromEntity(HousekeepingMetadata metadata){ + return ExpiredEventDetails.builder() + .partitionName(metadata.getPartitionName()) + .cleanupDelay(metadata.getCleanupDelay().toString()) + .creationTimestamp(Objects.toString(metadata.getCreationTimestamp(), "")) + .modifiedTimestamp(Objects.toString(metadata.getModifiedTimestamp(), "")) + .cleanupTimestamp(Objects.toString(metadata.getCleanupTimestamp(), "")) + .cleanupAttempts(metadata.getCleanupAttempts()) + .build(); + } + + public static String stringFromEntity(HousekeepingMetadata metadata){ + return fromEntity(metadata).toString(); + } + + @Override + public String toString(){ + ObjectMapper mapper = new ObjectMapper(); + mapper.findAndRegisterModules(); + mapper.registerModule(new JavaTimeModule()); + mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + try { + return mapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + log.warn("Error encountered writing object as string", e); + } + return "{}"; + } +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/HistoryEventDetails.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/HistoryEventDetails.java new file mode 100644 index 00000000..f1a9dfdc --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/HistoryEventDetails.java @@ -0,0 +1,21 @@ +package com.expediagroup.beekeeper.core.model.history; + +import java.time.LocalDateTime; + +import lombok.Builder; +import lombok.Data; +import lombok.extern.jackson.Jacksonized; + +@Builder +@Data +@Jacksonized +public class HistoryEventDetails { + + Long id; + LocalDateTime eventTimestamp; + String databaseName; + String tableName; + String lifecycleType; + String housekeepingStatus; + String eventDetails; +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/UnreferencedEventDetails.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/UnreferencedEventDetails.java new file mode 100644 index 00000000..620e400d --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/UnreferencedEventDetails.java @@ -0,0 +1,56 @@ +package com.expediagroup.beekeeper.core.model.history; + +import java.time.LocalDateTime; +import java.util.Objects; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import lombok.Builder; +import lombok.Data; +import lombok.extern.jackson.Jacksonized; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingPath; + +@Builder +@Data +@Jacksonized +public class UnreferencedEventDetails { + + private static Logger log = LoggerFactory.getLogger(UnreferencedEventDetails.class); + + String cleanupDelay; + String creationTimestamp; + String modifiedTimestamp; + String cleanupTimestamp; + int cleanupAttempts; + + public static UnreferencedEventDetails fromEntity(HousekeepingPath path) { + return UnreferencedEventDetails.builder() + .cleanupDelay(path.getCleanupDelay().toString()) + .creationTimestamp(Objects.toString(path.getCreationTimestamp(), "")) + .modifiedTimestamp(Objects.toString(path.getModifiedTimestamp(), "")) + .cleanupTimestamp(Objects.toString(path.getCleanupTimestamp(), "")) + .cleanupAttempts(path.getCleanupAttempts()) + .build(); + } + + public static String stringFromEntity(HousekeepingPath path){ + return fromEntity(path).toString(); + } + + @Override + public String toString(){ + ObjectMapper mapper = new ObjectMapper(); + try { + return mapper.writeValueAsString(this); + } catch (JsonProcessingException e) { + log.warn("Error encountered writing object as string", e); + } + return "{}"; + } +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java new file mode 100644 index 00000000..55c59e6b --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepository.java @@ -0,0 +1,19 @@ +package com.expediagroup.beekeeper.core.repository; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; +import org.springframework.data.jpa.repository.JpaSpecificationExecutor; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.PagingAndSortingRepository; +import org.springframework.data.repository.query.Param; + +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; + +public interface BeekeeperHistoryRepository extends PagingAndSortingRepository, + JpaSpecificationExecutor { + + @Query(value = "from BeekeeperHistory t where t.lifecycleType = :lifecycle") + Slice findRecordsByLifecycleType( + @Param("lifecycle") String lifecycle, + Pageable pageable); +} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java new file mode 100644 index 00000000..aad6f546 --- /dev/null +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -0,0 +1,36 @@ +package com.expediagroup.beekeeper.core.service; + +import java.time.LocalDateTime; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; + +public class BeekeeperHistoryService { + + private static final Logger log = LoggerFactory.getLogger(BeekeeperHistoryService.class); + + private final BeekeeperHistoryRepository beekeeperHistoryRepository; + + public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + this.beekeeperHistoryRepository = beekeeperHistoryRepository; + } + + public void saveHistory(HousekeepingEntity housekeepingEntity, String eventDetails, String status) { + BeekeeperHistory event = BeekeeperHistory.builder() + .id(housekeepingEntity.getId()) + .eventTimestamp(LocalDateTime.now()) + .databaseName(housekeepingEntity.getDatabaseName()) + .tableName(housekeepingEntity.getTableName()) + .lifecycleType(housekeepingEntity.getLifecycleType()) + .housekeepingStatus(status) + .eventDetails(eventDetails) + .build(); + + log.info("Saving activity in Beekeeper History table; {}", event); + beekeeperHistoryRepository.save(event); + } +} diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java new file mode 100644 index 00000000..ac895ac9 --- /dev/null +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java @@ -0,0 +1,147 @@ +package com.expediagroup.beekeeper.core.repository; + +import static org.assertj.core.api.Assertions.assertThat; + +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; + +import org.assertj.core.util.Lists; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.domain.PageRequest; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.springframework.test.context.support.AnnotationConfigContextLoader; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; + +import com.expediagroup.beekeeper.core.TestApplication; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; +import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; +import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; +import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; + +@ExtendWith(SpringExtension.class) +@TestPropertySource(properties = { + "hibernate.data-source.driver-class-name=org.h2.Driver", + "hibernate.dialect=org.hibernate.dialect.H2Dialect", + "hibernate.hbm2ddl.auto=create", + "spring.jpa.show-sql=true", + "spring.datasource.url=jdbc:h2:mem:beekeeper;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE;MODE=MySQL" }) +@ContextConfiguration(classes = { TestApplication.class }, loader = AnnotationConfigContextLoader.class) +@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS) +public class BeekeeperHistoryRepositoryTest { + + protected static final String DATABASE_NAME = "database"; + protected static final String TABLE_NAME = "table"; + protected static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); + protected static final LocalDateTime COLUMN_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC")); + protected static final LocalDateTime EVENT_TIMESTAMP = COLUMN_TIMESTAMP.plus(CLEANUP_DELAY); + + private static final int PAGE = 0; + private static final int PAGE_SIZE = 500; + protected final ObjectMapper mapper = new ObjectMapper(); + + @Autowired + private BeekeeperHistoryRepository repository; + + @BeforeEach + public void setupDb() { + repository.deleteAll(); + } + + @Test + public void typicalSave() throws JsonProcessingException { + BeekeeperHistory expiredEntry = createExpiredEventDetails(SCHEDULED); + BeekeeperHistory unreferencedEntry = createUnreferencedEventDetails(SCHEDULED); + + repository.save(expiredEntry); + repository.save(unreferencedEntry); + + List historyList = Lists.newArrayList( + repository.findRecordsByLifecycleType("EXPIRED", PageRequest.of(PAGE, PAGE_SIZE))); + assertThat(historyList.size()).isEqualTo(1); + + historyList = Lists.newArrayList( + repository.findRecordsByLifecycleType("UNREFERENCED", PageRequest.of(PAGE, PAGE_SIZE))); + assertThat(historyList.size()).isEqualTo(1); + } + + @Test + public void expired_multipleStatuses() throws JsonProcessingException { + BeekeeperHistory scheduledEntry = createExpiredEventDetails(SCHEDULED); + BeekeeperHistory deletedEntry = createExpiredEventDetails(DELETED); + BeekeeperHistory failedEntry = createExpiredEventDetails(FAILED); + + repository.save(scheduledEntry); + repository.save(deletedEntry); + repository.save(failedEntry); + + List historyList = Lists.newArrayList( + repository.findRecordsByLifecycleType("EXPIRED", PageRequest.of(PAGE, PAGE_SIZE))); + assertThat(historyList.size()).isEqualTo(3); + } + + @Test + public void unreferenced_multipleStatuses() throws JsonProcessingException { + BeekeeperHistory scheduledEntry = createUnreferencedEventDetails(SCHEDULED); + BeekeeperHistory deletedEntry = createUnreferencedEventDetails(DELETED); + BeekeeperHistory failedEntry = createUnreferencedEventDetails(FAILED); + + repository.save(scheduledEntry); + repository.save(deletedEntry); + repository.save(failedEntry); + + List historyList = Lists.newArrayList( + repository.findRecordsByLifecycleType("UNREFERENCED", PageRequest.of(PAGE, PAGE_SIZE))); + assertThat(historyList.size()).isEqualTo(3); + } + + protected BeekeeperHistory createExpiredEventDetails(HousekeepingStatus status) throws JsonProcessingException { + ExpiredEventDetails expiredEventDetails = ExpiredEventDetails.builder() + .cleanupAttempts(3) + .cleanupDelay("P1D") + .partitionName("event_date") + .cleanupTimestamp(COLUMN_TIMESTAMP.toString()) + .build(); + + mapper.findAndRegisterModules(); + String stringDetails = mapper.writeValueAsString(expiredEventDetails); + return createHistoryEntry("EXPIRED", status, stringDetails); + } + + protected BeekeeperHistory createUnreferencedEventDetails(HousekeepingStatus status) throws JsonProcessingException { + UnreferencedEventDetails expiredEventDetails = UnreferencedEventDetails.builder() + .cleanupAttempts(3) + .cleanupDelay("P1D") + .cleanupTimestamp(COLUMN_TIMESTAMP.toString()) + .build(); + + mapper.findAndRegisterModules(); + String stringDetails = mapper.writeValueAsString(expiredEventDetails); + return createHistoryEntry("UNREFERENCED", status, stringDetails); + } + + protected BeekeeperHistory createHistoryEntry(String lifecycleType, HousekeepingStatus status, + String eventDetails) { + return BeekeeperHistory.builder() + .eventTimestamp(EVENT_TIMESTAMP) + .databaseName(DATABASE_NAME) + .tableName(TABLE_NAME) + .lifecycleType(lifecycleType) + .housekeepingStatus(status.name()) + .eventDetails(eventDetails) + .build(); + } +} diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java new file mode 100644 index 00000000..eb2ef8f6 --- /dev/null +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java @@ -0,0 +1,105 @@ +package com.expediagroup.beekeeper.core.service; + +import static org.mockito.Mockito.verify; + +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; + +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; + +import com.expediagroup.beekeeper.core.model.HousekeepingEntity; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; +import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; +import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; + +@ExtendWith(MockitoExtension.class) +public class BeekeeperHistoryServiceTest { + + private BeekeeperHistoryService beekeeperHistoryService; + + private @Mock BeekeeperHistoryRepository repository; + + private static final String DATABASE = "database"; + private static final String TABLE_NAME = "tableName"; + private static final String VALID_TABLE_PATH = "s3://bucket/table"; + private static final String VALID_PARTITION_PATH = "s3://bucket/table/partition"; + private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; + private static final LocalDateTime CLEANUP_INSTANCE = LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC); + private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); + + @BeforeEach + public void setup() { + beekeeperHistoryService = new BeekeeperHistoryService(repository); + } + + @Test + void expiredHistory() { + HousekeepingMetadata metadata = createHousekeepingMetadata(); + String details = ExpiredEventDetails.stringFromEntity(metadata); + BeekeeperHistory history = createHistoryEvent(metadata, details, "DELETED"); + + beekeeperHistoryService.saveHistory(metadata, details, "DELETED"); + verify(repository).save(history); + } + + @Test + void unreferencedHistory() { + HousekeepingPath path = createHousekeepingPath(); + String details = UnreferencedEventDetails.stringFromEntity(path); + BeekeeperHistory history = createHistoryEvent(path, details, "SCHEDULED"); + + beekeeperHistoryService.saveHistory(path, details, "SCHEDULED"); + verify(repository).save(history); + } + + private BeekeeperHistory createHistoryEvent(HousekeepingEntity entity, String eventDetails, String status) { + return BeekeeperHistory.builder() + .id(entity.getId()) + .databaseName(entity.getDatabaseName()) + .tableName(entity.getTableName()) + .lifecycleType(entity.getLifecycleType()) + .housekeepingStatus(status) + .eventDetails(eventDetails) + .build(); + } + + private HousekeepingMetadata createHousekeepingMetadata() { + return HousekeepingMetadata.builder() + .path(VALID_TABLE_PATH) + .databaseName(DATABASE) + .tableName(TABLE_NAME) + .partitionName(PARTITION_NAME) + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CLEANUP_INSTANCE) + .cleanupDelay(CLEANUP_DELAY) + .cleanupAttempts(0) + .lifecycleType(EXPIRED.name()) + .build(); + } + + private HousekeepingPath createHousekeepingPath() { + return HousekeepingPath.builder() + .path(VALID_PARTITION_PATH) + .databaseName(DATABASE) + .tableName(TABLE_NAME) + .housekeepingStatus(SCHEDULED) + .creationTimestamp(CLEANUP_INSTANCE) + .cleanupDelay(CLEANUP_DELAY) + .cleanupAttempts(0) + .lifecycleType(UNREFERENCED.name()) + .build(); + } +} diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java index 353adfd9..9e1b818f 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java @@ -48,7 +48,9 @@ import com.expediagroup.beekeeper.cleanup.service.CleanupService; import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.metadata.cleanup.handler.ExpiredMetadataHandler; import com.expediagroup.beekeeper.metadata.cleanup.handler.MetadataHandler; import com.expediagroup.beekeeper.metadata.cleanup.service.MetadataDisableTablesService; @@ -148,9 +150,15 @@ public ExpiredMetadataHandler expiredMetadataHandler( @Qualifier("hiveClientFactory") CleanerClientFactory cleanerClientFactory, HousekeepingMetadataRepository housekeepingMetadataRepository, @Qualifier("hiveTableCleaner") MetadataCleaner metadataCleaner, - @Qualifier("s3PathCleaner") PathCleaner pathCleaner) { + @Qualifier("s3PathCleaner") PathCleaner pathCleaner, + BeekeeperHistoryService beekeeperHistoryService) { return new ExpiredMetadataHandler(cleanerClientFactory, housekeepingMetadataRepository, metadataCleaner, - pathCleaner); + pathCleaner, beekeeperHistoryService); + } + + @Bean + public BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + return new BeekeeperHistoryService(beekeeperHistoryRepository); } @Bean diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java index 28a3cfe4..a8143225 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java @@ -19,6 +19,7 @@ import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_DELETE; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; import java.time.LocalDateTime; @@ -34,7 +35,9 @@ import com.expediagroup.beekeeper.cleanup.path.PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; +import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.core.validation.S3PathValidator; public class ExpiredMetadataHandler implements MetadataHandler { @@ -45,16 +48,19 @@ public class ExpiredMetadataHandler implements MetadataHandler { private final HousekeepingMetadataRepository housekeepingMetadataRepository; private final MetadataCleaner metadataCleaner; private final PathCleaner pathCleaner; + private final BeekeeperHistoryService historyService; public ExpiredMetadataHandler( CleanerClientFactory cleanerClientFactory, HousekeepingMetadataRepository housekeepingMetadataRepository, MetadataCleaner metadataCleaner, - PathCleaner pathCleaner) { + PathCleaner pathCleaner, + BeekeeperHistoryService historyService) { this.cleanerClientFactory = cleanerClientFactory; this.housekeepingMetadataRepository = housekeepingMetadataRepository; this.metadataCleaner = metadataCleaner; this.pathCleaner = pathCleaner; + this.historyService = historyService; } @Override @@ -74,11 +80,15 @@ public Slice findRecordsToClean(LocalDateTime instant, Pag public void cleanupMetadata(HousekeepingMetadata housekeepingMetadata, LocalDateTime instant, boolean dryRunEnabled) { try (CleanerClient client = cleanerClientFactory.newInstance()) { boolean deleted = cleanup(client, housekeepingMetadata, instant, dryRunEnabled); - if (deleted && !dryRunEnabled) { - updateAttemptsAndStatus(housekeepingMetadata, DELETED); + if (deleted) { + if (!dryRunEnabled) { + updateAttemptsAndStatus(housekeepingMetadata, DELETED); + } + saveHistory(housekeepingMetadata, DELETED, dryRunEnabled); } } catch (Exception e) { updateAttemptsAndStatus(housekeepingMetadata, FAILED); + saveHistory(housekeepingMetadata, FAILED_TO_DELETE, dryRunEnabled); log .warn("Unexpected exception when deleting metadata for table \"{}.{}\"", housekeepingMetadata.getDatabaseName(), housekeepingMetadata.getTableName(), e); @@ -107,6 +117,7 @@ private boolean cleanUpTable(CleanerClient client, HousekeepingMetadata housekee if (!S3PathValidator.validTablePath(housekeepingMetadata.getPath())) { log.warn("Will not clean up table path \"{}\" because it is not valid.", housekeepingMetadata.getPath()); updateStatus(housekeepingMetadata, SKIPPED, dryRunEnabled); + saveHistory(housekeepingMetadata, SKIPPED, dryRunEnabled); return false; } String databaseName = housekeepingMetadata.getDatabaseName(); @@ -128,6 +139,7 @@ private boolean cleanupPartition( if (!S3PathValidator.validPartitionPath(housekeepingMetadata.getPath())) { log.warn("Will not clean up partition path \"{}\" because it is not valid.", housekeepingMetadata.getPath()); updateStatus(housekeepingMetadata, SKIPPED, dryRunEnabled); + saveHistory(housekeepingMetadata, SKIPPED, dryRunEnabled); return false; } String databaseName = housekeepingMetadata.getDatabaseName(); @@ -175,4 +187,16 @@ private Long countPartitionsForDatabaseAndTable( return housekeepingMetadataRepository .countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(databaseName, tableName); } + + private void saveHistory(HousekeepingMetadata metadata, HousekeepingStatus housekeepingStatus, + boolean dryRunEnabled) { + String eventDetails = ExpiredEventDetails.stringFromEntity(metadata); + + String status = String.valueOf(housekeepingStatus); + if (dryRunEnabled) { + status = "DRY_RUN_" + housekeepingStatus; + } + + historyService.saveHistory(metadata, eventDetails, status); + } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java index 36085496..67c21b55 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java @@ -50,6 +50,7 @@ import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.metadata.cleanup.handler.ExpiredMetadataHandler; import com.expediagroup.beekeeper.metadata.cleanup.service.MetadataDisableTablesService; import com.expediagroup.beekeeper.metadata.cleanup.service.MetadataRepositoryCleanupService; @@ -76,6 +77,7 @@ public class CommonBeansTest { private @Mock PathCleaner pathCleaner; private @Mock MeterRegistry meterRegistry; private @Mock HiveClientFactory hiveClientFactory; + private @Mock BeekeeperHistoryService beekeeperHistoryService; @BeforeEach public void awsSetUp() { @@ -159,8 +161,7 @@ void verifyS3pathCleaner() { @Test public void verifyExpiredMetadataHandler() { ExpiredMetadataHandler expiredMetadataHandler = commonBeans.expiredMetadataHandler(hiveClientFactory, - metadataRepository, - metadataCleaner, pathCleaner); + metadataRepository, metadataCleaner, pathCleaner, beekeeperHistoryService); assertThat(expiredMetadataHandler).isInstanceOf(ExpiredMetadataHandler.class); } @@ -169,7 +170,8 @@ public void verifyCleanupService() { HiveClientFactory hiveClientFactory = Mockito.mock(HiveClientFactory.class); CleanupService cleanupService = commonBeans.cleanupService( List.of( - commonBeans.expiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner)), 2, + commonBeans.expiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner, + beekeeperHistoryService)), 2, false); assertThat(cleanupService).isInstanceOf(PagingMetadataCleanupService.class); } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java index 1a4a1391..8041a804 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java @@ -16,6 +16,9 @@ package com.expediagroup.beekeeper.metadata.cleanup.handler; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -23,6 +26,7 @@ import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_DELETE; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; @@ -45,7 +49,9 @@ import com.expediagroup.beekeeper.cleanup.hive.HiveMetadataCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class ExpiredMetadataHandlerTest { @@ -56,6 +62,7 @@ public class ExpiredMetadataHandlerTest { private @Mock HiveMetadataCleaner hiveMetadataCleaner; private @Mock S3PathCleaner s3PathCleaner; private @Mock HousekeepingMetadata housekeepingMetadata; + private @Mock BeekeeperHistoryService beekeeperHistoryService; private static final LifecycleEventType lifecycleEventType = EXPIRED; private static final String DATABASE = "database"; @@ -65,14 +72,14 @@ public class ExpiredMetadataHandlerTest { private static final String INVALID_PATH = "s3://bucket"; private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; private static final LocalDateTime CLEANUP_INSTANCE = LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC); + private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); private ExpiredMetadataHandler expiredMetadataHandler; @BeforeEach public void init() { expiredMetadataHandler = new ExpiredMetadataHandler(hiveClientFactory, housekeepingMetadataRepository, - hiveMetadataCleaner, - s3PathCleaner); + hiveMetadataCleaner, s3PathCleaner, beekeeperHistoryService); } @Test @@ -106,6 +113,7 @@ public void typicalRunDroppingTable() { when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when( housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, TABLE_NAME)) @@ -119,6 +127,29 @@ public void typicalRunDroppingTable() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); + } + + @Test + public void typicalDroppingTable_DryRun() { + boolean dryRunEnabled = true; + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); + when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); + when(housekeepingMetadata.getPartitionName()).thenReturn(null); + when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); + when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); + when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); + + expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, dryRunEnabled); + verify(hiveMetadataCleaner).dropTable(housekeepingMetadata, hiveClient); + verify(s3PathCleaner).cleanupPath(housekeepingMetadata); + verify(hiveMetadataCleaner, never()).dropPartition(housekeepingMetadata, hiveClient); + verify(housekeepingMetadata, never()).setCleanupAttempts(1); + verify(housekeepingMetadata, never()).setHousekeepingStatus(DELETED); + verify(housekeepingMetadataRepository, never()).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq("DRY_RUN_DELETED")); } @Test @@ -129,6 +160,8 @@ public void typicalRunDroppingPartition() { when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(PeriodDuration.parse("P3D")); + when(hiveMetadataCleaner.dropPartition(Mockito.any(), Mockito.any())).thenReturn(true); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); @@ -138,6 +171,7 @@ public void typicalRunDroppingPartition() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); } @Test @@ -147,6 +181,7 @@ public void dontDropTableWithInvalidPath() { when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(INVALID_PATH); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(PeriodDuration.parse("P3D")); when(housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, TABLE_NAME)) .thenReturn(Long.valueOf(0)); @@ -158,6 +193,27 @@ public void dontDropTableWithInvalidPath() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(SKIPPED.name())); + } + + @Test + public void dontDropTableWithInvalidPath_DryRun() { + boolean dryRunEnabled = true; + when(hiveClientFactory.newInstance()).thenReturn(hiveClient); + when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); + when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); + when(housekeepingMetadata.getPartitionName()).thenReturn(null); + when(housekeepingMetadata.getPath()).thenReturn(INVALID_PATH); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(PeriodDuration.parse("P3D")); + + expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, dryRunEnabled); + verify(hiveMetadataCleaner, never()).dropTable(housekeepingMetadata, hiveClient); + verify(s3PathCleaner, never()).cleanupPath(housekeepingMetadata); + verify(hiveMetadataCleaner, never()).dropPartition(housekeepingMetadata, hiveClient); + verify(housekeepingMetadata, never()).setCleanupAttempts(1); + verify(housekeepingMetadata, never()).setHousekeepingStatus(SKIPPED); + verify(housekeepingMetadataRepository, never()).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq("DRY_RUN_SKIPPED")); } @Test @@ -187,6 +243,7 @@ public void dontDropTableOrPathWhenTableDoesntExist() { when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(PeriodDuration.parse("P3D")); when( housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, TABLE_NAME)) @@ -200,6 +257,7 @@ public void dontDropTableOrPathWhenTableDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); } @Test @@ -207,6 +265,7 @@ public void dontDropPartitionWithInvalidPartitionPath() { when(hiveClientFactory.newInstance()).thenReturn(hiveClient); when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(INVALID_PATH); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, false); verify(hiveMetadataCleaner, never()).dropPartition(housekeepingMetadata, hiveClient); @@ -215,6 +274,7 @@ public void dontDropPartitionWithInvalidPartitionPath() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(SKIPPED.name())); } @Test @@ -225,6 +285,7 @@ public void dontDropPartitionWhenTableDoesntExist() { when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(false); expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, false); @@ -234,6 +295,7 @@ public void dontDropPartitionWhenTableDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); } @Test @@ -243,6 +305,7 @@ public void dontDropPathWhenPartitionDoesntExist() { when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.dropPartition(Mockito.any(), Mockito.any())).thenReturn(false); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); @@ -252,6 +315,7 @@ public void dontDropPathWhenPartitionDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); } @Test @@ -260,8 +324,9 @@ public void expectedTableDropFailure() { when(housekeepingMetadata.getDatabaseName()).thenReturn(DATABASE); when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); when(housekeepingMetadata.getPartitionName()).thenReturn(null); - when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH);; + when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when( housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, TABLE_NAME)) @@ -273,6 +338,7 @@ public void expectedTableDropFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(FAILED_TO_DELETE.name())); } @Test @@ -283,6 +349,7 @@ public void expectedPathDeleteFailure() { when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); doThrow(RuntimeException.class).when(s3PathCleaner).cleanupPath(housekeepingMetadata); @@ -290,6 +357,7 @@ public void expectedPathDeleteFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(FAILED_TO_DELETE.name())); } @Test @@ -300,6 +368,7 @@ public void expectedPartitionDropFailure() { when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); + when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); doThrow(RuntimeException.class).when(hiveMetadataCleaner).dropPartition(housekeepingMetadata, hiveClient); @@ -307,5 +376,6 @@ public void expectedPartitionDropFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(FAILED_TO_DELETE.name())); } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java index 1a14b3f8..f8c64a68 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/service/PagingMetadataCleanupServiceTest.java @@ -46,6 +46,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; import org.mockito.Captor; +import org.mockito.Mock; import org.mockito.Mockito; import org.mockito.junit.jupiter.MockitoExtension; import org.springframework.beans.factory.annotation.Autowired; @@ -66,6 +67,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.metadata.cleanup.TestApplication; import com.expediagroup.beekeeper.metadata.cleanup.handler.ExpiredMetadataHandler; import com.expediagroup.beekeeper.metadata.cleanup.handler.MetadataHandler; @@ -89,6 +91,7 @@ public class PagingMetadataCleanupServiceTest { private @MockBean PathCleaner pathCleaner; private @MockBean HiveClientFactory hiveClientFactory; private @MockBean HiveClient hiveClient; + private @Mock BeekeeperHistoryService beekeeperHistoryService; private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; @@ -103,7 +106,8 @@ public void init() { properties.put(UNREFERENCED.getTableParameterName(), "true"); when(hiveClient.getTableProperties(Mockito.any(), Mockito.any())).thenReturn(properties); when(hiveClientFactory.newInstance()).thenReturn(hiveClient); - handler = new ExpiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner); + handler = new ExpiredMetadataHandler(hiveClientFactory, metadataRepository, metadataCleaner, pathCleaner, + beekeeperHistoryService); handlers = List.of(handler); pagingCleanupService = new PagingMetadataCleanupService(handlers, 2, false); } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java index 30442c22..b5debb3a 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java @@ -15,6 +15,10 @@ */ package com.expediagroup.beekeeper.path.cleanup.handler; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_DELETE; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; + import java.time.LocalDateTime; import java.util.List; @@ -26,7 +30,9 @@ import com.expediagroup.beekeeper.cleanup.path.PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; +import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.core.validation.S3PathValidator; public abstract class GenericPathHandler { @@ -35,10 +41,13 @@ public abstract class GenericPathHandler { private final HousekeepingPathRepository housekeepingPathRepository; private final PathCleaner pathCleaner; + private final BeekeeperHistoryService beekeeperHistoryService; - public GenericPathHandler(HousekeepingPathRepository housekeepingPathRepository, PathCleaner pathCleaner) { + public GenericPathHandler(HousekeepingPathRepository housekeepingPathRepository, PathCleaner pathCleaner, + BeekeeperHistoryService beekeeperHistoryService) { this.housekeepingPathRepository = housekeepingPathRepository; this.pathCleaner = pathCleaner; + this.beekeeperHistoryService = beekeeperHistoryService; } public abstract Slice findRecordsToClean(LocalDateTime instant, Pageable pageable); @@ -49,11 +58,11 @@ public GenericPathHandler(HousekeepingPathRepository housekeepingPathRepository, * @param pageable Pageable to iterate through for dryRun * @param page Page to get content from * @param dryRunEnabled Dry Run boolean flag + * @return Pageable to pass to query. In the case of dry runs, this is the next page. * @implNote This parent handler expects the child's cleanupPath call to update & remove the record from this call - * such that subsequent DB queries will not return the record. Hence why we only call next during dryRuns - * where no updates occur. + * such that subsequent DB queries will not return the record. Hence why we only call next during dryRuns + * where no updates occur. * @implNote Note that we only expect pageable.next to be called during a dry run. - * @return Pageable to pass to query. In the case of dry runs, this is the next page. */ public Pageable processPage(Pageable pageable, Slice page, boolean dryRunEnabled) { List pageContent = page.getContent(); @@ -79,12 +88,14 @@ private void cleanupContent(HousekeepingPath housekeepingPath) { try { log.info("Cleaning up path \"{}\"", housekeepingPath.getPath()); if (cleanUpPath(housekeepingPath)) { - updateAttemptsAndStatus(housekeepingPath, HousekeepingStatus.DELETED); + updateAttemptsAndStatus(housekeepingPath, DELETED); + saveHistory(housekeepingPath, DELETED); } else { - updateStatus(housekeepingPath, HousekeepingStatus.SKIPPED); + updateStatus(housekeepingPath, SKIPPED); } } catch (Exception e) { updateAttemptsAndStatus(housekeepingPath, HousekeepingStatus.FAILED); + saveHistory(housekeepingPath, FAILED_TO_DELETE); log.warn("Unexpected exception deleting \"{}\"", housekeepingPath.getPath(), e); } } @@ -98,5 +109,11 @@ private void updateAttemptsAndStatus(HousekeepingPath housekeepingPath, Housekee private void updateStatus(HousekeepingPath housekeepingPath, HousekeepingStatus status) { housekeepingPath.setHousekeepingStatus(status); housekeepingPathRepository.save(housekeepingPath); + saveHistory(housekeepingPath, status); + } + + private void saveHistory(HousekeepingPath housekeepingPath, HousekeepingStatus status) { + String eventDetails = UnreferencedEventDetails.stringFromEntity(housekeepingPath); + beekeeperHistoryService.saveHistory(housekeepingPath, eventDetails, status.name()); } } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandler.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandler.java index 82d68b69..8242e977 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandler.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandler.java @@ -26,6 +26,7 @@ import com.expediagroup.beekeeper.cleanup.path.PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @Component public class UnreferencedPathHandler extends GenericPathHandler { @@ -35,8 +36,9 @@ public class UnreferencedPathHandler extends GenericPathHandler { @Autowired public UnreferencedPathHandler( HousekeepingPathRepository housekeepingPathRepository, - @Qualifier("s3PathCleaner") PathCleaner pathCleaner) { - super(housekeepingPathRepository, pathCleaner); + @Qualifier("s3PathCleaner") PathCleaner pathCleaner, + BeekeeperHistoryService beekeeperHistoryService) { + super(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); this.housekeepingPathRepository = housekeepingPathRepository; } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java index aa66d192..ff76dc19 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java @@ -16,6 +16,9 @@ package com.expediagroup.beekeeper.path.cleanup.handler; import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; import static org.mockito.Mockito.verify; @@ -23,6 +26,7 @@ import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_DELETE; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SKIPPED; import java.util.List; @@ -37,7 +41,9 @@ import com.expediagroup.beekeeper.cleanup.aws.S3PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class GenericPathHandlerTest { @@ -47,6 +53,8 @@ public class GenericPathHandlerTest { @Mock private S3PathCleaner pathCleaner; @Mock + private BeekeeperHistoryService beekeeperHistoryService; + @Mock private HousekeepingPath mockPath; @Mock private Pageable mockPageable; @@ -55,12 +63,13 @@ public class GenericPathHandlerTest { @Mock private PageImpl mockPage; private static final String VALID_TABLE_PATH = "s3://bucket/table"; + private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); private UnreferencedPathHandler handler; @BeforeEach public void initTest() { - handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); when(mockPath.getPath()).thenReturn(VALID_TABLE_PATH); } @@ -76,6 +85,7 @@ public void typicalProcessDryRunPage() { @Test public void typicalProcessPage() { when(mockPath.getCleanupAttempts()).thenReturn(0); + when(mockPath.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(mockPage.getContent()).thenReturn(List.of(mockPath)); Pageable pageable = handler.processPage(mockPageable, mockPage, false); verify(pathCleaner).cleanupPath(mockPath); @@ -83,12 +93,14 @@ public void typicalProcessPage() { verify(mockPath).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(DELETED); verify(housekeepingPathRepository).save(mockPath); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); assertThat(pageable).isEqualTo(pageable); } @Test public void processPageFails() { when(mockPath.getCleanupAttempts()).thenReturn(0); + when(mockPath.getCleanupDelay()).thenReturn(CLEANUP_DELAY); doThrow(RuntimeException.class).when(pathCleaner).cleanupPath(mockPath); when(mockPage.getContent()).thenReturn(List.of(mockPath)); Pageable pageable = handler.processPage(mockPageable, mockPage, false); @@ -96,12 +108,14 @@ public void processPageFails() { verify(mockPath).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(FAILED); verify(housekeepingPathRepository).save(mockPath); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(FAILED_TO_DELETE.name())); assertThat(pageable).isEqualTo(pageable); } @Test public void processPageInvalidPath() { when(mockPath.getPath()).thenReturn("invalid"); + when(mockPath.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(mockPage.getContent()).thenReturn(List.of(mockPath)); Pageable pageable = handler.processPage(mockPageable, mockPage, false); verify(pathCleaner, never()).cleanupPath(mockPath); @@ -109,6 +123,7 @@ public void processPageInvalidPath() { verify(mockPath, never()).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(SKIPPED); verify(housekeepingPathRepository).save(mockPath); + verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(SKIPPED.name())); assertThat(pageable).isEqualTo(pageable); } } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandlerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandlerTest.java index c00e9ca6..e911cc78 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandlerTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/UnreferencedPathHandlerTest.java @@ -33,6 +33,7 @@ import com.expediagroup.beekeeper.cleanup.aws.S3PathCleaner; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class UnreferencedPathHandlerTest { @@ -41,13 +42,16 @@ public class UnreferencedPathHandlerTest { private HousekeepingPathRepository housekeepingPathRepository; @Mock private S3PathCleaner s3PathCleaner; + @Mock + private BeekeeperHistoryService beekeeperHistoryService; + private LifecycleEventType lifecycleEventType = UNREFERENCED; private UnreferencedPathHandler handler; @BeforeEach public void initTest() { - handler = new UnreferencedPathHandler(housekeepingPathRepository, s3PathCleaner); + handler = new UnreferencedPathHandler(housekeepingPathRepository, s3PathCleaner, beekeeperHistoryService); } @Test diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/PagingCleanupServiceTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/PagingCleanupServiceTest.java index db654ed4..af64ca93 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/PagingCleanupServiceTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/service/PagingCleanupServiceTest.java @@ -54,6 +54,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.path.cleanup.TestApplication; import com.expediagroup.beekeeper.path.cleanup.handler.UnreferencedPathHandler; @@ -73,10 +74,11 @@ public class PagingCleanupServiceTest { private @Captor ArgumentCaptor pathCaptor; private @Autowired HousekeepingPathRepository housekeepingPathRepository; private @MockBean PathCleaner pathCleaner; + private @MockBean BeekeeperHistoryService beekeeperHistoryService; @Test public void typicalWithPaging() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 2, false); List paths = List.of("s3://bucket/some_foo", "s3://bucket/some_bar", "s3://bucket/some_foobar"); @@ -97,7 +99,7 @@ public void typicalWithPaging() { @Test public void mixOfScheduledAndFailedPaths() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 2, false); List paths = List .of(createEntityHousekeepingPath("s3://bucket/some_foo", SCHEDULED), @@ -113,7 +115,7 @@ public void mixOfScheduledAndFailedPaths() { @Test public void mixOfAllPaths() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 2, false); List paths = List .of(createEntityHousekeepingPath("s3://bucket/some_foo", SCHEDULED), @@ -130,7 +132,7 @@ public void mixOfAllPaths() { @Test void pathCleanerException() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 2, false); doThrow(new RuntimeException("Error")).doNothing().when(pathCleaner).cleanupPath(any(HousekeepingPath.class)); @@ -158,7 +160,7 @@ void pathCleanerException() { @Test @Timeout(value = 10) void doNotInfiniteLoopOnRepeatedFailures() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 1, false); List paths = List .of(createEntityHousekeepingPath("s3://bucket/some_foo", FAILED), @@ -186,7 +188,7 @@ void doNotInfiniteLoopOnRepeatedFailures() { @Test @Timeout(value = 10) void doNotInfiniteLoopOnDryRunCleanup() { - UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner); + UnreferencedPathHandler handler = new UnreferencedPathHandler(housekeepingPathRepository, pathCleaner, beekeeperHistoryService); pagingCleanupService = new PagingPathCleanupService(List.of(handler), 1, true); List paths = List .of(createEntityHousekeepingPath("s3://bucket/some_foo", SCHEDULED), diff --git a/beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql b/beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql new file mode 100644 index 00000000..601f3bfa --- /dev/null +++ b/beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql @@ -0,0 +1,12 @@ +USE beekeeper; + +CREATE TABLE IF NOT EXISTS beekeeper_history ( + id BIGINT(20) AUTO_INCREMENT, + event_timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + database_name VARCHAR(512), + table_name VARCHAR(512), + lifecycle_type VARCHAR(255) NOT NULL, + housekeeping_status VARCHAR(50) NOT NULL, + event_details TEXT, + PRIMARY KEY (id) +); \ No newline at end of file diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java index cff64747..07fc4f78 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java @@ -17,6 +17,8 @@ import static java.lang.String.format; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_SCHEDULE; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import java.time.LocalDateTime; @@ -30,9 +32,12 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; import com.expediagroup.beekeeper.core.monitoring.TimedTaggable; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @Service public class ExpiredHousekeepingMetadataSchedulerService implements SchedulerService { @@ -41,10 +46,13 @@ public class ExpiredHousekeepingMetadataSchedulerService implements SchedulerSer private static final LifecycleEventType LIFECYCLE_EVENT_TYPE = EXPIRED; private final HousekeepingMetadataRepository housekeepingMetadataRepository; + private final BeekeeperHistoryService beekeeperHistoryService; @Autowired - public ExpiredHousekeepingMetadataSchedulerService(HousekeepingMetadataRepository housekeepingMetadataRepository) { + public ExpiredHousekeepingMetadataSchedulerService(HousekeepingMetadataRepository housekeepingMetadataRepository, + BeekeeperHistoryService beekeeperHistoryService) { this.housekeepingMetadataRepository = housekeepingMetadataRepository; + this.beekeeperHistoryService = beekeeperHistoryService; } @Override @@ -60,7 +68,9 @@ public void scheduleForHousekeeping(HousekeepingEntity housekeepingEntity) { try { housekeepingMetadataRepository.save(housekeepingMetadata); log.info(format("Successfully scheduled %s", housekeepingMetadata)); + saveHistory(housekeepingMetadata, SCHEDULED); } catch (Exception e) { + saveHistory(housekeepingMetadata, FAILED_TO_SCHEDULE); throw new BeekeeperException(format("Unable to schedule %s", housekeepingMetadata), e); } } @@ -140,4 +150,9 @@ private boolean isPartitionedTable(HousekeepingMetadata housekeepingMetadata) { return numPartitions > 0 && housekeepingMetadata.getPartitionName() == null; } + + private void saveHistory(HousekeepingMetadata housekeepingMetadata, HousekeepingStatus status) { + String eventDetails = ExpiredEventDetails.stringFromEntity(housekeepingMetadata); + beekeeperHistoryService.saveHistory(housekeepingMetadata, eventDetails, status.name()); + } } diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java index 01879195..a7b6e0a5 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java @@ -17,6 +17,8 @@ import static java.lang.String.format; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_SCHEDULE; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import org.slf4j.Logger; @@ -27,9 +29,12 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; import com.expediagroup.beekeeper.core.monitoring.TimedTaggable; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @Service public class UnreferencedHousekeepingPathSchedulerService implements SchedulerService { @@ -38,10 +43,13 @@ public class UnreferencedHousekeepingPathSchedulerService implements SchedulerSe private static final LifecycleEventType LIFECYCLE_EVENT_TYPE = UNREFERENCED; private final HousekeepingPathRepository housekeepingPathRepository; + private final BeekeeperHistoryService beekeeperHistoryService; @Autowired - public UnreferencedHousekeepingPathSchedulerService(HousekeepingPathRepository housekeepingPathRepository) { + public UnreferencedHousekeepingPathSchedulerService(HousekeepingPathRepository housekeepingPathRepository, + BeekeeperHistoryService beekeeperHistoryService) { this.housekeepingPathRepository = housekeepingPathRepository; + this.beekeeperHistoryService = beekeeperHistoryService; } @Override @@ -56,8 +64,15 @@ public void scheduleForHousekeeping(HousekeepingEntity housekeepingEntity) { try { housekeepingPathRepository.save(housekeepingPath); log.info(format("Successfully scheduled %s", housekeepingPath)); + saveHistory(housekeepingPath, SCHEDULED); } catch (Exception e) { + saveHistory(housekeepingPath, FAILED_TO_SCHEDULE); throw new BeekeeperException(format("Unable to schedule %s", housekeepingPath), e); } } + + private void saveHistory(HousekeepingPath housekeepingPath, HousekeepingStatus status) { + String eventDetails = UnreferencedEventDetails.stringFromEntity(housekeepingPath); + beekeeperHistoryService.saveHistory(housekeepingPath, eventDetails, status.name()); + } } diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java index 61f588bf..9e99ee12 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java @@ -19,10 +19,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_SCHEDULE; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; @@ -40,7 +43,9 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class ExpiredHousekeepingMetadataSchedulerServiceTest { @@ -54,6 +59,9 @@ public class ExpiredHousekeepingMetadataSchedulerServiceTest { @Mock private HousekeepingMetadataRepository housekeepingMetadataRepository; + @Mock + private BeekeeperHistoryService beekeeperHistoryService; + @InjectMocks private ExpiredHousekeepingMetadataSchedulerService expiredHousekeepingMetadataSchedulerService; @@ -65,8 +73,10 @@ public void typicalCreateScheduleForHousekeeping() { .thenReturn(Optional.empty()); expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); + String eventDetails = ExpiredEventDetails.stringFromEntity(metadata); verify(housekeepingMetadataRepository).save(metadata); + verify(beekeeperHistoryService).saveHistory(metadata, eventDetails, SCHEDULED.name()); } @Test @@ -76,13 +86,15 @@ public void typicalCreatePartitionScheduleForHousekeeping() { when(housekeepingMetadataRepository .findRecordForCleanupByDbTableAndPartitionName(DATABASE_NAME, TABLE_NAME, PARTITION_NAME)) - .thenReturn(Optional.empty()); + .thenReturn(Optional.empty()); when(housekeepingMetadataRepository.findRecordForCleanupByDbTableAndPartitionName(DATABASE_NAME, TABLE_NAME, null)) .thenReturn(Optional.of(tableMetadata)); expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); + String eventDetails = ExpiredEventDetails.stringFromEntity(metadata); verify(housekeepingMetadataRepository).save(metadata); + verify(beekeeperHistoryService).saveHistory(metadata, eventDetails, SCHEDULED.name()); } @Test @@ -95,12 +107,14 @@ public void typicalUpdateScheduleForHousekeepingWhenChangingCleanupDelay() { .thenReturn(Optional.of(existingTable)); expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); + String eventDetails = ExpiredEventDetails.stringFromEntity(metadata); verify(existingTable).setPath(metadata.getPath()); verify(existingTable).setHousekeepingStatus(metadata.getHousekeepingStatus()); verify(existingTable).setClientId(metadata.getClientId()); verify(existingTable).setCleanupDelay(metadata.getCleanupDelay()); verify(housekeepingMetadataRepository).save(existingTable); + verify(beekeeperHistoryService).saveHistory(metadata, eventDetails, SCHEDULED.name()); } @Test @@ -126,6 +140,7 @@ public void typicalUpdatePartitionedTableWithShorterCleanupDelay() { verify(existingTable).setCleanupTimestamp(CREATION_TIMESTAMP.plus(Duration.parse("P30D"))); verify(housekeepingMetadataRepository).save(existingTable); + verify(beekeeperHistoryService).saveHistory(any(), any(), eq(SCHEDULED.name())); } @Test @@ -143,6 +158,7 @@ public void scheduleFails() { .isThrownBy(() -> expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata)) .withMessage(format("Unable to schedule %s", metadata)); verify(housekeepingMetadataRepository).save(metadata); + verify(beekeeperHistoryService).saveHistory(any(), any(), eq(FAILED_TO_SCHEDULE.name())); } private HousekeepingMetadata createHousekeepingMetadataPartition() { diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java index bda98a65..fe722807 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java @@ -19,9 +19,13 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED_TO_SCHEDULE; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import java.time.LocalDateTime; @@ -35,7 +39,9 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @ExtendWith(MockitoExtension.class) public class UnreferencedHousekeepingPathSchedulerServiceTest { @@ -43,6 +49,9 @@ public class UnreferencedHousekeepingPathSchedulerServiceTest { @Mock private HousekeepingPathRepository housekeepingPathRepository; + @Mock + private BeekeeperHistoryService beekeeperHistoryService; + @InjectMocks private UnreferencedHousekeepingPathSchedulerService unreferencedHousekeepingPathSchedulerService; @@ -54,7 +63,11 @@ public void typicalScheduleForHousekeeping() { .cleanupDelay(PeriodDuration.parse("P3D")) .build(); unreferencedHousekeepingPathSchedulerService.scheduleForHousekeeping(path); + verify(housekeepingPathRepository).save(path); + + String eventDetails = UnreferencedEventDetails.stringFromEntity(path); + verify(beekeeperHistoryService).saveHistory(path, eventDetails, SCHEDULED.name()); } @Test @@ -77,5 +90,6 @@ public void scheduleFails() { .isThrownBy(() -> unreferencedHousekeepingPathSchedulerService.scheduleForHousekeeping(path)) .withMessage(format("Unable to schedule %s", path)); verify(housekeepingPathRepository).save(path); + verify(beekeeperHistoryService).saveHistory(any(), any(), eq(FAILED_TO_SCHEDULE.name())); } } diff --git a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java index fb89c8b9..7009ccfa 100644 --- a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java +++ b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/CommonBeans.java @@ -26,8 +26,10 @@ import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.google.common.base.Supplier; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.scheduler.service.SchedulerService; import com.expediagroup.beekeeper.scheduler.service.UnreferencedHousekeepingPathSchedulerService; +import com.expediagroup.beekeeper.vacuum.repository.BeekeeperEventsHistoryRepository; import com.expediagroup.beekeeper.vacuum.repository.BeekeeperRepository; import com.hotels.hcommon.hive.metastore.client.api.CloseableMetaStoreClient; @@ -75,7 +77,13 @@ Supplier metaStoreClientSupplier( } @Bean - public SchedulerService schedulerService(BeekeeperRepository beekeeperRepository) { - return new UnreferencedHousekeepingPathSchedulerService(beekeeperRepository); + public BeekeeperHistoryService beekeeperHistoryService(BeekeeperEventsHistoryRepository repository){ + return new BeekeeperHistoryService(repository); } + + @Bean + public SchedulerService schedulerService(BeekeeperRepository beekeeperRepository, BeekeeperHistoryService beekeeperHistoryService) { + return new UnreferencedHousekeepingPathSchedulerService(beekeeperRepository, beekeeperHistoryService); + } + } diff --git a/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperEventsHistoryRepository.java b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperEventsHistoryRepository.java new file mode 100644 index 00000000..a9d265dd --- /dev/null +++ b/beekeeper-vacuum-tool/src/main/java/com/expediagroup/beekeeper/vacuum/repository/BeekeeperEventsHistoryRepository.java @@ -0,0 +1,19 @@ +package com.expediagroup.beekeeper.vacuum.repository; + +import org.springframework.data.domain.Pageable; +import org.springframework.data.domain.Slice; +import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; +import org.springframework.stereotype.Repository; + +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; + +@Repository +public interface BeekeeperEventsHistoryRepository extends BeekeeperHistoryRepository { + + @Query(value = "from BeekeeperHistory t where t.lifecycleType = :lifecycle") + Slice findRecordsByLifecycleType( + @Param("lifecycle") String lifecycle, + Pageable pageable); +} diff --git a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/CommonBeansTest.java b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/CommonBeansTest.java index 7a6aaf69..bfe1e928 100644 --- a/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/CommonBeansTest.java +++ b/beekeeper-vacuum-tool/src/test/java/com/expediagroup/beekeeper/vacuum/CommonBeansTest.java @@ -29,6 +29,7 @@ import com.amazonaws.auth.EC2ContainerCredentialsProviderWrapper; import com.google.common.base.Supplier; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.scheduler.service.SchedulerService; import com.expediagroup.beekeeper.scheduler.service.UnreferencedHousekeepingPathSchedulerService; import com.expediagroup.beekeeper.vacuum.repository.BeekeeperRepository; @@ -48,6 +49,7 @@ class CommonBeansTest { private final CommonBeans commonBeans = new CommonBeans(); private final String metastoreUri = "thrift://localhost:1234"; private @Mock BeekeeperRepository repository; + private @Mock BeekeeperHistoryService historyService; @AfterAll static void teardown() { @@ -90,7 +92,8 @@ void metaStoreClientSupplier() { @Test void schedulerService() { - SchedulerService schedulerService = new UnreferencedHousekeepingPathSchedulerService(repository); - assertThat(schedulerService).isEqualToComparingFieldByField(commonBeans.schedulerService(repository)); + SchedulerService schedulerService = new UnreferencedHousekeepingPathSchedulerService(repository, historyService); + assertThat(schedulerService).isEqualToComparingFieldByField( + commonBeans.schedulerService(repository, historyService)); } } diff --git a/pom.xml b/pom.xml index 4683bea5..ba852d28 100644 --- a/pom.xml +++ b/pom.xml @@ -52,6 +52,7 @@ 5.3.25 1.17.1 11-slim + 2.17.2 From 44f99287e47e332e4df81c84b2c753287d446acd Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Fri, 22 Nov 2024 14:32:30 +0000 Subject: [PATCH 02/13] Tidy --- .../beekeeper/core/model/history/ExpiredEventDetails.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java index 58498193..2db0d9d5 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java @@ -50,9 +50,6 @@ public static String stringFromEntity(HousekeepingMetadata metadata){ @Override public String toString(){ ObjectMapper mapper = new ObjectMapper(); - mapper.findAndRegisterModules(); - mapper.registerModule(new JavaTimeModule()); - mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); try { return mapper.writeValueAsString(this); } catch (JsonProcessingException e) { From 11d996d308cc8518fa810bc82a20c08fc62666a0 Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Fri, 22 Nov 2024 15:36:11 +0000 Subject: [PATCH 03/13] Add index to history table --- .../resources/db/migration/V3_1__Create_history_table.sql | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql b/beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql index 601f3bfa..c8f2f977 100644 --- a/beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql +++ b/beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql @@ -9,4 +9,7 @@ CREATE TABLE IF NOT EXISTS beekeeper_history ( housekeeping_status VARCHAR(50) NOT NULL, event_details TEXT, PRIMARY KEY (id) -); \ No newline at end of file +); + +ALTER TABLE beekeeper_history ADD INDEX `beekeeper_history_index_table_name_upper` ((upper(table_name))); +ALTER TABLE beekeeper_history ADD INDEX `beekeeper_history_index_status` (`housekeeping_status`); From a938ee47b69c924c64a000369f1751ac9684205a Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Fri, 22 Nov 2024 16:27:52 +0000 Subject: [PATCH 04/13] Annotations --- .../beekeeper/core/service/BeekeeperHistoryService.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java index aad6f546..c18ee3c8 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -4,17 +4,21 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; +@Component public class BeekeeperHistoryService { private static final Logger log = LoggerFactory.getLogger(BeekeeperHistoryService.class); private final BeekeeperHistoryRepository beekeeperHistoryRepository; + @Autowired public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { this.beekeeperHistoryRepository = beekeeperHistoryRepository; } From 328cc9bbbf4585b32efb765b8639e78bd0461410 Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Fri, 22 Nov 2024 17:03:32 +0000 Subject: [PATCH 05/13] beans --- .../core/service/BeekeeperHistoryService.java | 2 -- .../metadata/cleanup/context/CommonBeans.java | 2 +- .../metadata/cleanup/context/CommonBeansTest.java | 8 ++++++++ .../path/cleanup/context/CommonBeans.java | 7 +++++++ .../path/cleanup/context/CommonBeansTest.java | 9 +++++++++ .../scheduler/apiary/context/CommonBeans.java | 7 +++++++ .../scheduler/apiary/context/CommonBeansTest.java | 15 ++++++++++++--- 7 files changed, 44 insertions(+), 6 deletions(-) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java index c18ee3c8..3f460b1d 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -11,14 +11,12 @@ import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; -@Component public class BeekeeperHistoryService { private static final Logger log = LoggerFactory.getLogger(BeekeeperHistoryService.class); private final BeekeeperHistoryRepository beekeeperHistoryRepository; - @Autowired public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { this.beekeeperHistoryRepository = beekeeperHistoryRepository; } diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java index 9e1b818f..26cb780e 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeans.java @@ -157,7 +157,7 @@ public ExpiredMetadataHandler expiredMetadataHandler( } @Bean - public BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { return new BeekeeperHistoryService(beekeeperHistoryRepository); } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java index 67c21b55..14eea45f 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/context/CommonBeansTest.java @@ -49,6 +49,7 @@ import com.expediagroup.beekeeper.cleanup.service.CleanupService; import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.metadata.cleanup.handler.ExpiredMetadataHandler; @@ -78,6 +79,7 @@ public class CommonBeansTest { private @Mock MeterRegistry meterRegistry; private @Mock HiveClientFactory hiveClientFactory; private @Mock BeekeeperHistoryService beekeeperHistoryService; + private @Mock BeekeeperHistoryRepository beekeeperHistoryRepository; @BeforeEach public void awsSetUp() { @@ -188,4 +190,10 @@ public void verifyDisableTablesService() { metadataRepository, hiveClientFactory, false); assertThat(disableTablesService).isInstanceOf(MetadataDisableTablesService.class); } + + @Test + public void verifyBeekeeperHistoryService(){ + BeekeeperHistoryService beekeeperHistoryService = commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); + assertThat(beekeeperHistoryService).isInstanceOf(BeekeeperHistoryService.class); + } } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java index 0b118243..605ddf9f 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeans.java @@ -40,7 +40,9 @@ import com.expediagroup.beekeeper.cleanup.service.CleanupService; import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.path.cleanup.handler.GenericPathHandler; import com.expediagroup.beekeeper.path.cleanup.service.PagingPathCleanupService; import com.expediagroup.beekeeper.path.cleanup.service.PathRepositoryCleanupService; @@ -108,4 +110,9 @@ RepositoryCleanupService repositoryCleanupService( DisableTablesService disableTablesService() { return () -> {}; } + + @Bean + BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + return new BeekeeperHistoryService(beekeeperHistoryRepository); + } } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeansTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeansTest.java index a6667b73..1a76b1c5 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeansTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/context/CommonBeansTest.java @@ -40,7 +40,9 @@ import com.expediagroup.beekeeper.cleanup.service.CleanupService; import com.expediagroup.beekeeper.cleanup.service.DisableTablesService; import com.expediagroup.beekeeper.cleanup.service.RepositoryCleanupService; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.path.cleanup.service.PagingPathCleanupService; import com.expediagroup.beekeeper.path.cleanup.service.PathRepositoryCleanupService; @@ -59,6 +61,7 @@ class CommonBeansTest { private final CommonBeans commonBeans = new CommonBeans(); private @Mock HousekeepingPathRepository repository; private @Mock BytesDeletedReporter bytesDeletedReporter; + private @Mock BeekeeperHistoryRepository beekeeperHistoryRepository; @BeforeEach void setUp() { @@ -121,4 +124,10 @@ public void verifyDisableTablesService() { DisableTablesService disableTablesService = commonBeans.disableTablesService(); assertThat(disableTablesService).isNotNull(); } + + @Test + public void verifyBeekeeperHistoryService(){ + BeekeeperHistoryService beekeeperHistoryService = commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); + assertThat(beekeeperHistoryService).isInstanceOf(BeekeeperHistoryService.class); + } } diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java index 492017c6..641a72f0 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java @@ -38,6 +38,8 @@ import com.expedia.apiary.extensions.receiver.sqs.messaging.SqsMessageReader; import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.scheduler.apiary.filter.EventTypeListenerEventFilter; import com.expediagroup.beekeeper.scheduler.apiary.filter.ListenerEventFilter; import com.expediagroup.beekeeper.scheduler.apiary.filter.LocationOnlyUpdateListenerEventFilter; @@ -139,4 +141,9 @@ public BeekeeperEventReader eventReader( return new MessageReaderAdapter(messageReader, handlers); } + + @Bean + BeekeeperHistoryService beekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) { + return new BeekeeperHistoryService(beekeeperHistoryRepository); + } } diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java index 2aa5f17b..e07c6b9f 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeansTest.java @@ -32,6 +32,8 @@ import com.expedia.apiary.extensions.receiver.sqs.messaging.SqsMessageReader; import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; +import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.scheduler.apiary.generator.ExpiredHousekeepingMetadataGenerator; import com.expediagroup.beekeeper.scheduler.apiary.generator.HousekeepingEntityGenerator; import com.expediagroup.beekeeper.scheduler.apiary.generator.UnreferencedHousekeepingPathGenerator; @@ -51,9 +53,10 @@ public class CommonBeansTest { private static final String BUCKET = "bucket"; private static final String KEY = "key"; private final CommonBeans commonBeans = new CommonBeans(); - @Mock private MessageReader messageReader; - @Mock private UnreferencedHousekeepingPathGenerator unreferencedHousekeepingPathGenerator; - @Mock private ExpiredHousekeepingMetadataGenerator expiredHousekeepingMetadataGenerator; + private @Mock MessageReader messageReader; + private @Mock UnreferencedHousekeepingPathGenerator unreferencedHousekeepingPathGenerator; + private @Mock ExpiredHousekeepingMetadataGenerator expiredHousekeepingMetadataGenerator; + private @Mock BeekeeperHistoryRepository beekeeperHistoryRepository; @AfterAll static void tearDown() { @@ -117,4 +120,10 @@ public void validatePathEventReader() { mock(MessageEventHandler.class)); assertThat(reader).isInstanceOf(BeekeeperEventReader.class); } + + @Test + public void verifyBeekeeperHistoryService() { + BeekeeperHistoryService beekeeperHistoryService = commonBeans.beekeeperHistoryService(beekeeperHistoryRepository); + assertThat(beekeeperHistoryService).isInstanceOf(BeekeeperHistoryService.class); + } } From 5e1162d65e81158be477d118657bf3c784bc7059 Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Fri, 22 Nov 2024 17:27:36 +0000 Subject: [PATCH 06/13] Integration test db var --- .../beekeeper/integration/BeekeeperIntegrationTestBase.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 6967cbb4..1a48ed94 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -86,6 +86,7 @@ public abstract class BeekeeperIntegrationTestBase { private static final String BEEKEEPER_FLYWAY_TABLE = "flyway_schema_history"; private static final String BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME = "housekeeping_path"; private static final String BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME = "housekeeping_metadata"; + private static final String BEEKEEPER_HISTORY_TABLE_NAME = "beekeeper_history"; // FIELDS TO INSERT INTO BEEKEEPER TABLES private Long id = 1L; @@ -148,6 +149,7 @@ public void dropMySQLTables() throws SQLException { mySQLTestUtils.dropTable(BEEKEEPER_DB_NAME, BEEKEEPER_FLYWAY_TABLE); mySQLTestUtils.dropTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME); mySQLTestUtils.dropTable(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_METADATA_TABLE_NAME); + mySQLTestUtils.dropTable(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME); } protected void insertUnreferencedPath(String path) throws SQLException { From f1bfebc1342b2e2169e025bc531160aac75c33e9 Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Fri, 22 Nov 2024 18:29:57 +0000 Subject: [PATCH 07/13] Cleanup --- .../model/history/ExpiredEventDetails.java | 60 ------------------- .../model/history/HistoryEventDetails.java | 21 ------- .../history/UnreferencedEventDetails.java | 56 ----------------- .../core/service/BeekeeperHistoryService.java | 24 +++++++- .../BeekeeperHistoryRepositoryTest.java | 33 ++++++---- .../service/BeekeeperHistoryServiceTest.java | 27 ++++++--- .../handler/ExpiredMetadataHandler.java | 6 +- .../handler/ExpiredMetadataHandlerTest.java | 40 ++++--------- .../cleanup/handler/GenericPathHandler.java | 4 +- .../handler/GenericPathHandlerTest.java | 12 +--- ...dHousekeepingMetadataSchedulerService.java | 4 +- ...encedHousekeepingPathSchedulerService.java | 4 +- ...sekeepingMetadataSchedulerServiceTest.java | 14 ++--- ...dHousekeepingPathSchedulerServiceTest.java | 6 +- 14 files changed, 86 insertions(+), 225 deletions(-) delete mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java delete mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/HistoryEventDetails.java delete mode 100644 beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/UnreferencedEventDetails.java diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java deleted file mode 100644 index 2db0d9d5..00000000 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/ExpiredEventDetails.java +++ /dev/null @@ -1,60 +0,0 @@ -package com.expediagroup.beekeeper.core.model.history; - -import java.time.LocalDateTime; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import lombok.Builder; -import lombok.Data; -import lombok.extern.jackson.Jacksonized; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - -import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.model.PeriodDuration; - -@Builder -@Data -@Jacksonized -public class ExpiredEventDetails { - - private static Logger log = LoggerFactory.getLogger(ExpiredEventDetails.class); - - String partitionName; - String cleanupDelay; - String creationTimestamp; - String modifiedTimestamp; - String cleanupTimestamp; - int cleanupAttempts; - - public static ExpiredEventDetails fromEntity(HousekeepingMetadata metadata){ - return ExpiredEventDetails.builder() - .partitionName(metadata.getPartitionName()) - .cleanupDelay(metadata.getCleanupDelay().toString()) - .creationTimestamp(Objects.toString(metadata.getCreationTimestamp(), "")) - .modifiedTimestamp(Objects.toString(metadata.getModifiedTimestamp(), "")) - .cleanupTimestamp(Objects.toString(metadata.getCleanupTimestamp(), "")) - .cleanupAttempts(metadata.getCleanupAttempts()) - .build(); - } - - public static String stringFromEntity(HousekeepingMetadata metadata){ - return fromEntity(metadata).toString(); - } - - @Override - public String toString(){ - ObjectMapper mapper = new ObjectMapper(); - try { - return mapper.writeValueAsString(this); - } catch (JsonProcessingException e) { - log.warn("Error encountered writing object as string", e); - } - return "{}"; - } -} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/HistoryEventDetails.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/HistoryEventDetails.java deleted file mode 100644 index f1a9dfdc..00000000 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/HistoryEventDetails.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.expediagroup.beekeeper.core.model.history; - -import java.time.LocalDateTime; - -import lombok.Builder; -import lombok.Data; -import lombok.extern.jackson.Jacksonized; - -@Builder -@Data -@Jacksonized -public class HistoryEventDetails { - - Long id; - LocalDateTime eventTimestamp; - String databaseName; - String tableName; - String lifecycleType; - String housekeepingStatus; - String eventDetails; -} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/UnreferencedEventDetails.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/UnreferencedEventDetails.java deleted file mode 100644 index 620e400d..00000000 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/history/UnreferencedEventDetails.java +++ /dev/null @@ -1,56 +0,0 @@ -package com.expediagroup.beekeeper.core.model.history; - -import java.time.LocalDateTime; -import java.util.Objects; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import lombok.Builder; -import lombok.Data; -import lombok.extern.jackson.Jacksonized; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; - -import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; -import com.expediagroup.beekeeper.core.model.HousekeepingPath; - -@Builder -@Data -@Jacksonized -public class UnreferencedEventDetails { - - private static Logger log = LoggerFactory.getLogger(UnreferencedEventDetails.class); - - String cleanupDelay; - String creationTimestamp; - String modifiedTimestamp; - String cleanupTimestamp; - int cleanupAttempts; - - public static UnreferencedEventDetails fromEntity(HousekeepingPath path) { - return UnreferencedEventDetails.builder() - .cleanupDelay(path.getCleanupDelay().toString()) - .creationTimestamp(Objects.toString(path.getCreationTimestamp(), "")) - .modifiedTimestamp(Objects.toString(path.getModifiedTimestamp(), "")) - .cleanupTimestamp(Objects.toString(path.getCleanupTimestamp(), "")) - .cleanupAttempts(path.getCleanupAttempts()) - .build(); - } - - public static String stringFromEntity(HousekeepingPath path){ - return fromEntity(path).toString(); - } - - @Override - public String toString(){ - ObjectMapper mapper = new ObjectMapper(); - try { - return mapper.writeValueAsString(this); - } catch (JsonProcessingException e) { - log.warn("Error encountered writing object as string", e); - } - return "{}"; - } -} diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java index 3f460b1d..856945e4 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -4,8 +4,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; @@ -21,7 +24,9 @@ public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryReposi this.beekeeperHistoryRepository = beekeeperHistoryRepository; } - public void saveHistory(HousekeepingEntity housekeepingEntity, String eventDetails, String status) { + public void saveHistory(HousekeepingEntity housekeepingEntity, String status) { + String eventDetails = createEventDetails(housekeepingEntity); + BeekeeperHistory event = BeekeeperHistory.builder() .id(housekeepingEntity.getId()) .eventTimestamp(LocalDateTime.now()) @@ -35,4 +40,17 @@ public void saveHistory(HousekeepingEntity housekeepingEntity, String eventDetai log.info("Saving activity in Beekeeper History table; {}", event); beekeeperHistoryRepository.save(event); } + + private String createEventDetails(HousekeepingEntity entity) { + ObjectMapper mapper = new ObjectMapper(); + mapper.findAndRegisterModules(); + mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + mapper.registerModule(new JavaTimeModule()); + try { + return mapper.writeValueAsString(entity); + } catch (JsonProcessingException e) { + log.warn("Error encountered writing object as string", e); + } + return "{}"; + } } diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java index ac895ac9..dc09c2ad 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java @@ -24,13 +24,15 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.expediagroup.beekeeper.core.TestApplication; +import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; +import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; -import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; -import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; @ExtendWith(SpringExtension.class) @TestPropertySource(properties = { @@ -51,7 +53,7 @@ public class BeekeeperHistoryRepositoryTest { private static final int PAGE = 0; private static final int PAGE_SIZE = 500; - protected final ObjectMapper mapper = new ObjectMapper(); + protected ObjectMapper mapper; @Autowired private BeekeeperHistoryRepository repository; @@ -59,6 +61,11 @@ public class BeekeeperHistoryRepositoryTest { @BeforeEach public void setupDb() { repository.deleteAll(); + + mapper = new ObjectMapper(); + mapper.findAndRegisterModules(); + mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + mapper.registerModule(new JavaTimeModule()); } @Test @@ -109,27 +116,27 @@ public void unreferenced_multipleStatuses() throws JsonProcessingException { } protected BeekeeperHistory createExpiredEventDetails(HousekeepingStatus status) throws JsonProcessingException { - ExpiredEventDetails expiredEventDetails = ExpiredEventDetails.builder() + HousekeepingMetadata entity = HousekeepingMetadata.builder() .cleanupAttempts(3) - .cleanupDelay("P1D") + .cleanupDelay(PeriodDuration.parse("P1D")) .partitionName("event_date") - .cleanupTimestamp(COLUMN_TIMESTAMP.toString()) + .creationTimestamp(COLUMN_TIMESTAMP) + .modifiedTimestamp(COLUMN_TIMESTAMP) .build(); - mapper.findAndRegisterModules(); - String stringDetails = mapper.writeValueAsString(expiredEventDetails); + String stringDetails = mapper.writeValueAsString(entity); return createHistoryEntry("EXPIRED", status, stringDetails); } protected BeekeeperHistory createUnreferencedEventDetails(HousekeepingStatus status) throws JsonProcessingException { - UnreferencedEventDetails expiredEventDetails = UnreferencedEventDetails.builder() + HousekeepingPath entity = HousekeepingPath.builder() .cleanupAttempts(3) - .cleanupDelay("P1D") - .cleanupTimestamp(COLUMN_TIMESTAMP.toString()) + .cleanupDelay(PeriodDuration.parse("P1D")) + .creationTimestamp(COLUMN_TIMESTAMP) + .modifiedTimestamp(COLUMN_TIMESTAMP) .build(); - mapper.findAndRegisterModules(); - String stringDetails = mapper.writeValueAsString(expiredEventDetails); + String stringDetails = mapper.writeValueAsString(entity); return createHistoryEntry("UNREFERENCED", status, stringDetails); } diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java index eb2ef8f6..9a08d244 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java @@ -16,13 +16,16 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; + import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; -import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; -import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; @ExtendWith(MockitoExtension.class) @@ -46,22 +49,22 @@ public void setup() { } @Test - void expiredHistory() { + void expiredHistory() throws JsonProcessingException { HousekeepingMetadata metadata = createHousekeepingMetadata(); - String details = ExpiredEventDetails.stringFromEntity(metadata); + String details = createEventDetails(metadata); BeekeeperHistory history = createHistoryEvent(metadata, details, "DELETED"); - beekeeperHistoryService.saveHistory(metadata, details, "DELETED"); + beekeeperHistoryService.saveHistory(metadata, "DELETED"); verify(repository).save(history); } @Test - void unreferencedHistory() { + void unreferencedHistory() throws JsonProcessingException { HousekeepingPath path = createHousekeepingPath(); - String details = UnreferencedEventDetails.stringFromEntity(path); + String details = createEventDetails(path); BeekeeperHistory history = createHistoryEvent(path, details, "SCHEDULED"); - beekeeperHistoryService.saveHistory(path, details, "SCHEDULED"); + beekeeperHistoryService.saveHistory(path, "SCHEDULED"); verify(repository).save(history); } @@ -102,4 +105,12 @@ private HousekeepingPath createHousekeepingPath() { .lifecycleType(UNREFERENCED.name()) .build(); } + + private String createEventDetails(HousekeepingEntity housekeepingEntity) throws JsonProcessingException { + ObjectMapper mapper = new ObjectMapper(); + mapper.findAndRegisterModules(); + mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); + mapper.registerModule(new JavaTimeModule()); + return mapper.writeValueAsString(housekeepingEntity); + } } diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java index a8143225..a17a3d76 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java @@ -35,7 +35,6 @@ import com.expediagroup.beekeeper.cleanup.path.PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; -import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.core.validation.S3PathValidator; @@ -190,13 +189,10 @@ private Long countPartitionsForDatabaseAndTable( private void saveHistory(HousekeepingMetadata metadata, HousekeepingStatus housekeepingStatus, boolean dryRunEnabled) { - String eventDetails = ExpiredEventDetails.stringFromEntity(metadata); - String status = String.valueOf(housekeepingStatus); if (dryRunEnabled) { status = "DRY_RUN_" + housekeepingStatus; } - - historyService.saveHistory(metadata, eventDetails, status); + historyService.saveHistory(metadata, status); } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java index 8041a804..3b3fa63f 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java @@ -17,7 +17,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; @@ -49,7 +48,6 @@ import com.expediagroup.beekeeper.cleanup.hive.HiveMetadataCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.LifecycleEventType; -import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @@ -72,7 +70,6 @@ public class ExpiredMetadataHandlerTest { private static final String INVALID_PATH = "s3://bucket"; private static final String PARTITION_NAME = "event_date=2020-01-01/event_hour=0/event_type=A"; private static final LocalDateTime CLEANUP_INSTANCE = LocalDateTime.ofInstant(Instant.now(), ZoneOffset.UTC); - private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); private ExpiredMetadataHandler expiredMetadataHandler; @@ -113,7 +110,6 @@ public void typicalRunDroppingTable() { when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when( housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, TABLE_NAME)) @@ -127,7 +123,7 @@ public void typicalRunDroppingTable() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); } @Test @@ -138,8 +134,6 @@ public void typicalDroppingTable_DryRun() { when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); - when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, dryRunEnabled); @@ -149,7 +143,7 @@ public void typicalDroppingTable_DryRun() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata, never()).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository, never()).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq("DRY_RUN_DELETED")); + verify(beekeeperHistoryService).saveHistory(any(), eq("DRY_RUN_DELETED")); } @Test @@ -160,7 +154,6 @@ public void typicalRunDroppingPartition() { when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(PeriodDuration.parse("P3D")); when(hiveMetadataCleaner.dropPartition(Mockito.any(), Mockito.any())).thenReturn(true); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); @@ -171,7 +164,7 @@ public void typicalRunDroppingPartition() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); } @Test @@ -181,7 +174,6 @@ public void dontDropTableWithInvalidPath() { when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(INVALID_PATH); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(PeriodDuration.parse("P3D")); when(housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, TABLE_NAME)) .thenReturn(Long.valueOf(0)); @@ -193,7 +185,7 @@ public void dontDropTableWithInvalidPath() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(SKIPPED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED.name())); } @Test @@ -204,7 +196,6 @@ public void dontDropTableWithInvalidPath_DryRun() { when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(INVALID_PATH); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(PeriodDuration.parse("P3D")); expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, dryRunEnabled); verify(hiveMetadataCleaner, never()).dropTable(housekeepingMetadata, hiveClient); @@ -213,7 +204,7 @@ public void dontDropTableWithInvalidPath_DryRun() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata, never()).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository, never()).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq("DRY_RUN_SKIPPED")); + verify(beekeeperHistoryService).saveHistory(any(), eq("DRY_RUN_SKIPPED")); } @Test @@ -243,7 +234,6 @@ public void dontDropTableOrPathWhenTableDoesntExist() { when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(PeriodDuration.parse("P3D")); when( housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, TABLE_NAME)) @@ -257,7 +247,7 @@ public void dontDropTableOrPathWhenTableDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); } @Test @@ -265,7 +255,6 @@ public void dontDropPartitionWithInvalidPartitionPath() { when(hiveClientFactory.newInstance()).thenReturn(hiveClient); when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(INVALID_PATH); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, false); verify(hiveMetadataCleaner, never()).dropPartition(housekeepingMetadata, hiveClient); @@ -274,7 +263,7 @@ public void dontDropPartitionWithInvalidPartitionPath() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(SKIPPED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED.name())); } @Test @@ -285,7 +274,6 @@ public void dontDropPartitionWhenTableDoesntExist() { when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(false); expiredMetadataHandler.cleanupMetadata(housekeepingMetadata, CLEANUP_INSTANCE, false); @@ -295,7 +283,7 @@ public void dontDropPartitionWhenTableDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); } @Test @@ -305,7 +293,6 @@ public void dontDropPathWhenPartitionDoesntExist() { when(housekeepingMetadata.getTableName()).thenReturn(TABLE_NAME); when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.dropPartition(Mockito.any(), Mockito.any())).thenReturn(false); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); @@ -315,7 +302,7 @@ public void dontDropPathWhenPartitionDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); } @Test @@ -326,7 +313,6 @@ public void expectedTableDropFailure() { when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when( housekeepingMetadataRepository.countRecordsForGivenDatabaseAndTableWherePartitionIsNotNull(DATABASE, TABLE_NAME)) @@ -338,7 +324,7 @@ public void expectedTableDropFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(FAILED_TO_DELETE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE.name())); } @Test @@ -349,7 +335,6 @@ public void expectedPathDeleteFailure() { when(housekeepingMetadata.getPartitionName()).thenReturn(null); when(housekeepingMetadata.getPath()).thenReturn(VALID_TABLE_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); doThrow(RuntimeException.class).when(s3PathCleaner).cleanupPath(housekeepingMetadata); @@ -357,7 +342,7 @@ public void expectedPathDeleteFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(FAILED_TO_DELETE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE.name())); } @Test @@ -368,7 +353,6 @@ public void expectedPartitionDropFailure() { when(housekeepingMetadata.getPartitionName()).thenReturn(PARTITION_NAME); when(housekeepingMetadata.getPath()).thenReturn(VALID_PARTITION_PATH); when(housekeepingMetadata.getCleanupAttempts()).thenReturn(0); - when(housekeepingMetadata.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(hiveMetadataCleaner.tableExists(hiveClient, DATABASE, TABLE_NAME)).thenReturn(true); doThrow(RuntimeException.class).when(hiveMetadataCleaner).dropPartition(housekeepingMetadata, hiveClient); @@ -376,6 +360,6 @@ public void expectedPartitionDropFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(FAILED_TO_DELETE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE.name())); } } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java index b5debb3a..a249f445 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java @@ -30,7 +30,6 @@ import com.expediagroup.beekeeper.cleanup.path.PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; -import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; import com.expediagroup.beekeeper.core.validation.S3PathValidator; @@ -113,7 +112,6 @@ private void updateStatus(HousekeepingPath housekeepingPath, HousekeepingStatus } private void saveHistory(HousekeepingPath housekeepingPath, HousekeepingStatus status) { - String eventDetails = UnreferencedEventDetails.stringFromEntity(housekeepingPath); - beekeeperHistoryService.saveHistory(housekeepingPath, eventDetails, status.name()); + beekeeperHistoryService.saveHistory(housekeepingPath, status.name()); } } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java index ff76dc19..52393a91 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java @@ -17,7 +17,6 @@ import static org.assertj.core.api.AssertionsForClassTypes.assertThat; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.never; @@ -41,7 +40,6 @@ import com.expediagroup.beekeeper.cleanup.aws.S3PathCleaner; import com.expediagroup.beekeeper.core.model.HousekeepingPath; -import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @@ -63,7 +61,6 @@ public class GenericPathHandlerTest { @Mock private PageImpl mockPage; private static final String VALID_TABLE_PATH = "s3://bucket/table"; - private static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D"); private UnreferencedPathHandler handler; @@ -85,7 +82,6 @@ public void typicalProcessDryRunPage() { @Test public void typicalProcessPage() { when(mockPath.getCleanupAttempts()).thenReturn(0); - when(mockPath.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(mockPage.getContent()).thenReturn(List.of(mockPath)); Pageable pageable = handler.processPage(mockPageable, mockPage, false); verify(pathCleaner).cleanupPath(mockPath); @@ -93,14 +89,13 @@ public void typicalProcessPage() { verify(mockPath).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(DELETED); verify(housekeepingPathRepository).save(mockPath); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); assertThat(pageable).isEqualTo(pageable); } @Test public void processPageFails() { when(mockPath.getCleanupAttempts()).thenReturn(0); - when(mockPath.getCleanupDelay()).thenReturn(CLEANUP_DELAY); doThrow(RuntimeException.class).when(pathCleaner).cleanupPath(mockPath); when(mockPage.getContent()).thenReturn(List.of(mockPath)); Pageable pageable = handler.processPage(mockPageable, mockPage, false); @@ -108,14 +103,13 @@ public void processPageFails() { verify(mockPath).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(FAILED); verify(housekeepingPathRepository).save(mockPath); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(FAILED_TO_DELETE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE.name())); assertThat(pageable).isEqualTo(pageable); } @Test public void processPageInvalidPath() { when(mockPath.getPath()).thenReturn("invalid"); - when(mockPath.getCleanupDelay()).thenReturn(CLEANUP_DELAY); when(mockPage.getContent()).thenReturn(List.of(mockPath)); Pageable pageable = handler.processPage(mockPageable, mockPage, false); verify(pathCleaner, never()).cleanupPath(mockPath); @@ -123,7 +117,7 @@ public void processPageInvalidPath() { verify(mockPath, never()).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(SKIPPED); verify(housekeepingPathRepository).save(mockPath); - verify(beekeeperHistoryService).saveHistory(any(), anyString(), eq(SKIPPED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED.name())); assertThat(pageable).isEqualTo(pageable); } } diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java index 07fc4f78..af6a9e9e 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java @@ -34,7 +34,6 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; -import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; import com.expediagroup.beekeeper.core.monitoring.TimedTaggable; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @@ -152,7 +151,6 @@ private boolean isPartitionedTable(HousekeepingMetadata housekeepingMetadata) { } private void saveHistory(HousekeepingMetadata housekeepingMetadata, HousekeepingStatus status) { - String eventDetails = ExpiredEventDetails.stringFromEntity(housekeepingMetadata); - beekeeperHistoryService.saveHistory(housekeepingMetadata, eventDetails, status.name()); + beekeeperHistoryService.saveHistory(housekeepingMetadata, status.name()); } } diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java index a7b6e0a5..34aacfa9 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java @@ -31,7 +31,6 @@ import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; -import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; import com.expediagroup.beekeeper.core.monitoring.TimedTaggable; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @@ -72,7 +71,6 @@ public void scheduleForHousekeeping(HousekeepingEntity housekeepingEntity) { } private void saveHistory(HousekeepingPath housekeepingPath, HousekeepingStatus status) { - String eventDetails = UnreferencedEventDetails.stringFromEntity(housekeepingPath); - beekeeperHistoryService.saveHistory(housekeepingPath, eventDetails, status.name()); + beekeeperHistoryService.saveHistory(housekeepingPath, status.name()); } } diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java index 9e99ee12..b85a25e9 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java @@ -43,7 +43,6 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.PeriodDuration; -import com.expediagroup.beekeeper.core.model.history.ExpiredEventDetails; import com.expediagroup.beekeeper.core.repository.HousekeepingMetadataRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @@ -73,10 +72,9 @@ public void typicalCreateScheduleForHousekeeping() { .thenReturn(Optional.empty()); expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); - String eventDetails = ExpiredEventDetails.stringFromEntity(metadata); verify(housekeepingMetadataRepository).save(metadata); - verify(beekeeperHistoryService).saveHistory(metadata, eventDetails, SCHEDULED.name()); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED.name()); } @Test @@ -91,10 +89,9 @@ public void typicalCreatePartitionScheduleForHousekeeping() { .thenReturn(Optional.of(tableMetadata)); expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); - String eventDetails = ExpiredEventDetails.stringFromEntity(metadata); verify(housekeepingMetadataRepository).save(metadata); - verify(beekeeperHistoryService).saveHistory(metadata, eventDetails, SCHEDULED.name()); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED.name()); } @Test @@ -107,14 +104,13 @@ public void typicalUpdateScheduleForHousekeepingWhenChangingCleanupDelay() { .thenReturn(Optional.of(existingTable)); expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); - String eventDetails = ExpiredEventDetails.stringFromEntity(metadata); verify(existingTable).setPath(metadata.getPath()); verify(existingTable).setHousekeepingStatus(metadata.getHousekeepingStatus()); verify(existingTable).setClientId(metadata.getClientId()); verify(existingTable).setCleanupDelay(metadata.getCleanupDelay()); verify(housekeepingMetadataRepository).save(existingTable); - verify(beekeeperHistoryService).saveHistory(metadata, eventDetails, SCHEDULED.name()); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED.name()); } @Test @@ -140,7 +136,7 @@ public void typicalUpdatePartitionedTableWithShorterCleanupDelay() { verify(existingTable).setCleanupTimestamp(CREATION_TIMESTAMP.plus(Duration.parse("P30D"))); verify(housekeepingMetadataRepository).save(existingTable); - verify(beekeeperHistoryService).saveHistory(any(), any(), eq(SCHEDULED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(SCHEDULED.name())); } @Test @@ -158,7 +154,7 @@ public void scheduleFails() { .isThrownBy(() -> expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata)) .withMessage(format("Unable to schedule %s", metadata)); verify(housekeepingMetadataRepository).save(metadata); - verify(beekeeperHistoryService).saveHistory(any(), any(), eq(FAILED_TO_SCHEDULE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_SCHEDULE.name())); } private HousekeepingMetadata createHousekeepingMetadataPartition() { diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java index fe722807..2f1e6ba0 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java @@ -39,7 +39,6 @@ import com.expediagroup.beekeeper.core.error.BeekeeperException; import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.PeriodDuration; -import com.expediagroup.beekeeper.core.model.history.UnreferencedEventDetails; import com.expediagroup.beekeeper.core.repository.HousekeepingPathRepository; import com.expediagroup.beekeeper.core.service.BeekeeperHistoryService; @@ -66,8 +65,7 @@ public void typicalScheduleForHousekeeping() { verify(housekeepingPathRepository).save(path); - String eventDetails = UnreferencedEventDetails.stringFromEntity(path); - verify(beekeeperHistoryService).saveHistory(path, eventDetails, SCHEDULED.name()); + verify(beekeeperHistoryService).saveHistory(path, SCHEDULED.name()); } @Test @@ -90,6 +88,6 @@ public void scheduleFails() { .isThrownBy(() -> unreferencedHousekeepingPathSchedulerService.scheduleForHousekeeping(path)) .withMessage(format("Unable to schedule %s", path)); verify(housekeepingPathRepository).save(path); - verify(beekeeperHistoryService).saveHistory(any(), any(), eq(FAILED_TO_SCHEDULE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_SCHEDULE.name())); } } From 3665e28db13595797713012c24e47f8e6c18dc11 Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Mon, 25 Nov 2024 14:08:51 +0000 Subject: [PATCH 08/13] Integration test setup --- ...etadataSchedulerApiaryIntegrationTest.java | 12 ++++ .../BeekeeperIntegrationTestBase.java | 55 ++++++++++++++++++- ...cedPathSchedulerApiaryIntegrationTest.java | 14 +++++ .../integration/CommonTestVariables.java | 2 + .../ResultSetToBeekeeperHistoryMapper.java | 27 +++++++++ 5 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index ccbf19c6..a5bb23c1 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -59,6 +59,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.model.AddPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage; @@ -213,6 +214,17 @@ public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLExcepti assertExpiredMetadata(expiredMetadata.get(1), LOCATION_B, PARTITION_B_NAME); } + @Test + public void addEventToHistoryTable() throws SQLException, IOException, URISyntaxException { + CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true); + amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1); + + List beekeeperHistory = getBeekeeperHistory(EXPIRED); + System.out.println(beekeeperHistory); + } + @Test public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 1a48ed94..0bbefc49 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -30,6 +30,8 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.CREATION_TIMESTAMP_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_DETAILS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_TIMESTAMP_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.HOUSEKEEPING_STATUS_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.ID_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.LIFECYCLE_TYPE_FIELD; @@ -39,6 +41,7 @@ import static com.expediagroup.beekeeper.integration.CommonTestVariables.SHORT_CLEANUP_DELAY_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_FIELD; import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; +import static com.expediagroup.beekeeper.integration.utils.ResultSetToBeekeeperHistoryMapper.mapToBeekeeperHistory; import static com.expediagroup.beekeeper.integration.utils.ResultSetToHousekeepingEntityMapper.mapToHousekeepingMetadata; import static com.expediagroup.beekeeper.integration.utils.ResultSetToHousekeepingEntityMapper.mapToHousekeepingPath; @@ -61,8 +64,10 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingPath; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.integration.utils.MySqlTestUtils; @@ -98,6 +103,8 @@ public abstract class BeekeeperIntegrationTestBase { .join(",", ID_FIELD, PATH_FIELD, DATABASE_NAME_FIELD, TABLE_NAME_FIELD, PARTITION_NAME_FIELD, HOUSEKEEPING_STATUS_FIELD, CREATION_TIMESTAMP_FIELD, MODIFIED_TIMESTAMP_FIELD, CLEANUP_TIMESTAMP_FIELD, CLEANUP_DELAY_FIELD, CLEANUP_ATTEMPTS_FIELD, CLIENT_ID_FIELD, LIFECYCLE_TYPE_FIELD); + private static final String BEEKEEPER_HISTORY_FIELDS = String.join(",", ID_FIELD, EVENT_TIMESTAMP_FIELD, + DATABASE_NAME_FIELD, TABLE_NAME_FIELD, LIFECYCLE_TYPE_FIELD, HOUSEKEEPING_STATUS_FIELD, EVENT_DETAILS_FIELD); private static final String LIFE_CYCLE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s' ORDER BY " + PATH_FIELD; private static final String LIFE_CYCLE_AND_UPDATE_FILTER = "WHERE " + LIFECYCLE_TYPE_FIELD @@ -178,7 +185,7 @@ protected void insertExpiredMetadata(String path, String partitionName) throws S } protected void insertExpiredMetadata(String tableName, String path, String partitionName, String cleanupDelay) - throws SQLException { + throws SQLException { HousekeepingMetadata metadata = createHousekeepingMetadata(tableName, path, partitionName, EXPIRED, cleanupDelay); insertExpiredMetadata(metadata); } @@ -198,6 +205,16 @@ protected void insertExpiredMetadata(HousekeepingMetadata metadata) throws SQLEx values); } + protected void insertBeekeeperHistory(BeekeeperHistory beekeeperHistory) throws SQLException { + String values = Stream.of(beekeeperHistory.getId().toString(), beekeeperHistory.getEventTimestamp(), + beekeeperHistory.getDatabaseName(), beekeeperHistory.getTableName(), beekeeperHistory.getLifecycleType(), + beekeeperHistory.getHousekeepingStatus(), beekeeperHistory.getEventDetails()) + .map(s -> s == null ? null : "\"" + s + "\"") + .collect(Collectors.joining(", ")); + + mySQLTestUtils.insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, BEEKEEPER_HISTORY_FIELDS, values); + } + protected int getUnreferencedPathsRowCount() throws SQLException { return mySQLTestUtils .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, @@ -216,6 +233,13 @@ protected int getUpdatedExpiredMetadataRowCount() throws SQLException { format(LIFE_CYCLE_AND_UPDATE_FILTER, EXPIRED)); } + protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) throws SQLException { + String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; + + return mySQLTestUtils.getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, + format(filter, lifecycleEventType)); + } + protected List getUnreferencedPaths() throws SQLException { List paths = new ArrayList<>(); ResultSet resultSet = mySQLTestUtils @@ -242,6 +266,19 @@ protected List getExpiredMetadata() throws SQLException { return metadata; } + protected List getBeekeeperHistory(LifecycleEventType lifecycleEventType) throws SQLException { + String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; + List history = new ArrayList<>(); + ResultSet resultSet = mySQLTestUtils.getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, + format(filter, lifecycleEventType)); + + while (resultSet.next()) { + history.add(mapToBeekeeperHistory(resultSet)); + } + + return history; + } + private HousekeepingPath createHousekeepingPath(String path, LifecycleEventType lifecycleEventType) { return HousekeepingPath .builder() @@ -282,4 +319,20 @@ private HousekeepingMetadata createHousekeepingMetadata( .build(); } + private BeekeeperHistory createBeekeeperHistory( + String tableName, + LifecycleEventType lifecycleEventType, + HousekeepingStatus housekeepingStatus, + String eventDetails + ) { + return BeekeeperHistory.builder() + .id(id++) + .eventTimestamp(CREATION_TIMESTAMP_VALUE) + .databaseName(DATABASE_NAME_VALUE) + .tableName(tableName) + .lifecycleType(lifecycleEventType.toString()) + .housekeepingStatus(housekeepingStatus.name()) + .eventDetails(eventDetails) + .build(); + } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 92a3d0c0..d43ab5dc 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -20,6 +20,7 @@ import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SQS; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.CLEANUP_ATTEMPTS_VALUE; @@ -59,6 +60,7 @@ import com.expediagroup.beekeeper.core.model.HousekeepingPath; import com.expediagroup.beekeeper.core.model.PeriodDuration; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.model.AlterPartitionSqsMessage; import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage; import com.expediagroup.beekeeper.integration.model.DropPartitionSqsMessage; @@ -196,6 +198,18 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy assertUnreferencedPath(unreferencedPaths.get(1), "s3://bucket/tableLocation2"); } + @Test + public void addBeekeeperHistoryEvent() throws IOException, URISyntaxException, SQLException { + AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", + "s3://bucket/oldTableLocation", true, true); + amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + + List beekeeperHistory = getBeekeeperHistory(EXPIRED); + System.out.println(beekeeperHistory); + } + @Test public void healthCheck() { CloseableHttpClient client = HttpClientBuilder.create().build(); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java index 5f263448..c3d56db8 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/CommonTestVariables.java @@ -41,6 +41,8 @@ private CommonTestVariables() {} public static final String CLEANUP_ATTEMPTS_FIELD = "cleanup_attempts"; public static final String CLIENT_ID_FIELD = "client_id"; public static final String LIFECYCLE_TYPE_FIELD = "lifecycle_type"; + public static final String EVENT_DETAILS_FIELD = "event_details"; + public static final String EVENT_TIMESTAMP_FIELD = "event_timestamp"; // HOUSEKEEPINGENTITY DEFAULT VALUES public static final String DATABASE_NAME_VALUE = "some_database"; diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java new file mode 100644 index 00000000..46bec68b --- /dev/null +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/utils/ResultSetToBeekeeperHistoryMapper.java @@ -0,0 +1,27 @@ +package com.expediagroup.beekeeper.integration.utils; + +import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.EVENT_DETAILS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.HOUSEKEEPING_STATUS_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.ID_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.LIFECYCLE_TYPE_FIELD; +import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_FIELD; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; + +public class ResultSetToBeekeeperHistoryMapper { + + public static BeekeeperHistory mapToBeekeeperHistory(ResultSet resultSet) throws SQLException { + return BeekeeperHistory.builder() + .id(resultSet.getLong(ID_FIELD)) + .databaseName(resultSet.getString(DATABASE_NAME_FIELD)) + .tableName(resultSet.getString(TABLE_NAME_FIELD)) + .lifecycleType(resultSet.getString(LIFECYCLE_TYPE_FIELD)) + .housekeepingStatus(resultSet.getString(HOUSEKEEPING_STATUS_FIELD)) + .eventDetails(resultSet.getString(EVENT_DETAILS_FIELD)) + .build(); + } +} From ea6d5a9d5308f7c677b624d13b72f80b0a9666fc Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Mon, 25 Nov 2024 17:22:20 +0000 Subject: [PATCH 09/13] Simplify event details --- .../core/service/BeekeeperHistoryService.java | 15 ++----- .../BeekeeperHistoryRepositoryTest.java | 43 +++++++------------ .../service/BeekeeperHistoryServiceTest.java | 11 +---- ...etadataSchedulerApiaryIntegrationTest.java | 8 +++- .../BeekeeperIntegrationTestBase.java | 32 +------------- ...ekeeperMetadataCleanupIntegrationTest.java | 26 +++++++++++ .../BeekeeperPathCleanupIntegrationTest.java | 28 ++++++++++++ ...cedPathSchedulerApiaryIntegrationTest.java | 10 +++-- 8 files changed, 89 insertions(+), 84 deletions(-) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java index 856945e4..52cc858d 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -25,7 +25,7 @@ public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryReposi } public void saveHistory(HousekeepingEntity housekeepingEntity, String status) { - String eventDetails = createEventDetails(housekeepingEntity); + String eventDetails = getEventDetails(housekeepingEntity); BeekeeperHistory event = BeekeeperHistory.builder() .id(housekeepingEntity.getId()) @@ -41,16 +41,7 @@ public void saveHistory(HousekeepingEntity housekeepingEntity, String status) { beekeeperHistoryRepository.save(event); } - private String createEventDetails(HousekeepingEntity entity) { - ObjectMapper mapper = new ObjectMapper(); - mapper.findAndRegisterModules(); - mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - mapper.registerModule(new JavaTimeModule()); - try { - return mapper.writeValueAsString(entity); - } catch (JsonProcessingException e) { - log.warn("Error encountered writing object as string", e); - } - return "{}"; + private String getEventDetails(HousekeepingEntity entity) { + return entity.toString(); } } diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java index dc09c2ad..2c2993cd 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/repository/BeekeeperHistoryRepositoryTest.java @@ -22,11 +22,6 @@ import org.springframework.test.context.junit.jupiter.SpringExtension; import org.springframework.test.context.support.AnnotationConfigContextLoader; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - import com.expediagroup.beekeeper.core.TestApplication; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingPath; @@ -53,7 +48,6 @@ public class BeekeeperHistoryRepositoryTest { private static final int PAGE = 0; private static final int PAGE_SIZE = 500; - protected ObjectMapper mapper; @Autowired private BeekeeperHistoryRepository repository; @@ -61,17 +55,12 @@ public class BeekeeperHistoryRepositoryTest { @BeforeEach public void setupDb() { repository.deleteAll(); - - mapper = new ObjectMapper(); - mapper.findAndRegisterModules(); - mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - mapper.registerModule(new JavaTimeModule()); } @Test - public void typicalSave() throws JsonProcessingException { - BeekeeperHistory expiredEntry = createExpiredEventDetails(SCHEDULED); - BeekeeperHistory unreferencedEntry = createUnreferencedEventDetails(SCHEDULED); + public void typicalSave() { + BeekeeperHistory expiredEntry = createExpiredEvent(SCHEDULED); + BeekeeperHistory unreferencedEntry = createUnreferencedEvent(SCHEDULED); repository.save(expiredEntry); repository.save(unreferencedEntry); @@ -86,10 +75,10 @@ public void typicalSave() throws JsonProcessingException { } @Test - public void expired_multipleStatuses() throws JsonProcessingException { - BeekeeperHistory scheduledEntry = createExpiredEventDetails(SCHEDULED); - BeekeeperHistory deletedEntry = createExpiredEventDetails(DELETED); - BeekeeperHistory failedEntry = createExpiredEventDetails(FAILED); + public void expired_multipleStatuses() { + BeekeeperHistory scheduledEntry = createExpiredEvent(SCHEDULED); + BeekeeperHistory deletedEntry = createExpiredEvent(DELETED); + BeekeeperHistory failedEntry = createExpiredEvent(FAILED); repository.save(scheduledEntry); repository.save(deletedEntry); @@ -101,10 +90,10 @@ public void expired_multipleStatuses() throws JsonProcessingException { } @Test - public void unreferenced_multipleStatuses() throws JsonProcessingException { - BeekeeperHistory scheduledEntry = createUnreferencedEventDetails(SCHEDULED); - BeekeeperHistory deletedEntry = createUnreferencedEventDetails(DELETED); - BeekeeperHistory failedEntry = createUnreferencedEventDetails(FAILED); + public void unreferenced_multipleStatuses() { + BeekeeperHistory scheduledEntry = createUnreferencedEvent(SCHEDULED); + BeekeeperHistory deletedEntry = createUnreferencedEvent(DELETED); + BeekeeperHistory failedEntry = createUnreferencedEvent(FAILED); repository.save(scheduledEntry); repository.save(deletedEntry); @@ -115,7 +104,7 @@ public void unreferenced_multipleStatuses() throws JsonProcessingException { assertThat(historyList.size()).isEqualTo(3); } - protected BeekeeperHistory createExpiredEventDetails(HousekeepingStatus status) throws JsonProcessingException { + protected BeekeeperHistory createExpiredEvent(HousekeepingStatus status) { HousekeepingMetadata entity = HousekeepingMetadata.builder() .cleanupAttempts(3) .cleanupDelay(PeriodDuration.parse("P1D")) @@ -124,11 +113,10 @@ protected BeekeeperHistory createExpiredEventDetails(HousekeepingStatus status) .modifiedTimestamp(COLUMN_TIMESTAMP) .build(); - String stringDetails = mapper.writeValueAsString(entity); - return createHistoryEntry("EXPIRED", status, stringDetails); + return createHistoryEntry("EXPIRED", status, entity.toString()); } - protected BeekeeperHistory createUnreferencedEventDetails(HousekeepingStatus status) throws JsonProcessingException { + protected BeekeeperHistory createUnreferencedEvent(HousekeepingStatus status) { HousekeepingPath entity = HousekeepingPath.builder() .cleanupAttempts(3) .cleanupDelay(PeriodDuration.parse("P1D")) @@ -136,8 +124,7 @@ protected BeekeeperHistory createUnreferencedEventDetails(HousekeepingStatus sta .modifiedTimestamp(COLUMN_TIMESTAMP) .build(); - String stringDetails = mapper.writeValueAsString(entity); - return createHistoryEntry("UNREFERENCED", status, stringDetails); + return createHistoryEntry("UNREFERENCED", status, entity.toString()); } protected BeekeeperHistory createHistoryEntry(String lifecycleType, HousekeepingStatus status, diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java index 9a08d244..9ffba99d 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java @@ -17,9 +17,6 @@ import org.mockito.junit.jupiter.MockitoExtension; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; @@ -106,11 +103,7 @@ private HousekeepingPath createHousekeepingPath() { .build(); } - private String createEventDetails(HousekeepingEntity housekeepingEntity) throws JsonProcessingException { - ObjectMapper mapper = new ObjectMapper(); - mapper.findAndRegisterModules(); - mapper.disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS); - mapper.registerModule(new JavaTimeModule()); - return mapper.writeValueAsString(housekeepingEntity); + private String createEventDetails(HousekeepingEntity housekeepingEntity) { + return housekeepingEntity.toString(); } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index a5bb23c1..4f12fadd 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -215,14 +215,18 @@ public void expiredMetadataMultipleAlterPartitionTableEvents() throws SQLExcepti } @Test - public void addEventToHistoryTable() throws SQLException, IOException, URISyntaxException { + public void testEventAddedToHistoryTable() throws SQLException, IOException, URISyntaxException { CreateTableSqsMessage createTableSqsMessage = new CreateTableSqsMessage(LOCATION_A, true); amazonSQS.sendMessage(sendMessageRequest(createTableSqsMessage.getFormattedString())); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(EXPIRED) == 1); List beekeeperHistory = getBeekeeperHistory(EXPIRED); - System.out.println(beekeeperHistory); + BeekeeperHistory history = beekeeperHistory.get(0); + assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); + assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); + assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString()); + assertThat(history.getHousekeepingStatus()).isEqualTo(SCHEDULED.name()); } @Test diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java index 0bbefc49..6060f5cf 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperIntegrationTestBase.java @@ -64,7 +64,6 @@ import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingPath; -import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.core.model.PeriodDuration; import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; @@ -205,16 +204,6 @@ protected void insertExpiredMetadata(HousekeepingMetadata metadata) throws SQLEx values); } - protected void insertBeekeeperHistory(BeekeeperHistory beekeeperHistory) throws SQLException { - String values = Stream.of(beekeeperHistory.getId().toString(), beekeeperHistory.getEventTimestamp(), - beekeeperHistory.getDatabaseName(), beekeeperHistory.getTableName(), beekeeperHistory.getLifecycleType(), - beekeeperHistory.getHousekeepingStatus(), beekeeperHistory.getEventDetails()) - .map(s -> s == null ? null : "\"" + s + "\"") - .collect(Collectors.joining(", ")); - - mySQLTestUtils.insertToTable(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, BEEKEEPER_HISTORY_FIELDS, values); - } - protected int getUnreferencedPathsRowCount() throws SQLException { return mySQLTestUtils .getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HOUSEKEEPING_PATH_TABLE_NAME, @@ -234,7 +223,7 @@ protected int getUpdatedExpiredMetadataRowCount() throws SQLException { } protected int getBeekeeperHistoryRowCount(LifecycleEventType lifecycleEventType) throws SQLException { - String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; + String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; return mySQLTestUtils.getTableRowCount(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, format(filter, lifecycleEventType)); @@ -267,7 +256,7 @@ protected List getExpiredMetadata() throws SQLException { } protected List getBeekeeperHistory(LifecycleEventType lifecycleEventType) throws SQLException { - String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; + String filter = "WHERE " + LIFECYCLE_TYPE_FIELD + " = '%s'"; List history = new ArrayList<>(); ResultSet resultSet = mySQLTestUtils.getTableRows(BEEKEEPER_DB_NAME, BEEKEEPER_HISTORY_TABLE_NAME, format(filter, lifecycleEventType)); @@ -318,21 +307,4 @@ private HousekeepingMetadata createHousekeepingMetadata( .clientId(CLIENT_ID_FIELD) .build(); } - - private BeekeeperHistory createBeekeeperHistory( - String tableName, - LifecycleEventType lifecycleEventType, - HousekeepingStatus housekeepingStatus, - String eventDetails - ) { - return BeekeeperHistory.builder() - .id(id++) - .eventTimestamp(CREATION_TIMESTAMP_VALUE) - .databaseName(DATABASE_NAME_VALUE) - .tableName(tableName) - .lifecycleType(lifecycleEventType.toString()) - .housekeepingStatus(housekeepingStatus.name()) - .eventDetails(eventDetails) - .build(); - } } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java index d0e52df9..b2eeda28 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java @@ -26,6 +26,9 @@ import static com.expediagroup.beekeeper.cleanup.monitoring.DeletedMetadataReporter.METRIC_NAME; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DISABLED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.LONG_CLEANUP_DELAY_VALUE; @@ -67,6 +70,8 @@ import com.google.common.collect.ImmutableMap; import com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter; +import com.expediagroup.beekeeper.core.model.LifecycleEventType; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.integration.utils.HiveTestUtils; import com.expediagroup.beekeeper.metadata.cleanup.BeekeeperMetadataCleanup; @@ -368,6 +373,27 @@ public void onlyCleanupLocationWhenPartitionExists() throws TException, SQLExcep assertThat(amazonS3.doesObjectExist(BUCKET, PARTITIONED_OBJECT_KEY)).isTrue(); } + @Test + public void testEventAddedToHistoryTable() throws TException, SQLException { + hiveTestUtils.createTable(UNPARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, false); + amazonS3.putObject(BUCKET, UNPARTITIONED_OBJECT_KEY, TABLE_DATA); + + insertExpiredMetadata(UNPARTITIONED_TABLE_PATH, null); + await() + .atMost(TIMEOUT, TimeUnit.SECONDS) + .until(() -> getBeekeeperHistoryRowCount(LifecycleEventType.EXPIRED) == 1); + + assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isFalse(); + assertThat(amazonS3.doesObjectExist(BUCKET, UNPARTITIONED_OBJECT_KEY)).isFalse(); + + List beekeeperHistory = getBeekeeperHistory(UNREFERENCED); + BeekeeperHistory history = beekeeperHistory.get(0); + assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); + assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); + assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString()); + assertThat(history.getHousekeepingStatus()).isEqualTo(DELETED.name()); + } + @Test public void metrics() throws Exception { Table table = hiveTestUtils.createTable(PARTITIONED_TABLE_PATH, TABLE_NAME_VALUE, true); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java index 0b8c29b0..af0bbd7f 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java @@ -21,6 +21,9 @@ import static com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter.METRIC_NAME; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; +import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.TABLE_NAME_VALUE; @@ -52,6 +55,7 @@ import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.model.CreateBucketRequest; +import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.path.cleanup.BeekeeperPathCleanup; @@ -244,6 +248,30 @@ public void cleanupSentinelForNonEmptyParent() throws SQLException { assertThat(amazonS3.doesObjectExist(BUCKET, tableSentinel)).isTrue(); } + @Test + public void testEventAddedToHistoryTable() throws SQLException { + amazonS3.putObject(BUCKET, OBJECT_KEY1, CONTENT); + amazonS3.putObject(BUCKET, OBJECT_KEY_OTHER, CONTENT); + amazonS3.putObject(BUCKET, OBJECT_KEY_SENTINEL, ""); + + String path = "s3://" + BUCKET + "/" + OBJECT_KEY1; + insertUnreferencedPath(path); + + await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); + + assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY1)).isFalse(); + // deleting a file shouldn't delete a folder sentinel + assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY_SENTINEL)).isTrue(); + assertThat(amazonS3.doesObjectExist(BUCKET, OBJECT_KEY_OTHER)).isTrue(); + + List beekeeperHistory = getBeekeeperHistory(UNREFERENCED); + BeekeeperHistory history = beekeeperHistory.get(0); + assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); + assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); + assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString()); + assertThat(history.getHousekeepingStatus()).isEqualTo(DELETED.name()); + } + @Test public void metrics() throws SQLException { amazonS3.putObject(BUCKET, OBJECT_KEY1, CONTENT); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index d43ab5dc..4c311174 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -199,15 +199,19 @@ public void unreferencedDropTableEvent() throws SQLException, IOException, URISy } @Test - public void addBeekeeperHistoryEvent() throws IOException, URISyntaxException, SQLException { + public void testEventAddedToHistoryTable() throws IOException, URISyntaxException, SQLException { AlterTableSqsMessage alterTableSqsMessage = new AlterTableSqsMessage("s3://bucket/tableLocation", "s3://bucket/oldTableLocation", true, true); amazonSQS.sendMessage(sendMessageRequest(alterTableSqsMessage.getFormattedString())); await().atMost(TIMEOUT, TimeUnit.SECONDS).until(() -> getBeekeeperHistoryRowCount(UNREFERENCED) == 1); - List beekeeperHistory = getBeekeeperHistory(EXPIRED); - System.out.println(beekeeperHistory); + List beekeeperHistory = getBeekeeperHistory(UNREFERENCED); + BeekeeperHistory history = beekeeperHistory.get(0); + assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); + assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); + assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString()); + assertThat(history.getHousekeepingStatus()).isEqualTo(SCHEDULED.name()); } @Test From c8f3010e5b72e20744d7b28ad9b282fa1fd0106b Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Mon, 25 Nov 2024 17:46:19 +0000 Subject: [PATCH 10/13] Integration tests --- .../integration/BeekeeperMetadataCleanupIntegrationTest.java | 4 +--- .../integration/BeekeeperPathCleanupIntegrationTest.java | 4 +--- ...keeperUnreferencedPathSchedulerApiaryIntegrationTest.java | 5 ++--- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java index b2eeda28..e2c3942e 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperMetadataCleanupIntegrationTest.java @@ -26,9 +26,7 @@ import static com.expediagroup.beekeeper.cleanup.monitoring.DeletedMetadataReporter.METRIC_NAME; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DISABLED; -import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; -import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; import static com.expediagroup.beekeeper.integration.CommonTestVariables.LONG_CLEANUP_DELAY_VALUE; @@ -386,7 +384,7 @@ public void testEventAddedToHistoryTable() throws TException, SQLException { assertThat(metastoreClient.tableExists(DATABASE_NAME_VALUE, TABLE_NAME_VALUE)).isFalse(); assertThat(amazonS3.doesObjectExist(BUCKET, UNPARTITIONED_OBJECT_KEY)).isFalse(); - List beekeeperHistory = getBeekeeperHistory(UNREFERENCED); + List beekeeperHistory = getBeekeeperHistory(EXPIRED); BeekeeperHistory history = beekeeperHistory.get(0); assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java index af0bbd7f..8f4c4c17 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperPathCleanupIntegrationTest.java @@ -21,8 +21,6 @@ import static com.expediagroup.beekeeper.cleanup.monitoring.BytesDeletedReporter.METRIC_NAME; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; -import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; -import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.DATABASE_NAME_VALUE; @@ -268,7 +266,7 @@ public void testEventAddedToHistoryTable() throws SQLException { BeekeeperHistory history = beekeeperHistory.get(0); assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); - assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString()); + assertThat(history.getLifecycleType()).isEqualTo(UNREFERENCED.toString()); assertThat(history.getHousekeepingStatus()).isEqualTo(DELETED.name()); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index 4c311174..9d3678d1 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -20,7 +20,6 @@ import static org.testcontainers.containers.localstack.LocalStackContainer.Service.SQS; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; -import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; import static com.expediagroup.beekeeper.integration.CommonTestVariables.AWS_REGION; import static com.expediagroup.beekeeper.integration.CommonTestVariables.CLEANUP_ATTEMPTS_VALUE; @@ -159,7 +158,7 @@ public void unreferencedAlterPartitionEvent() throws SQLException, IOException, public void unreferencedMultipleAlterPartitionEvent() throws IOException, SQLException, URISyntaxException { List .of(new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation", - "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), + "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation2", "s3://bucket/table/partitionLocation2", "s3://bucket/table/partitionLocation", true, true)) .forEach(msg -> amazonSQS.sendMessage(sendMessageRequest(msg.getFormattedString()))); @@ -210,7 +209,7 @@ public void testEventAddedToHistoryTable() throws IOException, URISyntaxExceptio BeekeeperHistory history = beekeeperHistory.get(0); assertThat(history.getDatabaseName()).isEqualTo(DATABASE_NAME_VALUE); assertThat(history.getTableName()).isEqualTo(TABLE_NAME_VALUE); - assertThat(history.getLifecycleType()).isEqualTo(EXPIRED.toString()); + assertThat(history.getLifecycleType()).isEqualTo(UNREFERENCED.toString()); assertThat(history.getHousekeepingStatus()).isEqualTo(SCHEDULED.name()); } From e6a5faaedad81f078fb5b68f9cb517669f16203a Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Wed, 27 Nov 2024 12:42:02 +0000 Subject: [PATCH 11/13] Dont save dry-runs in history table --- .../core/service/BeekeeperHistoryService.java | 10 +++----- .../service/BeekeeperHistoryServiceTest.java | 5 ++-- .../handler/ExpiredMetadataHandler.java | 11 ++++----- .../handler/ExpiredMetadataHandlerTest.java | 24 +++++++++---------- .../cleanup/handler/GenericPathHandler.java | 2 +- .../handler/GenericPathHandlerTest.java | 6 ++--- ...ble.sql => V2_4__Create_history_table.sql} | 0 ...dHousekeepingMetadataSchedulerService.java | 2 +- ...encedHousekeepingPathSchedulerService.java | 2 +- ...sekeepingMetadataSchedulerServiceTest.java | 10 ++++---- ...dHousekeepingPathSchedulerServiceTest.java | 4 ++-- 11 files changed, 35 insertions(+), 41 deletions(-) rename beekeeper-scheduler-apiary/src/main/resources/db/migration/{V3_1__Create_history_table.sql => V2_4__Create_history_table.sql} (100%) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java index 52cc858d..aa86d976 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -5,12 +5,8 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.SerializationFeature; -import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; - import com.expediagroup.beekeeper.core.model.HousekeepingEntity; +import com.expediagroup.beekeeper.core.model.HousekeepingStatus; import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory; import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository; @@ -24,7 +20,7 @@ public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryReposi this.beekeeperHistoryRepository = beekeeperHistoryRepository; } - public void saveHistory(HousekeepingEntity housekeepingEntity, String status) { + public void saveHistory(HousekeepingEntity housekeepingEntity, HousekeepingStatus status) { String eventDetails = getEventDetails(housekeepingEntity); BeekeeperHistory event = BeekeeperHistory.builder() @@ -33,7 +29,7 @@ public void saveHistory(HousekeepingEntity housekeepingEntity, String status) { .databaseName(housekeepingEntity.getDatabaseName()) .tableName(housekeepingEntity.getTableName()) .lifecycleType(housekeepingEntity.getLifecycleType()) - .housekeepingStatus(status) + .housekeepingStatus(status.name()) .eventDetails(eventDetails) .build(); diff --git a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java index 9ffba99d..4037bd36 100644 --- a/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java +++ b/beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryServiceTest.java @@ -2,6 +2,7 @@ import static org.mockito.Mockito.verify; +import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED; import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.EXPIRED; import static com.expediagroup.beekeeper.core.model.LifecycleEventType.UNREFERENCED; @@ -51,7 +52,7 @@ void expiredHistory() throws JsonProcessingException { String details = createEventDetails(metadata); BeekeeperHistory history = createHistoryEvent(metadata, details, "DELETED"); - beekeeperHistoryService.saveHistory(metadata, "DELETED"); + beekeeperHistoryService.saveHistory(metadata, DELETED); verify(repository).save(history); } @@ -61,7 +62,7 @@ void unreferencedHistory() throws JsonProcessingException { String details = createEventDetails(path); BeekeeperHistory history = createHistoryEvent(path, details, "SCHEDULED"); - beekeeperHistoryService.saveHistory(path, "SCHEDULED"); + beekeeperHistoryService.saveHistory(path, SCHEDULED); verify(repository).save(history); } diff --git a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java index a17a3d76..7cf54786 100644 --- a/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java +++ b/beekeeper-metadata-cleanup/src/main/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandler.java @@ -79,10 +79,8 @@ public Slice findRecordsToClean(LocalDateTime instant, Pag public void cleanupMetadata(HousekeepingMetadata housekeepingMetadata, LocalDateTime instant, boolean dryRunEnabled) { try (CleanerClient client = cleanerClientFactory.newInstance()) { boolean deleted = cleanup(client, housekeepingMetadata, instant, dryRunEnabled); - if (deleted) { - if (!dryRunEnabled) { - updateAttemptsAndStatus(housekeepingMetadata, DELETED); - } + if (deleted && !dryRunEnabled) { + updateAttemptsAndStatus(housekeepingMetadata, DELETED); saveHistory(housekeepingMetadata, DELETED, dryRunEnabled); } } catch (Exception e) { @@ -189,10 +187,9 @@ private Long countPartitionsForDatabaseAndTable( private void saveHistory(HousekeepingMetadata metadata, HousekeepingStatus housekeepingStatus, boolean dryRunEnabled) { - String status = String.valueOf(housekeepingStatus); if (dryRunEnabled) { - status = "DRY_RUN_" + housekeepingStatus; + return; } - historyService.saveHistory(metadata, status); + historyService.saveHistory(metadata, housekeepingStatus); } } diff --git a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java index 3b3fa63f..1790d4c5 100644 --- a/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java +++ b/beekeeper-metadata-cleanup/src/test/java/com/expediagroup/beekeeper/metadata/cleanup/handler/ExpiredMetadataHandlerTest.java @@ -123,7 +123,7 @@ public void typicalRunDroppingTable() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -143,7 +143,7 @@ public void typicalDroppingTable_DryRun() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata, never()).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository, never()).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq("DRY_RUN_DELETED")); + verify(beekeeperHistoryService, never()).saveHistory(any(), any()); } @Test @@ -164,7 +164,7 @@ public void typicalRunDroppingPartition() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -185,7 +185,7 @@ public void dontDropTableWithInvalidPath() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED)); } @Test @@ -204,7 +204,7 @@ public void dontDropTableWithInvalidPath_DryRun() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata, never()).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository, never()).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq("DRY_RUN_SKIPPED")); + verify(beekeeperHistoryService, never()).saveHistory(any(), any()); } @Test @@ -247,7 +247,7 @@ public void dontDropTableOrPathWhenTableDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -263,7 +263,7 @@ public void dontDropPartitionWithInvalidPartitionPath() { verify(housekeepingMetadata, never()).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(SKIPPED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED)); } @Test @@ -283,7 +283,7 @@ public void dontDropPartitionWhenTableDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -302,7 +302,7 @@ public void dontDropPathWhenPartitionDoesntExist() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(DELETED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); } @Test @@ -324,7 +324,7 @@ public void expectedTableDropFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE)); } @Test @@ -342,7 +342,7 @@ public void expectedPathDeleteFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE)); } @Test @@ -360,6 +360,6 @@ public void expectedPartitionDropFailure() { verify(housekeepingMetadata).setCleanupAttempts(1); verify(housekeepingMetadata).setHousekeepingStatus(FAILED); verify(housekeepingMetadataRepository).save(housekeepingMetadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE)); } } diff --git a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java index a249f445..e9a440d4 100644 --- a/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java +++ b/beekeeper-path-cleanup/src/main/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandler.java @@ -112,6 +112,6 @@ private void updateStatus(HousekeepingPath housekeepingPath, HousekeepingStatus } private void saveHistory(HousekeepingPath housekeepingPath, HousekeepingStatus status) { - beekeeperHistoryService.saveHistory(housekeepingPath, status.name()); + beekeeperHistoryService.saveHistory(housekeepingPath, status); } } diff --git a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java index 52393a91..b9110c74 100644 --- a/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java +++ b/beekeeper-path-cleanup/src/test/java/com/expediagroup/beekeeper/path/cleanup/handler/GenericPathHandlerTest.java @@ -89,7 +89,7 @@ public void typicalProcessPage() { verify(mockPath).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(DELETED); verify(housekeepingPathRepository).save(mockPath); - verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(DELETED)); assertThat(pageable).isEqualTo(pageable); } @@ -103,7 +103,7 @@ public void processPageFails() { verify(mockPath).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(FAILED); verify(housekeepingPathRepository).save(mockPath); - verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_DELETE)); assertThat(pageable).isEqualTo(pageable); } @@ -117,7 +117,7 @@ public void processPageInvalidPath() { verify(mockPath, never()).setCleanupAttempts(1); verify(mockPath).setHousekeepingStatus(SKIPPED); verify(housekeepingPathRepository).save(mockPath); - verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(SKIPPED)); assertThat(pageable).isEqualTo(pageable); } } diff --git a/beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql b/beekeeper-scheduler-apiary/src/main/resources/db/migration/V2_4__Create_history_table.sql similarity index 100% rename from beekeeper-scheduler-apiary/src/main/resources/db/migration/V3_1__Create_history_table.sql rename to beekeeper-scheduler-apiary/src/main/resources/db/migration/V2_4__Create_history_table.sql diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java index af6a9e9e..768d804c 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerService.java @@ -151,6 +151,6 @@ private boolean isPartitionedTable(HousekeepingMetadata housekeepingMetadata) { } private void saveHistory(HousekeepingMetadata housekeepingMetadata, HousekeepingStatus status) { - beekeeperHistoryService.saveHistory(housekeepingMetadata, status.name()); + beekeeperHistoryService.saveHistory(housekeepingMetadata, status); } } diff --git a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java index 34aacfa9..6125944a 100644 --- a/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java +++ b/beekeeper-scheduler/src/main/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerService.java @@ -71,6 +71,6 @@ public void scheduleForHousekeeping(HousekeepingEntity housekeepingEntity) { } private void saveHistory(HousekeepingPath housekeepingPath, HousekeepingStatus status) { - beekeeperHistoryService.saveHistory(housekeepingPath, status.name()); + beekeeperHistoryService.saveHistory(housekeepingPath, status); } } diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java index b85a25e9..1e95ee09 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/ExpiredHousekeepingMetadataSchedulerServiceTest.java @@ -74,7 +74,7 @@ public void typicalCreateScheduleForHousekeeping() { expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); verify(housekeepingMetadataRepository).save(metadata); - verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED.name()); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED); } @Test @@ -91,7 +91,7 @@ public void typicalCreatePartitionScheduleForHousekeeping() { expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata); verify(housekeepingMetadataRepository).save(metadata); - verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED.name()); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED); } @Test @@ -110,7 +110,7 @@ public void typicalUpdateScheduleForHousekeepingWhenChangingCleanupDelay() { verify(existingTable).setClientId(metadata.getClientId()); verify(existingTable).setCleanupDelay(metadata.getCleanupDelay()); verify(housekeepingMetadataRepository).save(existingTable); - verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED.name()); + verify(beekeeperHistoryService).saveHistory(metadata, SCHEDULED); } @Test @@ -136,7 +136,7 @@ public void typicalUpdatePartitionedTableWithShorterCleanupDelay() { verify(existingTable).setCleanupTimestamp(CREATION_TIMESTAMP.plus(Duration.parse("P30D"))); verify(housekeepingMetadataRepository).save(existingTable); - verify(beekeeperHistoryService).saveHistory(any(), eq(SCHEDULED.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(SCHEDULED)); } @Test @@ -154,7 +154,7 @@ public void scheduleFails() { .isThrownBy(() -> expiredHousekeepingMetadataSchedulerService.scheduleForHousekeeping(metadata)) .withMessage(format("Unable to schedule %s", metadata)); verify(housekeepingMetadataRepository).save(metadata); - verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_SCHEDULE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_SCHEDULE)); } private HousekeepingMetadata createHousekeepingMetadataPartition() { diff --git a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java index 2f1e6ba0..27173b23 100644 --- a/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java +++ b/beekeeper-scheduler/src/test/java/com/expediagroup/beekeeper/scheduler/service/UnreferencedHousekeepingPathSchedulerServiceTest.java @@ -65,7 +65,7 @@ public void typicalScheduleForHousekeeping() { verify(housekeepingPathRepository).save(path); - verify(beekeeperHistoryService).saveHistory(path, SCHEDULED.name()); + verify(beekeeperHistoryService).saveHistory(path, SCHEDULED); } @Test @@ -88,6 +88,6 @@ public void scheduleFails() { .isThrownBy(() -> unreferencedHousekeepingPathSchedulerService.scheduleForHousekeeping(path)) .withMessage(format("Unable to schedule %s", path)); verify(housekeepingPathRepository).save(path); - verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_SCHEDULE.name())); + verify(beekeeperHistoryService).saveHistory(any(), eq(FAILED_TO_SCHEDULE)); } } From 9217a6b8c7bb3d72cc58344dc6563d2dc56920b6 Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Wed, 27 Nov 2024 13:40:24 +0000 Subject: [PATCH 12/13] Implement suggestion from PR --- .../beekeeper/core/service/BeekeeperHistoryService.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java index aa86d976..6f025eed 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -21,7 +21,7 @@ public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryReposi } public void saveHistory(HousekeepingEntity housekeepingEntity, HousekeepingStatus status) { - String eventDetails = getEventDetails(housekeepingEntity); + String eventDetails = housekeepingEntity.toString(); BeekeeperHistory event = BeekeeperHistory.builder() .id(housekeepingEntity.getId()) @@ -36,8 +36,4 @@ public void saveHistory(HousekeepingEntity housekeepingEntity, HousekeepingStatu log.info("Saving activity in Beekeeper History table; {}", event); beekeeperHistoryRepository.save(event); } - - private String getEventDetails(HousekeepingEntity entity) { - return entity.toString(); - } } From bd86171efb190c93bf2378048684d339e1fe7302 Mon Sep 17 00:00:00 2001 From: jgreenstevens Date: Wed, 27 Nov 2024 13:44:06 +0000 Subject: [PATCH 13/13] Implement suggestion from PR --- .../beekeeper/core/service/BeekeeperHistoryService.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java index 6f025eed..dc4b7427 100644 --- a/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java +++ b/beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/service/BeekeeperHistoryService.java @@ -21,8 +21,6 @@ public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryReposi } public void saveHistory(HousekeepingEntity housekeepingEntity, HousekeepingStatus status) { - String eventDetails = housekeepingEntity.toString(); - BeekeeperHistory event = BeekeeperHistory.builder() .id(housekeepingEntity.getId()) .eventTimestamp(LocalDateTime.now()) @@ -30,7 +28,7 @@ public void saveHistory(HousekeepingEntity housekeepingEntity, HousekeepingStatu .tableName(housekeepingEntity.getTableName()) .lifecycleType(housekeepingEntity.getLifecycleType()) .housekeepingStatus(status.name()) - .eventDetails(eventDetails) + .eventDetails(housekeepingEntity.toString()) .build(); log.info("Saving activity in Beekeeper History table; {}", event);