Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: Add Beekeeper history table #184

Merged
merged 15 commits into from
Nov 29, 2024
21 changes: 21 additions & 0 deletions beekeeper-core/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,27 @@
<version>27.1-jre</version>
</dependency>

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-annotations</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-core</artifactId>
<version>${jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.datatype</groupId>
<artifactId>jackson-datatype-jsr310</artifactId>
<version>${jackson.version}</version>
</dependency>

<!-- micrometer -->
<dependency>
<groupId>io.micrometer</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@
public enum HousekeepingStatus {
SCHEDULED,
FAILED,
FAILED_TO_DELETE,
FAILED_TO_SCHEDULE,
DELETED,
DISABLED,
SKIPPED
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
package com.expediagroup.beekeeper.core.model.history;

import java.time.LocalDateTime;

import javax.persistence.Column;
import javax.persistence.Entity;
import javax.persistence.GeneratedValue;
import javax.persistence.GenerationType;
import javax.persistence.Id;
import javax.persistence.Table;

import lombok.Builder;
import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;

import com.expediagroup.beekeeper.core.monitoring.MetricTag;
import com.expediagroup.beekeeper.core.monitoring.Taggable;

@Data
@NoArgsConstructor
@Entity
@Table(name = "beekeeper_history")
public class BeekeeperHistory implements Taggable {

@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
private Long id;

@EqualsAndHashCode.Exclude
@Column(name = "event_timestamp", nullable = false, updatable = false)
private LocalDateTime eventTimestamp;

@Column(name = "database_name", nullable = false)
private String databaseName;

@Column(name = "table_name", nullable = false)
private String tableName;

@Column(name = "lifecycle_type", nullable = false)
private String lifecycleType;

@Column(name = "housekeeping_status", nullable = false)
private String housekeepingStatus;

@Column(name = "event_details", columnDefinition = "TEXT")
private String eventDetails;

@Builder
public BeekeeperHistory(
Long id,
LocalDateTime eventTimestamp,
String databaseName,
String tableName,
String lifecycleType,
String housekeepingStatus,
String eventDetails
) {
this.id = id;
this.eventTimestamp = eventTimestamp;
this.databaseName = databaseName;
this.tableName = tableName;
this.lifecycleType = lifecycleType;
this.housekeepingStatus = housekeepingStatus;
this.eventDetails = eventDetails;
}

@Override
public MetricTag getMetricTag() {
return new MetricTag("table", String.join(".", databaseName, tableName));
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
package com.expediagroup.beekeeper.core.repository;

import org.springframework.data.domain.Pageable;
import org.springframework.data.domain.Slice;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.PagingAndSortingRepository;
import org.springframework.data.repository.query.Param;

import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;

public interface BeekeeperHistoryRepository extends PagingAndSortingRepository<BeekeeperHistory, Long>,
JpaSpecificationExecutor<BeekeeperHistory> {

@Query(value = "from BeekeeperHistory t where t.lifecycleType = :lifecycle")
Slice<BeekeeperHistory> findRecordsByLifecycleType(
@Param("lifecycle") String lifecycle,
Pageable pageable);
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package com.expediagroup.beekeeper.core.service;

import java.time.LocalDateTime;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.expediagroup.beekeeper.core.model.HousekeepingEntity;
import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;
import com.expediagroup.beekeeper.core.repository.BeekeeperHistoryRepository;

public class BeekeeperHistoryService {

private static final Logger log = LoggerFactory.getLogger(BeekeeperHistoryService.class);

private final BeekeeperHistoryRepository beekeeperHistoryRepository;

public BeekeeperHistoryService(BeekeeperHistoryRepository beekeeperHistoryRepository) {
this.beekeeperHistoryRepository = beekeeperHistoryRepository;
}

public void saveHistory(HousekeepingEntity housekeepingEntity, HousekeepingStatus status) {
BeekeeperHistory event = BeekeeperHistory.builder()
.id(housekeepingEntity.getId())
.eventTimestamp(LocalDateTime.now())
.databaseName(housekeepingEntity.getDatabaseName())
.tableName(housekeepingEntity.getTableName())
.lifecycleType(housekeepingEntity.getLifecycleType())
.housekeepingStatus(status.name())
.eventDetails(housekeepingEntity.toString())
.build();

log.info("Saving activity in Beekeeper History table; {}", event);
beekeeperHistoryRepository.save(event);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
package com.expediagroup.beekeeper.core.repository;

import static org.assertj.core.api.Assertions.assertThat;

import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.DELETED;
import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.FAILED;
import static com.expediagroup.beekeeper.core.model.HousekeepingStatus.SCHEDULED;

import java.time.LocalDateTime;
import java.time.ZoneId;
import java.util.List;

import org.assertj.core.util.Lists;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.PageRequest;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit.jupiter.SpringExtension;
import org.springframework.test.context.support.AnnotationConfigContextLoader;

import com.expediagroup.beekeeper.core.TestApplication;
import com.expediagroup.beekeeper.core.model.HousekeepingMetadata;
import com.expediagroup.beekeeper.core.model.HousekeepingPath;
import com.expediagroup.beekeeper.core.model.HousekeepingStatus;
import com.expediagroup.beekeeper.core.model.PeriodDuration;
import com.expediagroup.beekeeper.core.model.history.BeekeeperHistory;

@ExtendWith(SpringExtension.class)
@TestPropertySource(properties = {
"hibernate.data-source.driver-class-name=org.h2.Driver",
"hibernate.dialect=org.hibernate.dialect.H2Dialect",
"hibernate.hbm2ddl.auto=create",
"spring.jpa.show-sql=true",
"spring.datasource.url=jdbc:h2:mem:beekeeper;DB_CLOSE_DELAY=-1;DB_CLOSE_ON_EXIT=FALSE;MODE=MySQL" })
@ContextConfiguration(classes = { TestApplication.class }, loader = AnnotationConfigContextLoader.class)
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_CLASS)
public class BeekeeperHistoryRepositoryTest {

protected static final String DATABASE_NAME = "database";
protected static final String TABLE_NAME = "table";
protected static final PeriodDuration CLEANUP_DELAY = PeriodDuration.parse("P3D");
protected static final LocalDateTime COLUMN_TIMESTAMP = LocalDateTime.now(ZoneId.of("UTC"));
protected static final LocalDateTime EVENT_TIMESTAMP = COLUMN_TIMESTAMP.plus(CLEANUP_DELAY);

private static final int PAGE = 0;
private static final int PAGE_SIZE = 500;

@Autowired
private BeekeeperHistoryRepository repository;

@BeforeEach
public void setupDb() {
repository.deleteAll();
}

@Test
public void typicalSave() {
BeekeeperHistory expiredEntry = createExpiredEvent(SCHEDULED);
BeekeeperHistory unreferencedEntry = createUnreferencedEvent(SCHEDULED);

repository.save(expiredEntry);
repository.save(unreferencedEntry);

List<BeekeeperHistory> historyList = Lists.newArrayList(
repository.findRecordsByLifecycleType("EXPIRED", PageRequest.of(PAGE, PAGE_SIZE)));
assertThat(historyList.size()).isEqualTo(1);

historyList = Lists.newArrayList(
repository.findRecordsByLifecycleType("UNREFERENCED", PageRequest.of(PAGE, PAGE_SIZE)));
assertThat(historyList.size()).isEqualTo(1);
}

@Test
public void expired_multipleStatuses() {
BeekeeperHistory scheduledEntry = createExpiredEvent(SCHEDULED);
BeekeeperHistory deletedEntry = createExpiredEvent(DELETED);
BeekeeperHistory failedEntry = createExpiredEvent(FAILED);

repository.save(scheduledEntry);
repository.save(deletedEntry);
repository.save(failedEntry);

List<BeekeeperHistory> historyList = Lists.newArrayList(
repository.findRecordsByLifecycleType("EXPIRED", PageRequest.of(PAGE, PAGE_SIZE)));
assertThat(historyList.size()).isEqualTo(3);
}

@Test
public void unreferenced_multipleStatuses() {
BeekeeperHistory scheduledEntry = createUnreferencedEvent(SCHEDULED);
BeekeeperHistory deletedEntry = createUnreferencedEvent(DELETED);
BeekeeperHistory failedEntry = createUnreferencedEvent(FAILED);

repository.save(scheduledEntry);
repository.save(deletedEntry);
repository.save(failedEntry);

List<BeekeeperHistory> historyList = Lists.newArrayList(
repository.findRecordsByLifecycleType("UNREFERENCED", PageRequest.of(PAGE, PAGE_SIZE)));
assertThat(historyList.size()).isEqualTo(3);
}

protected BeekeeperHistory createExpiredEvent(HousekeepingStatus status) {
HousekeepingMetadata entity = HousekeepingMetadata.builder()
.cleanupAttempts(3)
.cleanupDelay(PeriodDuration.parse("P1D"))
.partitionName("event_date")
.creationTimestamp(COLUMN_TIMESTAMP)
.modifiedTimestamp(COLUMN_TIMESTAMP)
.build();

return createHistoryEntry("EXPIRED", status, entity.toString());
}

protected BeekeeperHistory createUnreferencedEvent(HousekeepingStatus status) {
HousekeepingPath entity = HousekeepingPath.builder()
.cleanupAttempts(3)
.cleanupDelay(PeriodDuration.parse("P1D"))
.creationTimestamp(COLUMN_TIMESTAMP)
.modifiedTimestamp(COLUMN_TIMESTAMP)
.build();

return createHistoryEntry("UNREFERENCED", status, entity.toString());
}

protected BeekeeperHistory createHistoryEntry(String lifecycleType, HousekeepingStatus status,
String eventDetails) {
return BeekeeperHistory.builder()
.eventTimestamp(EVENT_TIMESTAMP)
.databaseName(DATABASE_NAME)
.tableName(TABLE_NAME)
.lifecycleType(lifecycleType)
.housekeepingStatus(status.name())
.eventDetails(eventDetails)
.build();
}
}
Loading