From 804be2f17021f1cdd113c91683853aa765a92059 Mon Sep 17 00:00:00 2001 From: javsanbel2 Date: Wed, 27 Nov 2024 11:48:26 +0100 Subject: [PATCH] cleaning up --- .../beekeeper/cleanup/hive/HiveClient.java | 17 -------- .../cleanup/metadata/CleanerClient.java | 2 - .../cleanup/validation/IcebergValidator.java | 5 +-- ...etadataSchedulerApiaryIntegrationTest.java | 37 +++-------------- ...cedPathSchedulerApiaryIntegrationTest.java | 18 +-------- beekeeper-scheduler-apiary/pom.xml | 28 ------------- .../scheduler/apiary/context/CommonBeans.java | 40 ------------------- .../TableParameterListenerEventFilter.java | 1 - .../apiary/service/SchedulerApiary.java | 14 +------ .../apiary/service/SchedulerApiaryTest.java | 31 +------------- 10 files changed, 11 insertions(+), 182 deletions(-) diff --git a/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/hive/HiveClient.java b/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/hive/HiveClient.java index 007afeb1..d1009a60 100644 --- a/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/hive/HiveClient.java +++ b/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/hive/HiveClient.java @@ -126,23 +126,6 @@ public Map getTableProperties(String databaseName, String tableN } } - @Override - public String getOutputFormat(String databaseName, String tableName) { - String result = null; - try { - Table table = client.getTable(databaseName, tableName); - if (table.getSd() != null) { - result = table.getSd().getOutputFormat(); - } - } catch (NoSuchObjectException e) { - log.warn("Table {}.{} does not exist", databaseName, tableName); - } catch (TException e) { - throw new BeekeeperException( - "Unexpected exception when getting output format for \"" + databaseName + "." + tableName + ".", e); - } - return result; - } - @Override public void close() { client.close(); diff --git a/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/metadata/CleanerClient.java b/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/metadata/CleanerClient.java index 1946302b..ee1a3a58 100644 --- a/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/metadata/CleanerClient.java +++ b/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/metadata/CleanerClient.java @@ -27,6 +27,4 @@ public interface CleanerClient extends Closeable { boolean tableExists(String databaseName, String tableName); Map getTableProperties(String databaseName, String tableName); - - String getOutputFormat(String databaseName, String tableName); } diff --git a/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidator.java b/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidator.java index c4167232..55fb07aa 100644 --- a/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidator.java +++ b/beekeeper-cleanup/src/main/java/com/expediagroup/beekeeper/cleanup/validation/IcebergValidator.java @@ -49,9 +49,8 @@ public void throwExceptionIfIceberg(String databaseName, String tableName) { Map parameters = client.getTableProperties(databaseName, tableName); String tableType = parameters.getOrDefault("table_type", "").toLowerCase(); String format = parameters.getOrDefault("format", "").toLowerCase(); - String outputFormat = client.getOutputFormat(databaseName, tableName); - if (tableType.contains("iceberg") || format.contains("iceberg") || (outputFormat != null - && outputFormat.toLowerCase().contains("iceberg"))) { + String metadataLocation = parameters.getOrDefault("metadata_location", "").toLowerCase(); + if (tableType.contains("iceberg") || format.contains("iceberg") || !metadataLocation.isEmpty()) { throw new BeekeeperIcebergException( format("Iceberg table %s.%s is not currently supported in Beekeeper.", databaseName, tableName)); } diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java index 321a89ad..ccbf19c6 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest.java @@ -36,7 +36,6 @@ import java.util.Set; import java.util.concurrent.TimeUnit; -import org.apache.hadoop.hive.metastore.HiveMetaStoreClient; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; @@ -46,7 +45,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -56,7 +54,6 @@ import io.micrometer.core.instrument.composite.CompositeMeterRegistry; import com.amazonaws.services.sqs.AmazonSQS; -import com.amazonaws.services.sqs.model.CreateQueueResult; import com.amazonaws.services.sqs.model.PurgeQueueRequest; import com.amazonaws.services.sqs.model.SendMessageRequest; @@ -67,27 +64,19 @@ import com.expediagroup.beekeeper.integration.model.AlterTableSqsMessage; import com.expediagroup.beekeeper.integration.model.CreateTableSqsMessage; import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; -import com.expediagroup.beekeeper.integration.utils.HiveTestUtils; import com.expediagroup.beekeeper.scheduler.apiary.BeekeeperSchedulerApiary; -import com.hotels.beeju.extensions.ThriftHiveMetaStoreJUnitExtension; - @Testcontainers public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { private static final int TIMEOUT = 30; - private static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; private static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; - private static final String METASTORE_URI_PROPERTY = "properties.metastore-uri"; private static final String QUEUE = "apiary-receiver-queue"; private static final String SCHEDULED_EXPIRED_METRIC = "metadata-scheduled"; private static final String HEALTHCHECK_URI = "http://localhost:8080/actuator/health"; private static final String PROMETHEUS_URI = "http://localhost:8080/actuator/prometheus"; - private static final String S3_ACCESS_KEY = "access"; - private static final String S3_SECRET_KEY = "secret"; - private static final String PARTITION_KEYS = "{ \"event_date\": \"date\", \"event_hour\": \"smallint\"}"; private static final String PARTITION_A_VALUES = "[ \"2020-01-01\", \"0\" ]"; private static final String PARTITION_B_VALUES = "[ \"2020-01-01\", \"1\" ]"; @@ -95,44 +84,30 @@ public class BeekeeperExpiredMetadataSchedulerApiaryIntegrationTest extends Beek private static final String PARTITION_B_NAME = "event_date=2020-01-01/event_hour=1"; private static final String LOCATION_A = "s3://bucket/table1/partition"; private static final String LOCATION_B = "s3://bucket/table2/partition"; - private static final String TABLE_PATH = "/tmp/bucket/" + DATABASE_NAME_VALUE + "/" + TABLE_NAME_VALUE + "/"; @Container private static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); private static AmazonSQS amazonSQS; - private static String queueUrl; - - @RegisterExtension - public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( - DATABASE_NAME_VALUE); - - private HiveTestUtils hiveTestUtils; - private HiveMetaStoreClient metastoreClient; @BeforeAll public static void init() { - System.setProperty(DRY_RUN_ENABLED_PROPERTY, "false"); - amazonSQS = ContainerTestUtils.sqsClient(SQS_CONTAINER, AWS_REGION); - CreateQueueResult queue = amazonSQS.createQueue(QUEUE); - queueUrl = queue.getQueueUrl(); + String queueUrl = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); System.setProperty(APIARY_QUEUE_URL_PROPERTY, queueUrl); + + amazonSQS = ContainerTestUtils.sqsClient(SQS_CONTAINER, AWS_REGION); + amazonSQS.createQueue(QUEUE); } @AfterAll public static void teardown() { System.clearProperty(APIARY_QUEUE_URL_PROPERTY); - System.clearProperty(DRY_RUN_ENABLED_PROPERTY); amazonSQS.shutdown(); } @BeforeEach public void setup() { - System.setProperty(METASTORE_URI_PROPERTY, thriftHiveMetaStore.getThriftConnectionUri()); - metastoreClient = thriftHiveMetaStore.client(); - hiveTestUtils = new HiveTestUtils(metastoreClient); - - amazonSQS.purgeQueue(new PurgeQueueRequest(queueUrl)); + amazonSQS.purgeQueue(new PurgeQueueRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE))); executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {})); await().atMost(Duration.ONE_MINUTE).until(BeekeeperSchedulerApiary::isRunning); } @@ -255,7 +230,7 @@ public void prometheus() { } private SendMessageRequest sendMessageRequest(String payload) { - return new SendMessageRequest(queueUrl, payload); + return new SendMessageRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE), payload); } private void assertExpiredMetadata(HousekeepingMetadata actual, String expectedPath, String partitionName) { diff --git a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java index af66ac2e..1c3fd3a4 100644 --- a/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java +++ b/beekeeper-integration-tests/src/test/java/com/expediagroup/beekeeper/integration/BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest.java @@ -45,7 +45,6 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.RegisterExtension; import org.testcontainers.containers.localstack.LocalStackContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -67,14 +66,11 @@ import com.expediagroup.beekeeper.integration.utils.ContainerTestUtils; import com.expediagroup.beekeeper.scheduler.apiary.BeekeeperSchedulerApiary; -import com.hotels.beeju.extensions.ThriftHiveMetaStoreJUnitExtension; - @Testcontainers public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest extends BeekeeperIntegrationTestBase { private static final int TIMEOUT = 5; private static final String APIARY_QUEUE_URL_PROPERTY = "properties.apiary.queue-url"; - private static final String DRY_RUN_ENABLED_PROPERTY = "properties.dry-run-enabled"; private static final String QUEUE = "apiary-receiver-queue"; private static final String SCHEDULED_ORPHANED_METRIC = "paths-scheduled"; @@ -85,10 +81,6 @@ public class BeekeeperUnreferencedPathSchedulerApiaryIntegrationTest extends Bee private static final LocalStackContainer SQS_CONTAINER = ContainerTestUtils.awsContainer(SQS); private static AmazonSQS amazonSQS; - @RegisterExtension - public ThriftHiveMetaStoreJUnitExtension thriftHiveMetaStore = new ThriftHiveMetaStoreJUnitExtension( - DATABASE_NAME_VALUE); - @BeforeAll public static void init() { String queueUrl = ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE); @@ -101,17 +93,12 @@ public static void init() { @AfterAll public static void teardown() { System.clearProperty(APIARY_QUEUE_URL_PROPERTY); - System.clearProperty("properties.metastore-uri"); - System.clearProperty("properties.dry-run-enabled"); amazonSQS.shutdown(); } @BeforeEach public void setup() { - System.setProperty("properties.metastore-uri", thriftHiveMetaStore.getThriftConnectionUri()); - System.setProperty("properties.dry-run-enabled", "false"); - amazonSQS.purgeQueue(new PurgeQueueRequest(ContainerTestUtils.queueUrl(SQS_CONTAINER, QUEUE))); executorService.execute(() -> BeekeeperSchedulerApiary.main(new String[] {})); await().atMost(Duration.ONE_MINUTE).until(BeekeeperSchedulerApiary::isRunning); @@ -121,9 +108,6 @@ public void setup() { public void stop() throws InterruptedException { BeekeeperSchedulerApiary.stop(); executorService.awaitTermination(5, TimeUnit.SECONDS); - - System.clearProperty("properties.metastore-uri"); - System.clearProperty("properties.dry-run-enabled"); } @Test @@ -173,7 +157,7 @@ public void unreferencedAlterPartitionEvent() throws SQLException, IOException, public void unreferencedMultipleAlterPartitionEvent() throws IOException, SQLException, URISyntaxException { List .of(new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation", - "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), + "s3://bucket/table/partitionLocation", "s3://bucket/table/unreferencedPartitionLocation", true, true), new AlterPartitionSqsMessage("s3://bucket/table/expiredTableLocation2", "s3://bucket/table/partitionLocation2", "s3://bucket/table/partitionLocation", true, true)) .forEach(msg -> amazonSQS.sendMessage(sendMessageRequest(msg.getFormattedString()))); diff --git a/beekeeper-scheduler-apiary/pom.xml b/beekeeper-scheduler-apiary/pom.xml index 9ee238f2..bb0b2a7d 100644 --- a/beekeeper-scheduler-apiary/pom.xml +++ b/beekeeper-scheduler-apiary/pom.xml @@ -11,31 +11,8 @@ beekeeper-scheduler-apiary - - 2.8.1 - 2.3.7 - UTF-8 - - - - - org.apache.hadoop - hadoop-mapreduce-client-core - ${hadoop.version} - - - org.slf4j - slf4j-log4j12 - - - javax.servlet - servlet-api - - - - com.amazonaws aws-java-sdk-sts @@ -46,11 +23,6 @@ beekeeper-scheduler ${project.version} - - com.expediagroup - beekeeper-cleanup - ${project.version} - ch.qos.logback diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java index fc2ee418..492017c6 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/context/CommonBeans.java @@ -17,9 +17,7 @@ import java.util.EnumMap; import java.util.List; -import java.util.function.Supplier; -import org.apache.hadoop.hive.conf.HiveConf; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.autoconfigure.domain.EntityScan; @@ -39,9 +37,6 @@ import com.expedia.apiary.extensions.receiver.common.messaging.MessageReader; import com.expedia.apiary.extensions.receiver.sqs.messaging.SqsMessageReader; -import com.expediagroup.beekeeper.cleanup.hive.HiveClientFactory; -import com.expediagroup.beekeeper.cleanup.metadata.CleanerClientFactory; -import com.expediagroup.beekeeper.cleanup.validation.IcebergValidator; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.scheduler.apiary.filter.EventTypeListenerEventFilter; import com.expediagroup.beekeeper.scheduler.apiary.filter.ListenerEventFilter; @@ -57,10 +52,6 @@ import com.expediagroup.beekeeper.scheduler.apiary.messaging.RetryingMessageReader; import com.expediagroup.beekeeper.scheduler.service.SchedulerService; -import com.hotels.hcommon.hive.metastore.client.api.CloseableMetaStoreClient; -import com.hotels.hcommon.hive.metastore.client.closeable.CloseableMetaStoreClientFactory; -import com.hotels.hcommon.hive.metastore.client.supplier.HiveMetaStoreClientSupplier; - @Configuration @ComponentScan(basePackages = { "com.expediagroup.beekeeper.core", "com.expediagroup.beekeeper.scheduler" }) @EntityScan(basePackages = { "com.expediagroup.beekeeper.core" }) @@ -148,35 +139,4 @@ public BeekeeperEventReader eventReader( return new MessageReaderAdapter(messageReader, handlers); } - - @Bean - public HiveConf hiveConf(@Value("${properties.metastore-uri}") String metastoreUri) { - HiveConf conf = new HiveConf(); - conf.setVar(HiveConf.ConfVars.METASTOREURIS, metastoreUri); - return conf; - } - - @Bean - public CloseableMetaStoreClientFactory metaStoreClientFactory() { - return new CloseableMetaStoreClientFactory(); - } - - @Bean - Supplier metaStoreClientSupplier( - CloseableMetaStoreClientFactory metaStoreClientFactory, HiveConf hiveConf) { - String name = "beekeeper-scheduler-apiary"; - return new HiveMetaStoreClientSupplier(metaStoreClientFactory, hiveConf, name); - } - - @Bean(name = "hiveClientFactory") - public CleanerClientFactory clientFactory( - Supplier metaStoreClientSupplier, - @Value("${properties.dry-run-enabled}") boolean dryRunEnabled) { - return new HiveClientFactory(metaStoreClientSupplier, dryRunEnabled); - } - - @Bean - public IcebergValidator icebergValidator(CleanerClientFactory clientFactory) { - return new IcebergValidator(clientFactory); - } } diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/TableParameterListenerEventFilter.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/TableParameterListenerEventFilter.java index 2c46f936..d3be3b3d 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/TableParameterListenerEventFilter.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/TableParameterListenerEventFilter.java @@ -37,4 +37,3 @@ public boolean isFiltered(ListenerEvent listenerEvent, LifecycleEventType lifecy return !Boolean.parseBoolean(tableParameters.get(lifecycleEventType.getTableParameterName())); } } - diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiary.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiary.java index e7722530..fc27e5be 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiary.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiary.java @@ -20,7 +20,6 @@ import java.io.IOException; import java.util.EnumMap; import java.util.List; -import java.util.Map; import java.util.Optional; import org.slf4j.Logger; @@ -29,16 +28,12 @@ import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; -import com.expediagroup.beekeeper.cleanup.validation.IcebergValidator; import com.expediagroup.beekeeper.core.error.BeekeeperException; -import com.expediagroup.beekeeper.core.error.BeekeeperIcebergException; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.LifecycleEventType; import com.expediagroup.beekeeper.scheduler.apiary.messaging.BeekeeperEventReader; -import com.expediagroup.beekeeper.scheduler.apiary.messaging.MessageReaderAdapter; import com.expediagroup.beekeeper.scheduler.apiary.model.BeekeeperEvent; import com.expediagroup.beekeeper.scheduler.service.SchedulerService; -import com.expedia.apiary.extensions.receiver.common.event.ListenerEvent; @Component public class SchedulerApiary { @@ -47,17 +42,14 @@ public class SchedulerApiary { private final BeekeeperEventReader beekeeperEventReader; private final EnumMap schedulerServiceMap; - private final IcebergValidator icebergValidator; @Autowired public SchedulerApiary( BeekeeperEventReader beekeeperEventReader, - EnumMap schedulerServiceMap, - IcebergValidator icebergValidator + EnumMap schedulerServiceMap ) { this.beekeeperEventReader = beekeeperEventReader; this.schedulerServiceMap = schedulerServiceMap; - this.icebergValidator = icebergValidator; } @Transactional @@ -69,13 +61,9 @@ public void scheduleBeekeeperEvent() { for (HousekeepingEntity entity : housekeepingEntities) { try { - icebergValidator.throwExceptionIfIceberg(entity.getDatabaseName(), entity.getTableName()); - LifecycleEventType eventType = LifecycleEventType.valueOf(entity.getLifecycleType()); SchedulerService scheduler = schedulerServiceMap.get(eventType); scheduler.scheduleForHousekeeping(entity); - } catch (BeekeeperIcebergException e) { - log.warn("Iceberg table are not supported in Beekeeper. Deleting message from queue", e); } catch (Exception e) { throw new BeekeeperException(format( "Unable to schedule %s deletion for entity, this message will go back on the queue", diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiaryTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiaryTest.java index 351c3562..73ab2b6d 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiaryTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/service/SchedulerApiaryTest.java @@ -20,7 +20,6 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.mock; @@ -44,9 +43,7 @@ import com.expedia.apiary.extensions.receiver.common.messaging.MessageEvent; -import com.expediagroup.beekeeper.cleanup.validation.IcebergValidator; import com.expediagroup.beekeeper.core.error.BeekeeperException; -import com.expediagroup.beekeeper.core.error.BeekeeperIcebergException; import com.expediagroup.beekeeper.core.model.HousekeepingEntity; import com.expediagroup.beekeeper.core.model.HousekeepingMetadata; import com.expediagroup.beekeeper.core.model.HousekeepingPath; @@ -65,7 +62,6 @@ public class SchedulerApiaryTest { @Mock private BeekeeperEventReader beekeeperEventReader; @Mock private HousekeepingPath path; @Mock private HousekeepingMetadata table; - @Mock private IcebergValidator icebergValidator; private SchedulerApiary scheduler; @@ -74,7 +70,7 @@ public void init() { EnumMap schedulerMap = new EnumMap<>(LifecycleEventType.class); schedulerMap.put(UNREFERENCED, pathSchedulerService); schedulerMap.put(EXPIRED, tableSchedulerService); - scheduler = new SchedulerApiary(beekeeperEventReader, schedulerMap, icebergValidator); + scheduler = new SchedulerApiary(beekeeperEventReader, schedulerMap); } @Test @@ -82,7 +78,6 @@ public void typicalPathSchedule() { Optional event = Optional.of(newHousekeepingEvent(path, UNREFERENCED)); when(beekeeperEventReader.read()).thenReturn(event); scheduler.scheduleBeekeeperEvent(); - verify(icebergValidator).throwExceptionIfIceberg(path.getDatabaseName(), path.getTableName()); verify(pathSchedulerService).scheduleForHousekeeping(path); verifyNoInteractions(tableSchedulerService); verify(beekeeperEventReader).delete(event.get()); @@ -94,7 +89,6 @@ public void typicalTableSchedule() { when(beekeeperEventReader.read()).thenReturn(event); scheduler.scheduleBeekeeperEvent(); - verify(icebergValidator).throwExceptionIfIceberg(table.getDatabaseName(), table.getTableName()); verify(tableSchedulerService).scheduleForHousekeeping(table); verifyNoInteractions(pathSchedulerService); verify(beekeeperEventReader).delete(event.get()); @@ -105,7 +99,6 @@ public void typicalNoSchedule() { when(beekeeperEventReader.read()).thenReturn(Optional.empty()); scheduler.scheduleBeekeeperEvent(); - verifyNoInteractions(icebergValidator); verifyNoInteractions(pathSchedulerService); verifyNoInteractions(tableSchedulerService); verify(beekeeperEventReader, times(0)).delete(any()); @@ -121,7 +114,6 @@ public void housekeepingPathRepositoryThrowsException() { scheduler.scheduleBeekeeperEvent(); fail("Should have thrown exception"); } catch (Exception e) { - verify(icebergValidator).throwExceptionIfIceberg(path.getDatabaseName(), path.getTableName()); verify(pathSchedulerService).scheduleForHousekeeping(path); verify(beekeeperEventReader, times(0)).delete(any()); verifyNoInteractions(tableSchedulerService); @@ -142,7 +134,6 @@ public void housekeepingTableRepositoryThrowsException() { scheduler.scheduleBeekeeperEvent(); fail("Should have thrown exception"); } catch (Exception e) { - verify(icebergValidator).throwExceptionIfIceberg(table.getDatabaseName(), table.getTableName()); verify(tableSchedulerService).scheduleForHousekeeping(table); verify(beekeeperEventReader, times(0)).delete(any()); verifyNoInteractions(pathSchedulerService); @@ -153,26 +144,6 @@ public void housekeepingTableRepositoryThrowsException() { } } - @Test - public void icebergValidatorThrowsException() { - String databaseName = "database"; - String tableName = "table"; - when(path.getDatabaseName()).thenReturn(databaseName); - when(path.getTableName()).thenReturn(tableName); - Optional event = Optional.of(newHousekeepingEvent(path, UNREFERENCED)); - when(beekeeperEventReader.read()).thenReturn(event); - - doThrow(new BeekeeperIcebergException("Iceberg table")) - .when(icebergValidator).throwExceptionIfIceberg(eq(databaseName), eq(tableName)); - - scheduler.scheduleBeekeeperEvent(); - - verify(icebergValidator).throwExceptionIfIceberg(databaseName, tableName); - verifyNoInteractions(pathSchedulerService); - verifyNoInteractions(tableSchedulerService); - verify(beekeeperEventReader).delete(event.get()); - } - @Test public void typicalClose() throws Exception { scheduler.close();