diff --git a/CHANGELOG.md b/CHANGELOG.md index b401f7c8..c77c347d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [3.5.4] - 2023-09-14 +### Fixed +- Added localisation normalization so locations like `s3:/a/b` and `s3:/a/b/` will be considered the same and path won't be scheduled for deletion. + ## [3.5.3] - 2023-03-08 ### Changed - Upgrade `Springboot` from `2.4.4` to `2.7.9`. diff --git a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java index a64a58f8..eda7fcbd 100644 --- a/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java +++ b/beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2021 Expedia, Inc. + * Copyright (C) 2019-2023 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/beekeeper-core/pom.xml b/beekeeper-core/pom.xml index 10b7d28f..bec18ce8 100644 --- a/beekeeper-core/pom.xml +++ b/beekeeper-core/pom.xml @@ -1,5 +1,6 @@ - + 4.0.0 @@ -84,4 +85,21 @@ + + + + com.mycila + license-maven-plugin + + + + src/main/java/com/expediagroup/beekeeper/core/model/PeriodDuration.java + src/main/java/com/expediagroup/beekeeper/core/monitoring/TimedTaggableAspect.java + src/test/java/com/expediagroup/beekeeper/core/model/PeriodDurationTest.java + + + + + + diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizer.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizer.java new file mode 100644 index 00000000..ae3d388a --- /dev/null +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizer.java @@ -0,0 +1,32 @@ +/** + * Copyright (C) 2019-2023 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.scheduler.apiary.filter; + +import org.apache.commons.lang3.StringUtils; + +/** + * Attempts to normalize string representing a location of a Hive Table, could have a wide variety of schemes, s3, s3a, + * hdfs etc.. + */ +public class LocationNormalizer { + + public String normalize(String location) { + //Not using File.seperator here we might not know what the location would be using. (Beekeeper might run on Windows...) + location =StringUtils.stripEnd(location, "/"); + return location; + } + +} diff --git a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilter.java b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilter.java index 1b9bc09c..a2147a02 100644 --- a/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilter.java +++ b/beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilter.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2023 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -26,7 +26,18 @@ @Component public class LocationOnlyUpdateListenerEventFilter implements ListenerEventFilter { + + private final LocationNormalizer locationNormalizer; + + public LocationOnlyUpdateListenerEventFilter () { + this.locationNormalizer = new LocationNormalizer(); + } + + public LocationOnlyUpdateListenerEventFilter (LocationNormalizer locationNormaliser) { + this.locationNormalizer = locationNormaliser; + } + @Override public boolean isFiltered(ListenerEvent listenerEvent, LifecycleEventType lifecycleEventType) { EventType eventType = listenerEvent.getEventType(); @@ -44,6 +55,11 @@ public boolean isFiltered(ListenerEvent listenerEvent, LifecycleEventType lifecy } private boolean isLocationSame(String oldLocation, String location) { - return location == null || oldLocation == null || oldLocation.equals(location); + if (location == null || oldLocation == null) { + return true; + } + String normalizedOldLocation = locationNormalizer.normalize(oldLocation); + String normalizedLocation = locationNormalizer.normalize(location); + return normalizedOldLocation.equals(normalizedLocation); } } diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizerTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizerTest.java new file mode 100644 index 00000000..c320bafd --- /dev/null +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizerTest.java @@ -0,0 +1,44 @@ +/** + * Copyright (C) 2019-2023 Expedia, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.expediagroup.beekeeper.scheduler.apiary.filter; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; + +class LocationNormalizerTest { + + private LocationNormalizer normalizer = new LocationNormalizer(); + + @Test + void noChange() { + assertThat(normalizer.normalize("s3://bucket/prefix")).isEqualTo("s3://bucket/prefix"); + assertThat(normalizer.normalize("/bucket/prefix")).isEqualTo("/bucket/prefix"); + assertThat(normalizer.normalize("hdfs://bucket/prefix")).isEqualTo("hdfs://bucket/prefix"); + assertThat(normalizer.normalize("")).isEqualTo(""); + assertThat(normalizer.normalize(null)).isEqualTo(null); + assertThat(normalizer.normalize("foo")).isEqualTo("foo"); + assertThat(normalizer.normalize("foo/bar")).isEqualTo("foo/bar"); + } + + @Test + void normalizeTrailingSlash() { + assertThat(normalizer.normalize("s3://bucket/prefix/")).isEqualTo("s3://bucket/prefix"); + assertThat(normalizer.normalize("s3://bucket/prefix///")).isEqualTo("s3://bucket/prefix"); + assertThat(normalizer.normalize("hdfs://bucket/prefix/")).isEqualTo("hdfs://bucket/prefix"); + } + +} diff --git a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilterTest.java b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilterTest.java index 368e430f..15c61669 100644 --- a/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilterTest.java +++ b/beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilterTest.java @@ -1,5 +1,5 @@ /** - * Copyright (C) 2019-2020 Expedia, Inc. + * Copyright (C) 2019-2023 Expedia, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -113,6 +113,43 @@ public void alterPartitionEventMetadataOnlyNullOldLocation() { assertThat(filter).isTrue(); } + @Test + public void alterTableEventMetadataLocationTrailingSlashOldShouldFilter() { + when(alterTableEvent.getEventType()).thenReturn(EventType.ALTER_TABLE); + when(alterTableEvent.getOldTableLocation()).thenReturn("/foo/bar/"); + when(alterTableEvent.getTableLocation()).thenReturn("/foo/bar"); + boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterTableEvent, UNREFERENCED); + assertThat(filter).isTrue(); + } + + @Test + public void alterPartitionEventMetadataLocationTrailingSlashOldShouldFilter() { + when(alterPartitionEvent.getEventType()).thenReturn(EventType.ALTER_PARTITION); + when(alterPartitionEvent.getOldPartitionLocation()).thenReturn("/foo/bar/"); + when(alterPartitionEvent.getPartitionLocation()).thenReturn("/foo/bar"); + boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterPartitionEvent, UNREFERENCED); + assertThat(filter).isTrue(); + } + + + @Test + public void alterTableEventMetadataLocationTrailingSlashNewShouldFilter() { + when(alterTableEvent.getEventType()).thenReturn(EventType.ALTER_TABLE); + when(alterTableEvent.getOldTableLocation()).thenReturn("/foo/bar"); + when(alterTableEvent.getTableLocation()).thenReturn("/foo/bar/"); + boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterTableEvent, UNREFERENCED); + assertThat(filter).isTrue(); + } + + @Test + public void alterPartitionEventMetadataLocationTrailingSlashNewShouldFilter() { + when(alterPartitionEvent.getEventType()).thenReturn(EventType.ALTER_PARTITION); + when(alterPartitionEvent.getOldPartitionLocation()).thenReturn("/foo/bar"); + when(alterPartitionEvent.getPartitionLocation()).thenReturn("/foo/bar/"); + boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterPartitionEvent, UNREFERENCED); + assertThat(filter).isTrue(); + } + @Test public void dropTableEvent() { when(dropTableEvent.getEventType()).thenReturn(EventType.DROP_TABLE);