Fix for trailing /

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.5.4] - 2023-09-14
+### Fixed
+- Added localisation normalization so locations like `s3:/a/b` and `s3:/a/b/` will be considered the same and path won't be scheduled for deletion.
+
 ## [3.5.3] - 2023-03-08
 ### Changed
 - Upgrade `Springboot` from `2.4.4` to `2.7.9`.

beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2021 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

beekeeper-core/pom.xml

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <parent>
@@ -84,4 +85,21 @@
     </dependency>
   </dependencies>
 
+  <build>
+    <plugins>
+      <plugin>
+        <groupId>com.mycila</groupId>
+        <artifactId>license-maven-plugin</artifactId>
+        <configuration>
+          <!-- excluding files that don't need a header update -->
+          <excludes>
+            <exclude>src/main/java/com/expediagroup/beekeeper/core/model/PeriodDuration.java</exclude>
+            <exclude>src/main/java/com/expediagroup/beekeeper/core/monitoring/TimedTaggableAspect.java</exclude>
+            <exclude>src/test/java/com/expediagroup/beekeeper/core/model/PeriodDurationTest.java</exclude>
+          </excludes>
+        </configuration>
+      </plugin>
+    </plugins>
+  </build>
+
 </project>

beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizer.java

@@ -0,0 +1,32 @@
+/**
+ * Copyright (C) 2019-2023 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.expediagroup.beekeeper.scheduler.apiary.filter;
+
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Attempts to normalize string representing a location of a Hive Table, could have a wide variety of schemes, s3, s3a,
+ * hdfs etc..
+ */
+public class LocationNormalizer {
+
+  public String normalize(String location) {
+    //Not using File.seperator here we might not know what the location would be using. (Beekeeper might run on Windows...)
+    location =StringUtils.stripEnd(location, "/");
+    return location;
+  }
+
+}

beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilter.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2020 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,18 @@
 
 @Component
 public class LocationOnlyUpdateListenerEventFilter implements ListenerEventFilter {
+
+  private final LocationNormalizer locationNormalizer;
+
+  public LocationOnlyUpdateListenerEventFilter () {
+    this.locationNormalizer = new LocationNormalizer();
+  }
+
+  public LocationOnlyUpdateListenerEventFilter (LocationNormalizer locationNormaliser) {
+    this.locationNormalizer = locationNormaliser;
+  }
 
+
   @Override
   public boolean isFiltered(ListenerEvent listenerEvent, LifecycleEventType lifecycleEventType) {
     EventType eventType = listenerEvent.getEventType();
@@ -44,6 +55,11 @@ public boolean isFiltered(ListenerEvent listenerEvent, LifecycleEventType lifecy
   }
 
   private boolean isLocationSame(String oldLocation, String location) {
-    return location == null || oldLocation == null || oldLocation.equals(location);
+    if (location == null || oldLocation == null) {
+      return true;
+    }
+    String normalizedOldLocation = locationNormalizer.normalize(oldLocation);
+    String normalizedLocation = locationNormalizer.normalize(location);
+    return normalizedOldLocation.equals(normalizedLocation);
   }
 }

beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizerTest.java

@@ -0,0 +1,44 @@
+/**
+ * Copyright (C) 2019-2023 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.expediagroup.beekeeper.scheduler.apiary.filter;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class LocationNormalizerTest {
+
+  private LocationNormalizer normalizer = new LocationNormalizer();
+
+  @Test
+  void noChange() {
+    assertThat(normalizer.normalize("s3://bucket/prefix")).isEqualTo("s3://bucket/prefix");
+    assertThat(normalizer.normalize("/bucket/prefix")).isEqualTo("/bucket/prefix");
+    assertThat(normalizer.normalize("hdfs://bucket/prefix")).isEqualTo("hdfs://bucket/prefix");
+    assertThat(normalizer.normalize("")).isEqualTo("");
+    assertThat(normalizer.normalize(null)).isEqualTo(null);
+    assertThat(normalizer.normalize("foo")).isEqualTo("foo");
+    assertThat(normalizer.normalize("foo/bar")).isEqualTo("foo/bar");
+  }
+
+  @Test
+  void normalizeTrailingSlash() {
+    assertThat(normalizer.normalize("s3://bucket/prefix/")).isEqualTo("s3://bucket/prefix");
+    assertThat(normalizer.normalize("s3://bucket/prefix///")).isEqualTo("s3://bucket/prefix");
+    assertThat(normalizer.normalize("hdfs://bucket/prefix/")).isEqualTo("hdfs://bucket/prefix");
+  }
+
+}

beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilterTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2020 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -113,6 +113,43 @@ public void alterPartitionEventMetadataOnlyNullOldLocation() {
     assertThat(filter).isTrue();
   }
 
+  @Test
+  public void alterTableEventMetadataLocationTrailingSlashOldShouldFilter() {
+    when(alterTableEvent.getEventType()).thenReturn(EventType.ALTER_TABLE);
+    when(alterTableEvent.getOldTableLocation()).thenReturn("/foo/bar/");
+    when(alterTableEvent.getTableLocation()).thenReturn("/foo/bar");
+    boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterTableEvent, UNREFERENCED);
+    assertThat(filter).isTrue();
+  }
+
+  @Test
+  public void alterPartitionEventMetadataLocationTrailingSlashOldShouldFilter() {
+    when(alterPartitionEvent.getEventType()).thenReturn(EventType.ALTER_PARTITION);
+    when(alterPartitionEvent.getOldPartitionLocation()).thenReturn("/foo/bar/");
+    when(alterPartitionEvent.getPartitionLocation()).thenReturn("/foo/bar");
+    boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterPartitionEvent, UNREFERENCED);
+    assertThat(filter).isTrue();
+  }
+
+
+  @Test
+  public void alterTableEventMetadataLocationTrailingSlashNewShouldFilter() {
+    when(alterTableEvent.getEventType()).thenReturn(EventType.ALTER_TABLE);
+    when(alterTableEvent.getOldTableLocation()).thenReturn("/foo/bar");
+    when(alterTableEvent.getTableLocation()).thenReturn("/foo/bar/");
+    boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterTableEvent, UNREFERENCED);
+    assertThat(filter).isTrue();
+  }
+
+  @Test
+  public void alterPartitionEventMetadataLocationTrailingSlashNewShouldFilter() {
+    when(alterPartitionEvent.getEventType()).thenReturn(EventType.ALTER_PARTITION);
+    when(alterPartitionEvent.getOldPartitionLocation()).thenReturn("/foo/bar");
+    when(alterPartitionEvent.getPartitionLocation()).thenReturn("/foo/bar/");
+    boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterPartitionEvent, UNREFERENCED);
+    assertThat(filter).isTrue();
+  }
+
   @Test
   public void dropTableEvent() {
     when(dropTableEvent.getEventType()).thenReturn(EventType.DROP_TABLE);