Fix for trailing /

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.5.4] - 2023-09-14
+### Fixed
+- Added localisation normalization so locations like `s3:/a/b` and `s3:/a/b/` will be considered the same and path won't be scheduled for deletion.
+
 ## [3.5.3] - 2023-03-08
 ### Changed
 - Upgrade `Springboot` from `2.4.4` to `2.7.9`.

beekeeper-api/src/main/java/com/expediagroup/beekeeper/api/controller/BeekeeperController.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2021 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

beekeeper-core/src/main/java/com/expediagroup/beekeeper/core/model/PeriodDuration.java

@@ -1,33 +1,17 @@
-/*
- * Copyright (c) 2007-present, Stephen Colebourne & Michael Nascimento Santos
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *  * Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
+/**
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
- *  * Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- *  * Neither the name of JSR-310 nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.expediagroup.beekeeper.core.model;
 

beekeeper-core/src/test/java/com/expediagroup/beekeeper/core/model/PeriodDurationTest.java

@@ -1,33 +1,17 @@
-/*
- * Copyright (c) 2007-present, Stephen Colebourne & Michael Nascimento Santos
- *
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions are met:
- *
- *  * Redistributions of source code must retain the above copyright notice,
- *    this list of conditions and the following disclaimer.
+/**
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
- *  * Redistributions in binary form must reproduce the above copyright notice,
- *    this list of conditions and the following disclaimer in the documentation
- *    and/or other materials provided with the distribution.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
  *
- *  * Neither the name of JSR-310 nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
+ * http://www.apache.org/licenses/LICENSE-2.0
  *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
- * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
  */
 package com.expediagroup.beekeeper.core.model;
 

beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizer.java

@@ -0,0 +1,32 @@
+/**
+ * Copyright (C) 2019-2023 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.expediagroup.beekeeper.scheduler.apiary.filter;
+
+import org.apache.commons.lang3.StringUtils;
+
+/**
+ * Attempts to normalize string representing a location of a Hive Table, could have a wide variety of schemes, s3, s3a,
+ * hdfs etc..
+ */
+public class LocationNormalizer {
+
+  public String normalize(String location) {
+    //Not using File.seperator here we might not know what the location would be using. (Beekeeper might run on Windows...)
+    location =StringUtils.stripEnd(location, "/");
+    return location;
+  }
+
+}

beekeeper-scheduler-apiary/src/main/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilter.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2020 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,7 +26,18 @@
 
 @Component
 public class LocationOnlyUpdateListenerEventFilter implements ListenerEventFilter {
+
+  private final LocationNormalizer locationNormalizer;
+
+  public LocationOnlyUpdateListenerEventFilter () {
+    this.locationNormalizer = new LocationNormalizer();
+  }
+
+  public LocationOnlyUpdateListenerEventFilter (LocationNormalizer locationNormaliser) {
+    this.locationNormalizer = locationNormaliser;
+  }
 
+
   @Override
   public boolean isFiltered(ListenerEvent listenerEvent, LifecycleEventType lifecycleEventType) {
     EventType eventType = listenerEvent.getEventType();
@@ -44,6 +55,11 @@ public boolean isFiltered(ListenerEvent listenerEvent, LifecycleEventType lifecy
   }
 
   private boolean isLocationSame(String oldLocation, String location) {
-    return location == null || oldLocation == null || oldLocation.equals(location);
+    if (location == null || oldLocation == null) {
+      return true;
+    }
+    String normalizedOldLocation = locationNormalizer.normalize(oldLocation);
+    String normalizedLocation = locationNormalizer.normalize(location);
+    return normalizedOldLocation.equals(normalizedLocation);
   }
 }

beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationNormalizerTest.java

@@ -0,0 +1,45 @@
+/**
+ * Copyright (C) 2019-2023 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.expediagroup.beekeeper.scheduler.apiary.filter;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import org.junit.jupiter.api.Test;
+
+class LocationNormalizerTest {
+
+  private LocationNormalizer normalizer = new LocationNormalizer();
+
+  @Test
+  void noChange() {
+    assertThat(normalizer.normalize("s3://bucket/prefix")).isEqualTo("s3://bucket/prefix");
+    assertThat(normalizer.normalize("/bucket/prefix")).isEqualTo("/bucket/prefix");
+    assertThat(normalizer.normalize("hdfs://bucket/prefix")).isEqualTo("hdfs://bucket/prefix");
+    assertThat(normalizer.normalize("")).isEqualTo("");
+    assertThat(normalizer.normalize(null)).isEqualTo(null);
+    assertThat(normalizer.normalize("foo")).isEqualTo("foo");
+    assertThat(normalizer.normalize("foo/bar")).isEqualTo("foo/bar");
+  }
+
+  @Test
+  void normalizeTrailingSlash() {
+    assertThat(normalizer.normalize("s3://bucket/prefix/")).isEqualTo("s3://bucket/prefix");
+    assertThat(normalizer.normalize("s3://bucket/prefix///")).isEqualTo("s3://bucket/prefix");
+    assertThat(normalizer.normalize("s3://bucket/prefix/../")).isEqualTo("s3://bucket/prefix/..");
+    assertThat(normalizer.normalize("hdfs://bucket/prefix/")).isEqualTo("hdfs://bucket/prefix");
+  }
+
+}

beekeeper-scheduler-apiary/src/test/java/com/expediagroup/beekeeper/scheduler/apiary/filter/LocationOnlyUpdateListenerEventFilterTest.java

@@ -1,5 +1,5 @@
 /**
- * Copyright (C) 2019-2020 Expedia, Inc.
+ * Copyright (C) 2019-2023 Expedia, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -113,6 +113,43 @@ public void alterPartitionEventMetadataOnlyNullOldLocation() {
     assertThat(filter).isTrue();
   }
 
+  @Test
+  public void alterTableEventMetadataLocationTrailingSlashOldShouldFilter() {
+    when(alterTableEvent.getEventType()).thenReturn(EventType.ALTER_TABLE);
+    when(alterTableEvent.getOldTableLocation()).thenReturn("/foo/bar/");
+    when(alterTableEvent.getTableLocation()).thenReturn("/foo/bar");
+    boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterTableEvent, UNREFERENCED);
+    assertThat(filter).isTrue();
+  }
+
+  @Test
+  public void alterPartitionEventMetadataLocationTrailingSlashOldShouldFilter() {
+    when(alterPartitionEvent.getEventType()).thenReturn(EventType.ALTER_PARTITION);
+    when(alterPartitionEvent.getOldPartitionLocation()).thenReturn("/foo/bar/");
+    when(alterPartitionEvent.getPartitionLocation()).thenReturn("/foo/bar");
+    boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterPartitionEvent, UNREFERENCED);
+    assertThat(filter).isTrue();
+  }
+
+
+  @Test
+  public void alterTableEventMetadataLocationTrailingSlashNewShouldFilter() {
+    when(alterTableEvent.getEventType()).thenReturn(EventType.ALTER_TABLE);
+    when(alterTableEvent.getOldTableLocation()).thenReturn("/foo/bar");
+    when(alterTableEvent.getTableLocation()).thenReturn("/foo/bar/");
+    boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterTableEvent, UNREFERENCED);
+    assertThat(filter).isTrue();
+  }
+
+  @Test
+  public void alterPartitionEventMetadataLocationTrailingSlashNewShouldFilter() {
+    when(alterPartitionEvent.getEventType()).thenReturn(EventType.ALTER_PARTITION);
+    when(alterPartitionEvent.getOldPartitionLocation()).thenReturn("/foo/bar");
+    when(alterPartitionEvent.getPartitionLocation()).thenReturn("/foo/bar/");
+    boolean filter = locationOnlyUpdateListenerEventFilter.isFiltered(alterPartitionEvent, UNREFERENCED);
+    assertThat(filter).isTrue();
+  }
+
   @Test
   public void dropTableEvent() {
     when(dropTableEvent.getEventType()).thenReturn(EventType.DROP_TABLE);