From f0a5039a7537a0a52322938ae092b12acbc5a164 Mon Sep 17 00:00:00 2001 From: t-rana Date: Wed, 11 Dec 2024 11:43:04 +0530 Subject: [PATCH 1/3] OAK-11266: add remote storage mode --- .../oak/blob/cloud/s3/RemoteStorageMode.java | 6 ++++++ .../jackrabbit/oak/blob/cloud/s3/S3Backend.java | 11 +++++++++++ .../jackrabbit/oak/blob/cloud/s3/S3Constants.java | 5 +++++ .../jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java | 2 +- 4 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/RemoteStorageMode.java diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/RemoteStorageMode.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/RemoteStorageMode.java new file mode 100644 index 00000000000..6bf6ac2cd0c --- /dev/null +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/RemoteStorageMode.java @@ -0,0 +1,6 @@ +package org.apache.jackrabbit.oak.blob.cloud.s3; + +public enum RemoteStorageMode { + S3, + GCP +} diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java index 484f6d5885f..0ad8fd71866 100644 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java @@ -490,6 +490,17 @@ public void setBucket(String bucket) { */ public void setProperties(Properties properties) { this.properties = properties; + setRemoteStorageMode(); + } + + private void setRemoteStorageMode() { + String s3EndPoint = properties.getProperty(S3Constants.S3_END_POINT, ""); + if (s3EndPoint.contains("https://storage.googleapis.com")) { + properties.putIfAbsent(S3Constants.MODE, RemoteStorageMode.GCP); + return; + } + // default mode is S3 + properties.putIfAbsent(S3Constants.MODE, RemoteStorageMode.S3); } @Override diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java index 3852e35b2fc..e77b4bd7f24 100644 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Constants.java @@ -144,6 +144,11 @@ public final class S3Constants { */ public static final String PROXY_PORT = "proxyPort"; + /** + * Constant to set remote storage mode + */ + public static final String MODE = "mode"; + public static final String PRESIGNED_HTTP_UPLOAD_URI_EXPIRY_SECONDS = "presignedHttpUploadURIExpirySeconds"; public static final String PRESIGNED_HTTP_DOWNLOAD_URI_EXPIRY_SECONDS = "presignedHttpDownloadURIExpirySeconds"; diff --git a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java index 122945aa59a..5761049b54a 100644 --- a/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java +++ b/oak-blob-cloud/src/test/java/org/apache/jackrabbit/oak/blob/cloud/s3/TestS3DataStore.java @@ -104,7 +104,7 @@ public void setUp() throws Exception { @Test public void testAccessParamLeakOnError() throws Exception { expectedEx.expect(RepositoryException.class); - expectedEx.expectMessage("Could not initialize S3 from {s3Region=us-standard, intValueKey=25}"); + expectedEx.expectMessage("Could not initialize S3 from {mode=S3, s3Region=us-standard, intValueKey=25}"); props.put(S3Constants.ACCESS_KEY, "abcd"); props.put(S3Constants.SECRET_KEY, "123456"); From 426f5fb5f32f7c98ab03d42ba523bf6e3c5909f5 Mon Sep 17 00:00:00 2001 From: t-rana Date: Wed, 11 Dec 2024 12:04:32 +0530 Subject: [PATCH 2/3] OAK-11266: review changes --- .../oak/blob/cloud/s3/RemoteStorageMode.java | 6 ------ .../jackrabbit/oak/blob/cloud/s3/S3Backend.java | 14 +++++++++++--- 2 files changed, 11 insertions(+), 9 deletions(-) delete mode 100644 oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/RemoteStorageMode.java diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/RemoteStorageMode.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/RemoteStorageMode.java deleted file mode 100644 index 6bf6ac2cd0c..00000000000 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/RemoteStorageMode.java +++ /dev/null @@ -1,6 +0,0 @@ -package org.apache.jackrabbit.oak.blob.cloud.s3; - -public enum RemoteStorageMode { - S3, - GCP -} diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java index 0ad8fd71866..d9c54332f89 100644 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java @@ -495,12 +495,12 @@ public void setProperties(Properties properties) { private void setRemoteStorageMode() { String s3EndPoint = properties.getProperty(S3Constants.S3_END_POINT, ""); - if (s3EndPoint.contains("https://storage.googleapis.com")) { - properties.putIfAbsent(S3Constants.MODE, RemoteStorageMode.GCP); + if (s3EndPoint.contains("googleapis")) { + properties.put(S3Constants.MODE, RemoteStorageMode.GCP); return; } // default mode is S3 - properties.putIfAbsent(S3Constants.MODE, RemoteStorageMode.S3); + properties.put(S3Constants.MODE, RemoteStorageMode.S3); } @Override @@ -1361,4 +1361,12 @@ public KeyRenameThread(String oldKey) { this.oldKey = oldKey; } } + + /** + * Enum to indicate remote storage mode + */ + private enum RemoteStorageMode { + S3, + GCP + } } From 2cfc5495c4e854eb1f35b23832fd1af0f563b7c9 Mon Sep 17 00:00:00 2001 From: t-rana Date: Wed, 11 Dec 2024 12:09:58 +0530 Subject: [PATCH 3/3] OAK-11266: add warn log --- .../org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java index d9c54332f89..0ab186d4bcf 100644 --- a/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java +++ b/oak-blob-cloud/src/main/java/org/apache/jackrabbit/oak/blob/cloud/s3/S3Backend.java @@ -496,6 +496,9 @@ public void setProperties(Properties properties) { private void setRemoteStorageMode() { String s3EndPoint = properties.getProperty(S3Constants.S3_END_POINT, ""); if (s3EndPoint.contains("googleapis")) { + if (properties.get(S3Constants.MODE) == RemoteStorageMode.S3) { + LOG.warn("Mismatch between remote storage mode and s3EndPoint, overriding mode to GCP"); + } properties.put(S3Constants.MODE, RemoteStorageMode.GCP); return; }