OAK-8413 Use the new Azure SDK in the Azure Segment Store

oak-commons/src/test/java/org/apache/jackrabbit/oak/commons/FixturesHelper.java

@@ -53,7 +53,7 @@ private FixturesHelper() { }
      * default fixtures when no {@code nsfixtures} system property is provided
      */
     public enum Fixture {
-       DOCUMENT_NS, @Deprecated SEGMENT_MK, DOCUMENT_RDB, MEMORY_NS, DOCUMENT_MEM, SEGMENT_TAR, SEGMENT_AWS, SEGMENT_AZURE, COMPOSITE_SEGMENT, COMPOSITE_MEM, COW_DOCUMENT
+       DOCUMENT_NS, @Deprecated SEGMENT_MK, DOCUMENT_RDB, MEMORY_NS, DOCUMENT_MEM, SEGMENT_TAR, SEGMENT_AWS, SEGMENT_AZURE_V8, SEGMENT_AZURE, COMPOSITE_SEGMENT, COMPOSITE_MEM, COW_DOCUMENT
     }
 
     private static final Set<Fixture> FIXTURES;

oak-it/src/test/java/org/apache/jackrabbit/oak/NodeStoreFixtures.java

@@ -33,6 +33,7 @@
 import org.apache.jackrabbit.oak.composite.CompositeSegmentStoreFixture;
 import org.apache.jackrabbit.oak.segment.aws.fixture.SegmentAwsFixture;
 import org.apache.jackrabbit.oak.segment.azure.fixture.SegmentAzureFixture;
+import org.apache.jackrabbit.oak.segment.azure.fixture.SegmentAzureFixtureV8;
 import org.apache.jackrabbit.oak.segment.fixture.SegmentTarFixture;
 
 public class NodeStoreFixtures {
@@ -43,6 +44,8 @@ public class NodeStoreFixtures {
 
     public static final NodeStoreFixture SEGMENT_AWS = new SegmentAwsFixture();
 
+    public static final NodeStoreFixture SEGMENT_AZURE_V8 = new SegmentAzureFixtureV8();
+
     public static final NodeStoreFixture SEGMENT_AZURE = new SegmentAzureFixture();
 
     public static final NodeStoreFixture DOCUMENT_NS = new DocumentMongoFixture();
@@ -80,6 +83,9 @@ public static Collection<Object[]> asJunitParameters(Set<FixturesHelper.Fixture>
         if (fixtures.contains(FixturesHelper.Fixture.SEGMENT_AZURE)) {
             configuredFixtures.add(SEGMENT_AZURE);
         }
+        if (fixtures.contains(FixturesHelper.Fixture.SEGMENT_AZURE_V8)) {
+            configuredFixtures.add(SEGMENT_AZURE_V8);
+        }
         if (fixtures.contains(FixturesHelper.Fixture.COMPOSITE_SEGMENT)) {
             configuredFixtures.add(COMPOSITE_SEGMENT);
         }

oak-it/src/test/java/org/apache/jackrabbit/oak/spi/state/NodeStoreTest.java

@@ -459,7 +459,7 @@ public void moveToDescendant() {
         if (fixture == NodeStoreFixtures.SEGMENT_TAR || fixture == NodeStoreFixtures.MEMORY_NS 
                 || fixture == NodeStoreFixtures.COMPOSITE_MEM || fixture == NodeStoreFixtures.COMPOSITE_SEGMENT
                 || fixture == NodeStoreFixtures.COW_DOCUMENT || fixture == NodeStoreFixtures.SEGMENT_AWS
-                || fixture == NodeStoreFixtures.SEGMENT_AZURE) {
+                || fixture == NodeStoreFixtures.SEGMENT_AZURE_V8 || fixture == NodeStoreFixtures.SEGMENT_AZURE) {
             assertTrue(x.moveTo(x, "xx"));
             assertFalse(x.exists());
             assertFalse(test.hasChildNode("x"));

oak-parent/pom.xml

@@ -693,6 +693,11 @@
         <artifactId>jackson-dataformat-smile</artifactId>
         <version>${jackson.version}</version>
       </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.dataformat</groupId>
+        <artifactId>jackson-dataformat-xml</artifactId>
+        <version>${jackson.version}</version>
+      </dependency>
       <dependency>
         <groupId>org.apache.httpcomponents</groupId>
         <artifactId>httpclient</artifactId>

oak-run-commons/src/main/java/org/apache/jackrabbit/oak/fixture/SegmentTarFixture.java

@@ -44,7 +44,7 @@
 import org.apache.jackrabbit.oak.segment.aws.AwsContext;
 import org.apache.jackrabbit.oak.segment.aws.AwsPersistence;
 import org.apache.jackrabbit.oak.segment.aws.Configuration;
-import org.apache.jackrabbit.oak.segment.azure.AzurePersistence;
+import org.apache.jackrabbit.oak.segment.azure.v8.AzurePersistenceV8;
 import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
 import org.apache.jackrabbit.oak.segment.file.FileStore;
 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
@@ -289,7 +289,7 @@ public Oak getOak(int clusterId) throws Exception {
             CloudBlobContainer container = cloud.createCloudBlobClient().getContainerReference(azureContainerName);
             container.createIfNotExists();
             CloudBlobDirectory directory = container.getDirectoryReference(azureRootPath);
-            fileStoreBuilder.withCustomPersistence(new AzurePersistence(directory));
+            fileStoreBuilder.withCustomPersistence(new AzurePersistenceV8(directory));
         }
 
         BlobStore blobStore = null;
@@ -341,7 +341,7 @@ public Oak[] setUpCluster(int n, StatisticsProvider statsProvider) throws Except
                 container.createIfNotExists();
                 containers[i] = container;
                 CloudBlobDirectory directory = container.getDirectoryReference(azureRootPath + "/primary-" + i);
-                builder.withCustomPersistence(new AzurePersistence(directory));
+                builder.withCustomPersistence(new AzurePersistenceV8(directory));
             }
 
             if (blobStore != null) {

oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/SegmentTarFixtureProvider.java

@@ -30,7 +30,7 @@
 import org.apache.jackrabbit.guava.common.io.Closer;
 import org.apache.commons.io.FileUtils;
 import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
-import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager;
+import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
 import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
 import org.apache.jackrabbit.oak.segment.file.FileStore;
 import org.apache.jackrabbit.oak.segment.file.FileStoreBuilder;
@@ -53,12 +53,12 @@ static NodeStore configureSegment(Options options, BlobStore blobStore, Whiteboa
 
         FileStoreBuilder builder;
         if (segmentStoreType == ToolUtils.SegmentStoreType.AZURE) {
-            final AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager();
+            final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8();
             SegmentNodeStorePersistence segmentNodeStorePersistence =
-                ToolUtils.newSegmentNodeStorePersistence(segmentStoreType, pathOrUri, azureStorageCredentialManager);
+                ToolUtils.newSegmentNodeStorePersistence(segmentStoreType, pathOrUri, azureStorageCredentialManagerV8);
             File tempDir = Files.createTempDirectory("azure-segment-store").toFile();
             closer.register(() -> FileUtils.deleteQuietly(tempDir));
-            closer.register(azureStorageCredentialManager);
+            closer.register(azureStorageCredentialManagerV8);
             builder = fileStoreBuilder(tempDir).withCustomPersistence(segmentNodeStorePersistence);
         } else {
             builder = fileStoreBuilder(new File(pathOrUri)).withMaxFileSize(256);

oak-run-elastic/pom.xml

@@ -42,7 +42,7 @@
         105 MB: Azure updates
         107 MB: RDB/Tomcat (OAK-10752)
         -->
-        <max.jar.size>112197632</max.jar.size>
+        <max.jar.size>113039632</max.jar.size>
 
     </properties>
 

oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/AzureSegmentStoreExplorerBackend.java

@@ -19,7 +19,7 @@
 package org.apache.jackrabbit.oak.explorer;
 
 import org.apache.jackrabbit.guava.common.io.Files;
-import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager;
+import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
 import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
 import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
 import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
@@ -39,16 +39,16 @@
 public class AzureSegmentStoreExplorerBackend extends AbstractSegmentTarExplorerBackend {
     private final String path;
     private SegmentNodeStorePersistence persistence;
-    private final AzureStorageCredentialManager azureStorageCredentialManager;
+    private final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8;
 
     public AzureSegmentStoreExplorerBackend(String path) {
         this.path = path;
-        this.azureStorageCredentialManager = new AzureStorageCredentialManager();
+        this.azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8();
     }
 
     @Override
     public void open() throws IOException {
-        this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, azureStorageCredentialManager);
+        this.persistence = newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, path, azureStorageCredentialManagerV8);
 
         try {
             this.store = fileStoreBuilder(Files.createTempDir())
@@ -63,7 +63,7 @@ public void open() throws IOException {
     @Override
     public void close() {
         super.close();
-        azureStorageCredentialManager.close();
+        azureStorageCredentialManagerV8.close();
     }
 
     @Override

oak-run/src/main/java/org/apache/jackrabbit/oak/run/FileStoreDiffCommand.java

@@ -28,7 +28,7 @@
 import joptsimple.OptionSet;
 import joptsimple.OptionSpec;
 import org.apache.jackrabbit.oak.run.commons.Command;
-import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager;
+import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
 import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
 import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
 import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
@@ -87,8 +87,8 @@ public void execute(String... args) throws Exception {
             }
         } else {
             if (pathOrURI.startsWith("az:")) {
-                try (AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager()) {
-                    SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI, azureStorageCredentialManager);
+                try (AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8()) {
+                    SegmentNodeStorePersistence azurePersistence = ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, pathOrURI, azureStorageCredentialManagerV8);
                     ReadOnlyFileStore store = fileStoreBuilder(Files.createTempDir()).withCustomPersistence(azurePersistence).withBlobStore(newBasicReadOnlyBlobStore()).buildReadOnly();
                     statusCode = Diff.builder()
                             .withPath(pathOrURI)

oak-run/src/test/java/org/apache/jackrabbit/oak/run/DataStoreCommandTest.java

@@ -76,8 +76,8 @@
 import org.apache.jackrabbit.oak.run.cli.BlobStoreOptions.Type;
 import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
 import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
-import org.apache.jackrabbit.oak.segment.azure.AzureStorageCredentialManager;
-import org.apache.jackrabbit.oak.segment.azure.AzureUtilities;
+import org.apache.jackrabbit.oak.segment.azure.v8.AzureStorageCredentialManagerV8;
+import org.apache.jackrabbit.oak.segment.azure.v8.AzureUtilitiesV8;
 import org.apache.jackrabbit.oak.segment.azure.tool.ToolUtils;
 import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
 import org.apache.jackrabbit.oak.segment.file.FileStore;
@@ -1133,7 +1133,7 @@ class SegmentStoreFixture implements StoreFixture {
         class AzureSegmentStoreFixture extends SegmentStoreFixture {
             private static final String AZURE_DIR = "repository";
             private String container;
-            private final AzureStorageCredentialManager azureStorageCredentialManager = new AzureStorageCredentialManager();
+            private final AzureStorageCredentialManagerV8 azureStorageCredentialManagerV8 = new AzureStorageCredentialManagerV8();
 
             @Override public NodeStore init(DataStoreBlobStore blobStore, File storeFile) throws Exception {
                 Properties props = AzureDataStoreUtils.getAzureConfig();
@@ -1143,14 +1143,14 @@ class AzureSegmentStoreFixture extends SegmentStoreFixture {
                 container = container + System.currentTimeMillis();
                 // Create the azure segment container
                 String connectionString = getAzureConnectionString(accessKey, secretKey, container, AZURE_DIR);
-                AzureUtilities.cloudBlobDirectoryFrom(connectionString, container, AZURE_DIR);
+                AzureUtilitiesV8.cloudBlobDirectoryFrom(connectionString, container, AZURE_DIR);
 
                 // get the azure uri expected by the command
                 storePath = getAzureUri(accessKey, container, AZURE_DIR);
 
                 // initialize azure segment for test setup
                 SegmentNodeStorePersistence segmentNodeStorePersistence =
-                    ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, storePath, azureStorageCredentialManager);
+                    ToolUtils.newSegmentNodeStorePersistence(ToolUtils.SegmentStoreType.AZURE, storePath, azureStorageCredentialManagerV8);
                 fileStore = fileStoreBuilder(storeFile).withBlobStore(blobStore)
                     .withCustomPersistence(segmentNodeStorePersistence).build();
 
@@ -1182,7 +1182,7 @@ protected String getAzureConnectionString(String accountName, String secret, Str
             public void after() {
                 try {
                     AzureDataStoreUtils.deleteContainer(container);
-                    azureStorageCredentialManager.close();
+                    azureStorageCredentialManagerV8.close();
                 } catch(Exception e) {
                     log.error("Error in cleaning the container {}", container, e);
                 }

oak-segment-azure/pom.xml

@@ -48,10 +48,12 @@
                             org.apache.jackrabbit.oak.segment.remote*,
                             com.fasterxml.jackson.annotation;resolution:=optional,
                             com.fasterxml.jackson.databind*;resolution:=optional,
-                            com.fasterxml.jackson.dataformat.xml;resolution:=optional,
+                            com.fasterxml.jackson.dataformat.*;resolution:=optional,
                             com.fasterxml.jackson.datatype*;resolution:=optional,
                             com.azure.identity.broker.implementation;resolution:=optional,
-                            com.azure.xml;resolution:=optional,
+                            com.azure.storage.blob*;resolution:=optional,
+                            com.azure.storage.common*;resolution:=optional,
+                            com.azure.storage.internal*;resolution:=optional,
                             com.microsoft.aad.msal4jextensions*;resolution:=optional,
                             com.nimbusds.langtag;resolution:=optional,
                             com.sun.jna*;resolution:=optional,
@@ -71,18 +73,28 @@
                             org.apache.jackrabbit.oak.segment.azure,
                             org.apache.jackrabbit.oak.segment.azure.queue,
                             org.apache.jackrabbit.oak.segment.azure.util,
+                            com.fasterxml.jackson.dataformat.xml,
+                            com.fasterxml.jackson.dataformat.xml.deser,
                             com.microsoft.azure.storage,
                             com.microsoft.azure.storage.core,
                             com.microsoft.azure.storage.blob,
                             com.azure.core.credential,
-                            com.azure.identity
+                            com.azure.identity,
+                            com.azure.blob,
+                            com.azure.storage.common,
+                            com.azure.storage.internal.avro.implementation
                         </Export-Package>
                         <Embed-Dependency>
                             azure-storage,
                             azure-keyvault-core,
                             azure-core,
                             azure-identity,
                             azure-json,
+                            azure-xml,
+                            azure-storage-blob,
+                            azure-storage-common,
+                            azure-storage-internal-avro,
+                            jackson-dataformat-xml,
                             guava,
                             jsr305,
                             reactive-streams,
@@ -188,6 +200,28 @@
             <artifactId>azure-keyvault-core</artifactId>
         </dependency>
 
+        <!-- Azure Blob Storage v12 dependencies -->
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-storage-blob</artifactId>
+            <version>12.25.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-storage-common</artifactId>
+            <version>12.24.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-storage-internal-avro</artifactId>
+            <version>12.10.3</version>
+        </dependency>
+        <dependency>
+            <groupId>com.azure</groupId>
+            <artifactId>azure-xml</artifactId>
+            <version>1.0.0</version>
+        </dependency>
+
         <!-- Azure Identity Client for Microsoft Entra ID dependency -->
         <dependency>
             <groupId>com.azure</groupId>
@@ -365,6 +399,12 @@
             <artifactId>junit</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>io.projectreactor</groupId>
+            <artifactId>reactor-test</artifactId>
+            <version>3.6.10</version>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>testcontainers</artifactId>