From a5533efe986cd96a6f634a3c58983823e1065928 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Wed, 3 Jul 2024 16:29:09 -0700 Subject: [PATCH 1/6] Add UseRBAC to Azure Blob Storage Extension --- CosmosDbDataMigrationTool.sln | 3 +++ .../AzureBlobDataSink.cs | 27 +++++++++++++++---- .../AzureBlobDataSource.cs | 27 +++++++++++++++---- .../AzureBlobSinkSettings.cs | 24 ++++++++++++++--- .../AzureBlobSourceSettings.cs | 24 ++++++++++++++--- ...osmos.DataTransfer.AzureBlobStorage.csproj | 1 + ...Cosmos.DataTransfer.CosmosExtension.csproj | 2 +- 7 files changed, 89 insertions(+), 19 deletions(-) diff --git a/CosmosDbDataMigrationTool.sln b/CosmosDbDataMigrationTool.sln index 10126ef..66d32cc 100644 --- a/CosmosDbDataMigrationTool.sln +++ b/CosmosDbDataMigrationTool.sln @@ -64,6 +64,9 @@ EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Cosmos.DataTransfer.Common", "Interfaces\Cosmos.DataTransfer.Common\Cosmos.DataTransfer.Common.csproj", "{0FAD9D89-2E41-4D65-8440-5C01885D9292}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AzureBlob", "AzureBlob", "{9627A42A-BEB0-4A39-B49C-C3C6D54E705A}" + ProjectSection(SolutionItems) = preProject + Extensions\AzureBlob\README.md = Extensions\AzureBlob\README.md + EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AwsS3", "AwsS3", "{502197E4-F554-4B5B-9235-FBFE7E49EBEF}" EndProject diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs index 752ef42..4c59dab 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs @@ -1,9 +1,10 @@ -using Azure.Storage.Blobs; +using Azure.Identity; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using Cosmos.DataTransfer.Interfaces; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Azure.Storage.Blobs.Specialized; -using Azure.Storage.Blobs.Models; namespace Cosmos.DataTransfer.AzureBlobStorage { @@ -14,12 +15,28 @@ public async Task WriteToTargetAsync(Func writeToStream, IConfigur var settings = config.Get(); settings.Validate(); - logger.LogInformation("Saving file '{File}' to Azure Blob Container '{ContainerName}'", settings.BlobName, settings.ContainerName); + BlobContainerClient account; + if (settings.UseRbacAuth) + { + logger.LogInformation("Connecting to Storage account {AccountName} using {UseRbacAuth}'", settings.AccountName, nameof(AzureBlobSourceSettings.UseRbacAuth)); + + var credential = new DefaultAzureCredential(); + var blobContainerUri = new Uri($"https://{settings.AccountName}.queue.core.windows.net"); + + account = new BlobContainerClient(blobContainerUri, credential); + } + else + { + logger.LogInformation("Connecting to Storage account using {ConnectionString}'", nameof(AzureBlobSourceSettings.ConnectionString)); + + account = new BlobContainerClient(settings.ConnectionString, settings.ContainerName); + } - var account = new BlobContainerClient(settings.ConnectionString, settings.ContainerName); await account.CreateIfNotExistsAsync(cancellationToken: cancellationToken); var blob = account.GetBlockBlobClient(settings.BlobName); + logger.LogInformation("Saving file '{File}' to Azure Blob Container '{ContainerName}'", settings.BlobName, settings.ContainerName); + await using var blobStream = await blob.OpenWriteAsync(true, new BlockBlobOpenWriteOptions { BufferSize = settings.MaxBlockSizeinKB * 1024L, diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs index 896a62f..373e985 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs @@ -1,10 +1,11 @@ -using System.Runtime.CompilerServices; +using Azure.Identity; using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Blobs.Specialized; using Cosmos.DataTransfer.Interfaces; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; -using Azure.Storage.Blobs.Specialized; -using Azure.Storage.Blobs.Models; +using System.Runtime.CompilerServices; namespace Cosmos.DataTransfer.AzureBlobStorage; @@ -15,14 +16,30 @@ public class AzureBlobDataSource : IComposableDataSource var settings = config.Get(); settings.Validate(); - logger.LogInformation("Reading file '{File}' from Azure Blob Container '{ContainerName}'", settings.BlobName, settings.ContainerName); + BlobContainerClient account; + if (settings.UseRbacAuth) + { + logger.LogInformation("Connecting to Storage account {AccountName} using {UseRbacAuth}'", settings.AccountName, nameof(AzureBlobSourceSettings.UseRbacAuth)); - var account = new BlobContainerClient(settings.ConnectionString, settings.ContainerName); + var credential = new DefaultAzureCredential(); + var blobContainerUri = new Uri($"https://{settings.AccountName}.queue.core.windows.net"); + + account = new BlobContainerClient(blobContainerUri, credential); + } + else + { + logger.LogInformation("Connecting to Storage account using {ConnectionString}'", nameof(AzureBlobSourceSettings.ConnectionString)); + + account = new BlobContainerClient(settings.ConnectionString, settings.ContainerName); + } + var blob = account.GetBlockBlobClient(settings.BlobName); var existsResponse = await blob.ExistsAsync(cancellationToken: cancellationToken); if (!existsResponse) yield break; + logger.LogInformation("Reading file '{File}' from Azure Blob Container '{ContainerName}'", settings.BlobName, settings.ContainerName); + var readStream = await blob.OpenReadAsync(new BlobOpenReadOptions(false) { BufferSize = settings.ReadBufferSizeInKB, diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs index 5d3a5c8..bab59fa 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs @@ -1,14 +1,15 @@ using Cosmos.DataTransfer.Interfaces; -using System.ComponentModel.DataAnnotations; using Cosmos.DataTransfer.Interfaces.Manifest; +using System.ComponentModel.DataAnnotations; namespace Cosmos.DataTransfer.AzureBlobStorage { - public class AzureBlobSinkSettings : IDataExtensionSettings + public class AzureBlobSinkSettings : IDataExtensionSettings, IValidatableObject { - [Required] [SensitiveValue] - public string ConnectionString { get; set; } = null!; + public string? ConnectionString { get; set; } = null!; + + public string? AccountName { get; set; } = null!; [Required] public string ContainerName { get; set; } = null!; @@ -17,5 +18,20 @@ public class AzureBlobSinkSettings : IDataExtensionSettings public string BlobName { get; set; } = null!; public int? MaxBlockSizeinKB { get; set; } + + public bool UseRbacAuth { get; set; } + + public virtual IEnumerable Validate(ValidationContext validationContext) + { + if (!UseRbacAuth && string.IsNullOrEmpty(ConnectionString)) + { + yield return new ValidationResult($"{nameof(ConnectionString)} must be specified unless {nameof(UseRbacAuth)} is true", new[] { nameof(ConnectionString) }); + } + + if (UseRbacAuth && string.IsNullOrEmpty(AccountName)) + { + yield return new ValidationResult($"{nameof(AccountName)} must be specified unless {nameof(UseRbacAuth)} is false", new[] { nameof(AccountName) }); + } + } } } \ No newline at end of file diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs index af10068..08b3e79 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs @@ -1,14 +1,15 @@ using Cosmos.DataTransfer.Interfaces; -using System.ComponentModel.DataAnnotations; using Cosmos.DataTransfer.Interfaces.Manifest; +using System.ComponentModel.DataAnnotations; namespace Cosmos.DataTransfer.AzureBlobStorage; -public class AzureBlobSourceSettings : IDataExtensionSettings +public class AzureBlobSourceSettings : IDataExtensionSettings, IValidatableObject { - [Required] [SensitiveValue] - public string ConnectionString { get; set; } = null!; + public string? ConnectionString { get; set; } = null!; + + public string? AccountName { get; set; } = null!; [Required] public string ContainerName { get; set; } = null!; @@ -17,4 +18,19 @@ public class AzureBlobSourceSettings : IDataExtensionSettings public string BlobName { get; set; } = null!; public int? ReadBufferSizeInKB { get; set; } + + public bool UseRbacAuth { get; set; } + + public virtual IEnumerable Validate(ValidationContext validationContext) + { + if (!UseRbacAuth && string.IsNullOrEmpty(ConnectionString)) + { + yield return new ValidationResult($"{nameof(ConnectionString)} must be specified unless {nameof(UseRbacAuth)} is true", new[] { nameof(ConnectionString) }); + } + + if (UseRbacAuth && string.IsNullOrEmpty(AccountName)) + { + yield return new ValidationResult($"{nameof(AccountName)} must be specified unless {nameof(UseRbacAuth)} is false", new[] { nameof(AccountName) }); + } + } } \ No newline at end of file diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/Cosmos.DataTransfer.AzureBlobStorage.csproj b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/Cosmos.DataTransfer.AzureBlobStorage.csproj index c0fe756..e51831c 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/Cosmos.DataTransfer.AzureBlobStorage.csproj +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/Cosmos.DataTransfer.AzureBlobStorage.csproj @@ -7,6 +7,7 @@ + diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/Cosmos.DataTransfer.CosmosExtension.csproj b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/Cosmos.DataTransfer.CosmosExtension.csproj index 277532c..cb98e24 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/Cosmos.DataTransfer.CosmosExtension.csproj +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/Cosmos.DataTransfer.CosmosExtension.csproj @@ -8,7 +8,7 @@ - + From ff66694285635c70ad6a0d8bbd058b9920b8469e Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Sun, 7 Jul 2024 19:22:38 -0700 Subject: [PATCH 2/6] Add RBAC to Azure Blob --- .../Cosmos.DataTransfer.Core.csproj | 3 ++- .../AzureBlobDataSink.cs | 9 ++++++--- .../AzureBlobDataSource.cs | 4 ++-- .../AzureBlobSinkSettings.cs | 8 +++++--- .../AzureBlobSourceSettings.cs | 2 ++ 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Core/Cosmos.DataTransfer.Core/Cosmos.DataTransfer.Core.csproj b/Core/Cosmos.DataTransfer.Core/Cosmos.DataTransfer.Core.csproj index 67838d7..eb5dd4b 100644 --- a/Core/Cosmos.DataTransfer.Core/Cosmos.DataTransfer.Core.csproj +++ b/Core/Cosmos.DataTransfer.Core/Cosmos.DataTransfer.Core.csproj @@ -19,7 +19,8 @@ - + + diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs index 4c59dab..b162c7f 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs @@ -18,10 +18,13 @@ public async Task WriteToTargetAsync(Func writeToStream, IConfigur BlobContainerClient account; if (settings.UseRbacAuth) { - logger.LogInformation("Connecting to Storage account {AccountName} using {UseRbacAuth}'", settings.AccountName, nameof(AzureBlobSourceSettings.UseRbacAuth)); + logger.LogInformation("Connecting to Storage account {AccountName} using {UseRbacAuth}'", settings.AccountEndpoint, nameof(AzureBlobSourceSettings.UseRbacAuth)); - var credential = new DefaultAzureCredential(); - var blobContainerUri = new Uri($"https://{settings.AccountName}.queue.core.windows.net"); + var credential = new DefaultAzureCredential(includeInteractiveCredentials: settings.EnableInteractiveCredentials); +#pragma warning disable CS8604 // Validate above ensures AccountEndpoint is not null + var baseUri = new Uri(settings.AccountEndpoint); + var blobContainerUri = new Uri(baseUri, settings.ContainerName); +#pragma warning restore CS8604 // Restore warning account = new BlobContainerClient(blobContainerUri, credential); } diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs index 373e985..5db0f68 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs @@ -19,9 +19,9 @@ public class AzureBlobDataSource : IComposableDataSource BlobContainerClient account; if (settings.UseRbacAuth) { - logger.LogInformation("Connecting to Storage account {AccountName} using {UseRbacAuth}'", settings.AccountName, nameof(AzureBlobSourceSettings.UseRbacAuth)); + logger.LogInformation("Connecting to Storage account {AccountName} using {UseRbacAuth} with {EnableInteractiveCredentials}'", settings.AccountName, nameof(AzureBlobSourceSettings.UseRbacAuth), nameof(AzureBlobSourceSettings.EnableInteractiveCredentials)); - var credential = new DefaultAzureCredential(); + var credential = new DefaultAzureCredential(includeInteractiveCredentials: settings.EnableInteractiveCredentials); var blobContainerUri = new Uri($"https://{settings.AccountName}.queue.core.windows.net"); account = new BlobContainerClient(blobContainerUri, credential); diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs index bab59fa..dc673c0 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs @@ -9,7 +9,7 @@ public class AzureBlobSinkSettings : IDataExtensionSettings, IValidatableObject [SensitiveValue] public string? ConnectionString { get; set; } = null!; - public string? AccountName { get; set; } = null!; + public string? AccountEndpoint { get; set; } = null!; [Required] public string ContainerName { get; set; } = null!; @@ -21,6 +21,8 @@ public class AzureBlobSinkSettings : IDataExtensionSettings, IValidatableObject public bool UseRbacAuth { get; set; } + public bool EnableInteractiveCredentials { get; set; } = true; + public virtual IEnumerable Validate(ValidationContext validationContext) { if (!UseRbacAuth && string.IsNullOrEmpty(ConnectionString)) @@ -28,9 +30,9 @@ public virtual IEnumerable Validate(ValidationContext validati yield return new ValidationResult($"{nameof(ConnectionString)} must be specified unless {nameof(UseRbacAuth)} is true", new[] { nameof(ConnectionString) }); } - if (UseRbacAuth && string.IsNullOrEmpty(AccountName)) + if (UseRbacAuth && string.IsNullOrEmpty(AccountEndpoint)) { - yield return new ValidationResult($"{nameof(AccountName)} must be specified unless {nameof(UseRbacAuth)} is false", new[] { nameof(AccountName) }); + yield return new ValidationResult($"{nameof(AccountEndpoint)} must be specified unless {nameof(UseRbacAuth)} is false", new[] { nameof(AccountEndpoint) }); } } } diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs index 08b3e79..c4070a9 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs @@ -21,6 +21,8 @@ public class AzureBlobSourceSettings : IDataExtensionSettings, IValidatableObjec public bool UseRbacAuth { get; set; } + public bool EnableInteractiveCredentials { get; set; } = true; + public virtual IEnumerable Validate(ValidationContext validationContext) { if (!UseRbacAuth && string.IsNullOrEmpty(ConnectionString)) From d6e896fbd0628d21ebd68525f0ea098b543d22bb Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 8 Jul 2024 11:38:42 -0700 Subject: [PATCH 3/6] Add an example config for Json-AzureBlob using RBAC --- ExampleConfigs.md | 193 +++++++++++++++++++-------------- Extensions/AzureBlob/README.md | 17 ++- Extensions/Cosmos/README.md | 6 +- 3 files changed, 133 insertions(+), 83 deletions(-) diff --git a/ExampleConfigs.md b/ExampleConfigs.md index ee32932..d80c830 100644 --- a/ExampleConfigs.md +++ b/ExampleConfigs.md @@ -1,123 +1,156 @@ # Example `migrationsettings.json` Files ## JSON to Cosmos-NoSQL + ```json { "Source": "json", "Sink": "cosmos-nosql", - "SourceSettings": { - "FilePath": "https://mytestfiles.local/sales-data.json" - }, - "SinkSettings": { - "ConnectionString": "AccountEndpoint=https://...", - "Database": "myDb", - "Container": "myContainer", - "PartitionKeyPath": "/id", - "RecreateContainer": true, - "WriteMode": "Insert", - "CreatedContainerMaxThroughput": 5000, - "IsServerlessAccount": false - } + "SourceSettings": { + "FilePath": "https://mytestfiles.local/sales-data.json" + }, + "SinkSettings": { + "ConnectionString": "AccountEndpoint=https://...", + "Database": "myDb", + "Container": "myContainer", + "PartitionKeyPath": "/id", + "RecreateContainer": true, + "WriteMode": "Insert", + "CreatedContainerMaxThroughput": 5000, + "IsServerlessAccount": false + } } ``` ## Cosmos-NoSQL to JSON + ```json { "Source": "Cosmos-NoSql", "Sink": "JSON", - "SourceSettings": - { - "ConnectionString": "AccountEndpoint=https://...", - "Database":"cosmicworks", - "Container":"customers", - "IncludeMetadataFields": true - }, - "SinkSettings": - { - "FilePath": "c:\\data\\cosmicworks\\customers.json", - "Indented": true - } + "SourceSettings": + { + "ConnectionString": "AccountEndpoint=https://...", + "Database":"cosmicworks", + "Container":"customers", + "IncludeMetadataFields": true + }, + "SinkSettings": + { + "FilePath": "c:\\data\\cosmicworks\\customers.json", + "Indented": true + } } ``` ## MongoDB to Cosmos-NoSQL + ```json { "Source": "mongodb", "Sink": "cosmos-nosql", - "SourceSettings": { - "ConnectionString": "mongodb://...", - "DatabaseName": "sales", - "Collection": "person" - }, - "SinkSettings": { - "ConnectionString": "AccountEndpoint=https://...", - "Database": "users", - "Container": "migrated", - "PartitionKeyPath": "/id", + "SourceSettings": { + "ConnectionString": "mongodb://...", + "DatabaseName": "sales", + "Collection": "person" + }, + "SinkSettings": { + "ConnectionString": "AccountEndpoint=https://...", + "Database": "users", + "Container": "migrated", + "PartitionKeyPath": "/id", "ConnectionMode": "Direct", - "WriteMode": "UpsertStream", - "CreatedContainerMaxThroughput": 8000, - "UseAutoscaleForCreatedContainer": false - } + "WriteMode": "UpsertStream", + "CreatedContainerMaxThroughput": 8000, + "UseAutoscaleForCreatedContainer": false + } } ``` ## SqlServer to AzureTableAPI + ```json { "Source": "SqlServer", "Sink": "AzureTableApi", - "SourceSettings": { - "ConnectionString": "Server=...", - "QueryText": "SELECT Id, Date, Amount FROM dbo.Payments WHERE Status = 'open'" - }, - "SinkSettings": { - "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=...", - "Table": "payments", - "RowKeyFieldName": "Id" - } + "SourceSettings": { + "ConnectionString": "Server=...", + "QueryText": "SELECT Id, Date, Amount FROM dbo.Payments WHERE Status = 'open'" + }, + "SinkSettings": { + "ConnectionString": "DefaultEndpointsProtocol=https;AccountName=...", + "Table": "payments", + "RowKeyFieldName": "Id" + } } ``` ## Cosmos-NoSQL to SqlServer + ```json { "Source": "cosmos-nosql", "Sink": "sqlserver", - "SourceSettings": - { - "ConnectionString": "AccountEndpoint=https://...", - "Database":"operations", - "Container":"alerts", - "PartitionKeyValue": "jan", + "SourceSettings": + { + "ConnectionString": "AccountEndpoint=https://...", + "Database":"operations", + "Container":"alerts", + "PartitionKeyValue": "jan", "Query": "SELECT a.name, a.description, a.count, a.id, a.isSet FROM a" - }, - "SinkSettings": - { - "ConnectionString": "Server=...", - "TableName": "Import", - "ColumnMappings": [ - { - "ColumnName": "Name" - }, - { - "ColumnName": "Description" - }, - { - "ColumnName": "Count", - "SourceFieldName": "number" - }, - { - "ColumnName": "Id" - }, - { - "ColumnName": "IsSet", - "AllowNull": false, - "DefaultValue": false - } - ] - } + }, + "SinkSettings": + { + "ConnectionString": "Server=...", + "TableName": "Import", + "ColumnMappings": [ + { + "ColumnName": "Name" + }, + { + "ColumnName": "Description" + }, + { + "ColumnName": "Count", + "SourceFieldName": "number" + }, + { + "ColumnName": "Id" + }, + { + "ColumnName": "IsSet", + "AllowNull": false, + "DefaultValue": false + } + ] + } +} +``` + +## Cosmos-NoSQL to Json-AzureBlob (Using RBAC) + +```json +{ + "Source": "Cosmos-nosql", + "Sink": "Json-AzureBlob", + "SourceSettings": { + "UseRbacAuth": true, + "Database": "operations", + "Container": "alerts", + "PartitionKeyValue": "jan", + "AccountEndpoint": "https://.documents.azure.com", + "EnableInteractiveCredentials": true, + "IncludeMetadataFields": false, + "Query": "SELECT a.name, a.description, a.count, a.id, a.isSet FROM a" + }, + "SinkSettings": { + "UseRbacAuth": true, + "ContainerName": "operations-archive", + "AccountEndpoint": "https://.blob.core.windows.net", + "EnableInteractiveCredentials": true, + "BlobName": "jan-alerts" + }, + "Operations": [ + ] } ``` diff --git a/Extensions/AzureBlob/README.md b/Extensions/AzureBlob/README.md index c7046a1..c7e60f6 100644 --- a/Extensions/AzureBlob/README.md +++ b/Extensions/AzureBlob/README.md @@ -6,7 +6,10 @@ The Azure Blob Storage extension provides reading and writing of formatted files ## Settings -Source and Sink settings require the parameters shown below. +Source and sink require settings used to locate and access the Azure Blob Storage account. This can be done in one of two ways: + +- Using a `ConnectionString` that includes an AccountEndpoint and AccountKey +- Using RBAC (Role Based Access Control) by setting `UseRbacAuth` to true and specifying `AccountEndpoint` and optionally `EnableInteractiveCredentials` to prompt the user to log in to Azure if default credentials are not available. ### Source @@ -20,6 +23,18 @@ An optional `ReadBufferSizeInKB` parameter can be used to control stream bufferi } ``` +Or with RBAC: + +```json +{ + "AccountEndpoint": "https://.blob.core.windows.net", + "ContainerName": "", + "BlobName": "", + "UseRbacAuth": true, + "EnableInteractiveCredentials": true +} +``` + ### Sink An optional `MaxBlockSizeInKB` parameter can also be specified to control the transfer. diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index b3b9de2..0e26dcd 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -7,10 +7,12 @@ The Cosmos data transfer extension provides source and sink capabilities for rea ## Settings Source and sink require settings used to locate and access the Cosmos DB account. This can be done in one of two ways: + - Using a `ConnectionString` that includes an AccountEndpoint and AccountKey -- Using RBAC (Role Based Access Control) by setting `UseRbacAuth` to true and specifying `AccountEndpoint` and optionally `EnableInteractiveCredentials` to prompt the user to log in to Azure if default credentials are not available. +- Using RBAC (Role Based Access Control) by setting `UseRbacAuth` to true and specifying `AccountEndpoint` and optionally `EnableInteractiveCredentials` to prompt the user to log in to Azure if default credentials are not available. See ([migrate-passwordless](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/migrate-passwordless?tabs=sign-in-azure-cli%2Cdotnet%2Cazure-portal-create%2Cazure-portal-associate%2Capp-service-identity) for how to configure Cosmos DB for passwordless access. + +Source and sink settings also both require parameters to specify the data location within a Cosmos DB account: -Source and sink settings also both require parameters to specify the data location within a Cosmos DB account: - `Database` - `Container` From 42b9c6051a541be62063287482bda7b9fd8f239d Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Mon, 8 Jul 2024 11:44:14 -0700 Subject: [PATCH 4/6] Rename AccountName in Source settings for Azure Blob Storage --- .../AzureBlobDataSink.cs | 2 +- .../AzureBlobDataSource.cs | 7 +++++-- .../AzureBlobSourceSettings.cs | 6 +++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs index b162c7f..2747b15 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSink.cs @@ -18,7 +18,7 @@ public async Task WriteToTargetAsync(Func writeToStream, IConfigur BlobContainerClient account; if (settings.UseRbacAuth) { - logger.LogInformation("Connecting to Storage account {AccountName} using {UseRbacAuth}'", settings.AccountEndpoint, nameof(AzureBlobSourceSettings.UseRbacAuth)); + logger.LogInformation("Connecting to Storage account {AccountEndpoint} using {UseRbacAuth} with {EnableInteractiveCredentials}'", settings.AccountEndpoint, nameof(AzureBlobSourceSettings.UseRbacAuth), nameof(AzureBlobSourceSettings.EnableInteractiveCredentials)); var credential = new DefaultAzureCredential(includeInteractiveCredentials: settings.EnableInteractiveCredentials); #pragma warning disable CS8604 // Validate above ensures AccountEndpoint is not null diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs index 5db0f68..ee36691 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobDataSource.cs @@ -19,10 +19,13 @@ public class AzureBlobDataSource : IComposableDataSource BlobContainerClient account; if (settings.UseRbacAuth) { - logger.LogInformation("Connecting to Storage account {AccountName} using {UseRbacAuth} with {EnableInteractiveCredentials}'", settings.AccountName, nameof(AzureBlobSourceSettings.UseRbacAuth), nameof(AzureBlobSourceSettings.EnableInteractiveCredentials)); + logger.LogInformation("Connecting to Storage account {AccountEndpoint} using {UseRbacAuth} with {EnableInteractiveCredentials}'", settings.AccountEndpoint, nameof(AzureBlobSourceSettings.UseRbacAuth), nameof(AzureBlobSourceSettings.EnableInteractiveCredentials)); var credential = new DefaultAzureCredential(includeInteractiveCredentials: settings.EnableInteractiveCredentials); - var blobContainerUri = new Uri($"https://{settings.AccountName}.queue.core.windows.net"); +#pragma warning disable CS8604 // Validate above ensures AccountEndpoint is not null + var baseUri = new Uri(settings.AccountEndpoint); + var blobContainerUri = new Uri(baseUri, settings.ContainerName); +#pragma warning restore CS8604 // Restore warning account = new BlobContainerClient(blobContainerUri, credential); } diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs index c4070a9..71857dc 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs @@ -9,7 +9,7 @@ public class AzureBlobSourceSettings : IDataExtensionSettings, IValidatableObjec [SensitiveValue] public string? ConnectionString { get; set; } = null!; - public string? AccountName { get; set; } = null!; + public string? AccountEndpoint { get; set; } = null!; [Required] public string ContainerName { get; set; } = null!; @@ -30,9 +30,9 @@ public virtual IEnumerable Validate(ValidationContext validati yield return new ValidationResult($"{nameof(ConnectionString)} must be specified unless {nameof(UseRbacAuth)} is true", new[] { nameof(ConnectionString) }); } - if (UseRbacAuth && string.IsNullOrEmpty(AccountName)) + if (UseRbacAuth && string.IsNullOrEmpty(AccountEndpoint)) { - yield return new ValidationResult($"{nameof(AccountName)} must be specified unless {nameof(UseRbacAuth)} is false", new[] { nameof(AccountName) }); + yield return new ValidationResult($"{nameof(AccountEndpoint)} must be specified unless {nameof(UseRbacAuth)} is false", new[] { nameof(AccountEndpoint) }); } } } \ No newline at end of file From ef94628184bb3e06bacc8e13e4835b808f2f9e16 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Thu, 5 Sep 2024 20:26:05 -0700 Subject: [PATCH 5/6] Remove en-us --- Extensions/Cosmos/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Extensions/Cosmos/README.md b/Extensions/Cosmos/README.md index 0e26dcd..4a921b3 100644 --- a/Extensions/Cosmos/README.md +++ b/Extensions/Cosmos/README.md @@ -9,7 +9,7 @@ The Cosmos data transfer extension provides source and sink capabilities for rea Source and sink require settings used to locate and access the Cosmos DB account. This can be done in one of two ways: - Using a `ConnectionString` that includes an AccountEndpoint and AccountKey -- Using RBAC (Role Based Access Control) by setting `UseRbacAuth` to true and specifying `AccountEndpoint` and optionally `EnableInteractiveCredentials` to prompt the user to log in to Azure if default credentials are not available. See ([migrate-passwordless](https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/migrate-passwordless?tabs=sign-in-azure-cli%2Cdotnet%2Cazure-portal-create%2Cazure-portal-associate%2Capp-service-identity) for how to configure Cosmos DB for passwordless access. +- Using RBAC (Role Based Access Control) by setting `UseRbacAuth` to true and specifying `AccountEndpoint` and optionally `EnableInteractiveCredentials` to prompt the user to log in to Azure if default credentials are not available. See ([migrate-passwordless](https://learn.microsoft.com/azure/cosmos-db/nosql/migrate-passwordless?tabs=sign-in-azure-cli%2Cdotnet%2Cazure-portal-create%2Cazure-portal-associate%2Capp-service-identity) for how to configure Cosmos DB for passwordless access. Source and sink settings also both require parameters to specify the data location within a Cosmos DB account: From 91e7ccb8ac90b15f36eb18920cfaaaaeb32d4630 Mon Sep 17 00:00:00 2001 From: Phil Nachreiner Date: Thu, 5 Sep 2024 20:29:29 -0700 Subject: [PATCH 6/6] Remove EnableInteractiveCredentials default of true --- .../AzureBlobSinkSettings.cs | 2 +- .../AzureBlobSourceSettings.cs | 2 +- .../Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs index dc673c0..477d6fc 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSinkSettings.cs @@ -21,7 +21,7 @@ public class AzureBlobSinkSettings : IDataExtensionSettings, IValidatableObject public bool UseRbacAuth { get; set; } - public bool EnableInteractiveCredentials { get; set; } = true; + public bool EnableInteractiveCredentials { get; set; } public virtual IEnumerable Validate(ValidationContext validationContext) { diff --git a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs index 71857dc..9fab3e9 100644 --- a/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs +++ b/Extensions/AzureBlob/Cosmos.DataTransfer.AzureBlobStorage/AzureBlobSourceSettings.cs @@ -21,7 +21,7 @@ public class AzureBlobSourceSettings : IDataExtensionSettings, IValidatableObjec public bool UseRbacAuth { get; set; } - public bool EnableInteractiveCredentials { get; set; } = true; + public bool EnableInteractiveCredentials { get; set; } public virtual IEnumerable Validate(ValidationContext validationContext) { diff --git a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs index 74fb17e..51a14ff 100644 --- a/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs +++ b/Extensions/Cosmos/Cosmos.DataTransfer.CosmosExtension/CosmosSettingsBase.cs @@ -14,7 +14,7 @@ public abstract class CosmosSettingsBase : IValidatableObject public bool UseRbacAuth { get; set; } public string? AccountEndpoint { get; set; } - public bool EnableInteractiveCredentials { get; set; } = true; + public bool EnableInteractiveCredentials { get; set; } public virtual IEnumerable Validate(ValidationContext validationContext) {