Skip to content

Commit

Permalink
Merge pull request #17 from dlmelendez/rel/2.x
Browse files Browse the repository at this point in the history 
Rel/2.x
  • Loading branch information
@dlmelendez
dlmelendez authored Aug 28, 2020
2 parents ab73193 + ecc09a7 commit 05ee09c
Show file tree
Hide file tree
Showing 32 changed files with 862 additions and 244 deletions.
33 changes: 30 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,34 @@
# identityserver4-azurestorage
Uses Azure Blob and Table Storage services as an alternative to [Entity Framework/SQL data access for IdentityServer4](https://identityserver4.readthedocs.io/en/latest/quickstarts/7_entity_framework.html#identityserver4-entityframework).
Uses Azure Blob and Table Storage services as an alternative to [Entity Framework/SQL data access for IdentityServer4](https://identityserver4.readthedocs.io/en/latest/quickstarts/5_entityframework.html).
Use the unit tests as a guide to seeding operational and configuration data.
- ElCamino.IdentityServer4.AzureStorage v1.x uses IdentityServer4 2.x & 3.x
- ElCamino.IdentityServer4.AzureStorage v2.x uses IdentityServer4 >= 4.x

[![Build Status](https://dev.azure.com/elcamino/Azure%20OpenSource/_apis/build/status/dlmelendez.identityserver4-azurestorage?branchName=master)](https://dev.azure.com/elcamino/Azure%20OpenSource/_build/latest?definitionId=11&branchName=master)

[![NuGet Badge](https://buildstats.info/nuget/ElCamino.IdentityServer4.AzureStorage)](https://www.nuget.org/packages/ElCamino.IdentityServer4.AzureStorage/)

# IdentityServer4 v3 to v4
There are breaking changes when moving to IdentityServer4 v3 to v4. and respectively upgrading ElCamino.IdentityServer4.AzureStorage v1.x to v2.x.

## Config changes to support ApiScope blobs
New config settings, complete settings further down.
```json
{
"IdentityServer4": {
...
"resourceStorageConfig": {
"apiScopeBlobContainerName": "idsrv4apiscopes",
"apiScopeBlobCacheContainerName": "idsrv4apiscopescache",
}...
}
}
```

## Changes to startup.cs

Shown below in complete context, add .MigrateResourceV3Storage() into the startup services pipline. Remove after the first run.

# Getting Started
## startup.cs
```C#
Expand All @@ -27,6 +50,8 @@ using IdentityServer4;
.CreateResourceStorage() //Can be removed after first run.
.AddDeviceFlowContext(Configuration.GetSection("IdentityServer4:deviceFlowStorageConfig"))
.CreateDeviceFlowStorage() //Can be removed after first run.
//Use for migrating ApiScopes from IdentityServer4 v3 ApiResources
//.MigrateResourceV3Storage();
// Adds IdentityServer
services.AddIdentityServer()
Expand Down Expand Up @@ -58,11 +83,13 @@ using IdentityServer4;
"cacheRefreshInterval": 1800
},
"resourceStorageConfig": {
"storageConnectionString": "UseDevelopmentStorage=true;",
"storageConnectionString": "UseDevelopmentStorage=true;",
"apiTableName": "idsrv4apiscopeindex",
"apiBlobContainerName": "idsrv4apiresources",
"apiScopeBlobContainerName": "idsrv4apiscopes",
"identityBlobContainerName": "idsrv4identityresources",
"apiBlobCacheContainerName": "idsrv4apiresourcescache",
"apiBlobCacheContainerName": "idsrv4apiresourcescache",
"apiScopeBlobCacheContainerName": "idsrv4apiscopescache",
"identityBlobCacheContainerName": "idsrv4identityresourcescache",
"enableCacheRefresh": true,
"cacheRefreshInterval": 1800
Expand Down
7 changes: 7 additions & 0 deletions src/Elcamino.IdentityServer4.AzureTableStorage/Configuration/ResourceStorageConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,22 @@ public class ResourceStorageConfig
[JsonProperty("apiBlobContainerName")]
public string ApiBlobContainerName { get; set; }

[JsonProperty("apiScopeBlobContainerName")]
public string ApiScopeBlobContainerName { get; set; }

[JsonProperty("identityBlobContainerName")]
public string IdentityBlobContainerName { get; set; }

[JsonProperty("apiBlobCacheContainerName")]
public string ApiBlobCacheContainerName { get; set; }

[JsonProperty("apiScopeBlobCacheContainerName")]
public string ApiScopeBlobCacheContainerName { get; set; }

[JsonProperty("identityBlobCacheContainerName")]
public string IdentityBlobCacheContainerName { get; set; }


/// <summary>
/// Gets or sets a value indicating whether blob cache will be refreshed on a schedule.
/// This is implemented by periodically connecting to blob storage(according to the CacheRefreshInterval) from the hosting application.
Expand Down
7 changes: 7 additions & 0 deletions src/Elcamino.IdentityServer4.AzureTableStorage/Configuration/ServiceCollectionExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,13 @@ public static IServiceCollection CreateResourceStorage(this IServiceCollection s
return services;
}

public static IServiceCollection MigrateResourceV3Storage(this IServiceCollection services)
{
ResourceStore resourceStore = services.BuildServiceProvider().GetService<ResourceStore>();
resourceStore.MigrateV3ApiScopesAsync().Wait();
return services;
}


}
}
32 changes: 27 additions & 5 deletions src/Elcamino.IdentityServer4.AzureTableStorage/Contexts/ResourceStorageContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,37 +17,42 @@ public class ResourceStorageContext : StorageContext
{
private ResourceStorageConfig _config = null;
private string ApiBlobContainerName = string.Empty;
private string ApiScopeBlobContainerName = string.Empty;
private string IdentityBlobContainerName = string.Empty;

private string ApiBlobCacheContainerName = string.Empty;
private string ApiScopeBlobCacheContainerName = string.Empty;
private string IdentityBlobCacheContainerName = string.Empty;

public const string DefaultApiBlobCacheContainerName = "resourceapiblobcache";
public const string DefaultApiScopeBlobCacheContainerName = "resourceapiscopeblobcache";
public const string DefaultIdentityBlobCacheContainerName = "resourceidentityblobcache";

public string ApiResourceTableName { get; private set; }

public CloudTable ApiResourceTable { get; private set; }

public BlobContainerClient ApiResourceBlobContainer { get; private set; }

public BlobContainerClient IdentityResourceBlobContainer { get; private set; }
public BlobContainerClient ApiResourceBlobContainer { get; private set; }

public BlobContainerClient ApiResourceBlobCacheContainer { get; private set; }

public BlobContainerClient IdentityResourceBlobCacheContainer { get; private set; }
public BlobContainerClient ApiScopeBlobContainer { get; private set; }

public BlobContainerClient ApiScopeBlobCacheContainer { get; private set; }

public BlobContainerClient IdentityResourceBlobContainer { get; private set; }

public BlobContainerClient IdentityResourceBlobCacheContainer { get; private set; }

public CloudTableClient TableClient { get; private set; }

public BlobServiceClient BlobClient { get; private set; }


public ResourceStorageContext(IOptions<ResourceStorageConfig> config) : this(config.Value)
{
}


public ResourceStorageContext(ResourceStorageConfig config)
{
if (config == null)
Expand All @@ -62,8 +67,10 @@ public async Task<bool> CreateStorageIfNotExists()
var tasks = new List<Task>() {
ApiResourceTable.CreateIfNotExistsAsync(),
ApiResourceBlobContainer.CreateIfNotExistsAsync(),
ApiScopeBlobContainer.CreateIfNotExistsAsync(),
IdentityResourceBlobContainer.CreateIfNotExistsAsync(),
ApiResourceBlobCacheContainer.CreateIfNotExistsAsync(),
ApiScopeBlobCacheContainer.CreateIfNotExistsAsync(),
IdentityResourceBlobCacheContainer.CreateIfNotExistsAsync()};
await Task.WhenAll(tasks).ConfigureAwait(false);
return tasks.Select(t => t.IsCompleted).All(a => a);
Expand All @@ -75,6 +82,7 @@ protected virtual void Initialize(ResourceStorageConfig config)
TableClient = Microsoft.Azure.Cosmos.Table.CloudStorageAccount.Parse(_config.StorageConnectionString).CreateCloudTableClient();
TableClient.DefaultRequestOptions.PayloadFormat = TablePayloadFormat.Json;

//ApiResourceTableName
ApiResourceTableName = config.ApiTableName;

if (string.IsNullOrWhiteSpace(ApiResourceTableName))
Expand All @@ -85,6 +93,8 @@ protected virtual void Initialize(ResourceStorageConfig config)
ApiResourceTable = TableClient.GetTableReference(ApiResourceTableName);

BlobClient = new BlobServiceClient(_config.StorageConnectionString);

// ApiResource blob config
ApiBlobContainerName = config.ApiBlobContainerName;
if (string.IsNullOrWhiteSpace(ApiBlobContainerName))
{
Expand All @@ -95,6 +105,18 @@ protected virtual void Initialize(ResourceStorageConfig config)
ApiBlobCacheContainerName = !string.IsNullOrWhiteSpace(config.ApiBlobCacheContainerName) ? config.ApiBlobCacheContainerName : DefaultApiBlobCacheContainerName;
ApiResourceBlobCacheContainer = BlobClient.GetBlobContainerClient(ApiBlobCacheContainerName);

// ApiScope blob config
ApiScopeBlobContainerName = config.ApiScopeBlobContainerName;
if (string.IsNullOrWhiteSpace(ApiScopeBlobContainerName))
{
throw new ArgumentException($"ApiScopeBlobContainerName cannot be null or empty, check your configuration.", nameof(config.ApiScopeBlobContainerName));
}
ApiScopeBlobContainer = BlobClient.GetBlobContainerClient(ApiScopeBlobContainerName);

ApiScopeBlobCacheContainerName = !string.IsNullOrWhiteSpace(config.ApiScopeBlobCacheContainerName) ? config.ApiScopeBlobCacheContainerName : DefaultApiScopeBlobCacheContainerName;
ApiScopeBlobCacheContainer = BlobClient.GetBlobContainerClient(ApiScopeBlobCacheContainerName);

//IdentityResource blob config
IdentityBlobContainerName = config.IdentityBlobContainerName;
if (string.IsNullOrWhiteSpace(IdentityBlobContainerName))
{
Expand Down
75 changes: 55 additions & 20 deletions src/Elcamino.IdentityServer4.AzureTableStorage/Contexts/StorageContext.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ protected StorageContext() { }
public async Task<string> GetBlobContentAsync(string keyNotHashed, BlobContainerClient container)
{
BlobClient blob = container.GetBlobClient(KeyGeneratorHelper.GenerateHashValue(keyNotHashed));
return await GetBlobContentAsync(blob);
return await GetBlobContentAsync(blob).ConfigureAwait(false);
}

public async Task<string> GetBlobContentAsync(BlobClient blob)
Expand All @@ -34,7 +34,7 @@ public async Task<string> GetBlobContentAsync(BlobClient blob)
Response<BlobDownloadInfo> download = await blob.DownloadAsync();
using (StreamReader sr = new StreamReader(download.Value.Content, Encoding.UTF8))
{
return await sr.ReadToEndAsync();
return await sr.ReadToEndAsync().ConfigureAwait(false);
}
}
catch (RequestFailedException ex)
Expand All @@ -49,13 +49,15 @@ public async Task<string> GetBlobContentAsync(BlobClient blob)
{
DateTime dateTimeNow = DateTime.UtcNow;
string blobName = KeyGeneratorHelper.GenerateDateTimeDecendingId(dateTimeNow);
await SaveBlobAsync(blobName, JsonConvert.SerializeObject(entities), cacheContainer);
await SaveBlobAsync(blobName, JsonConvert.SerializeObject(entities), cacheContainer)
.ConfigureAwait(false);
return (blobName, count: entities.Count());
}

public async Task<(string blobName, int count)> UpdateBlobCacheFileAsync<Entity>(IAsyncEnumerable<Entity> entities, BlobContainerClient cacheContainer)
{
return await UpdateBlobCacheFileAsync(await entities.ToListAsync(), cacheContainer);
return await UpdateBlobCacheFileAsync(await entities.ToListAsync().ConfigureAwait(false), cacheContainer)
.ConfigureAwait(false);
}

/// <summary>
Expand All @@ -71,18 +73,18 @@ public async Task DeleteBlobCacheFilesAsync(string latestBlobName, BlobContainer
if (String.Compare(latestBlobName, blobName.Name) == -1)
{
BlobClient blobClient = cacheContainer.GetBlobClient(blobName.Name);
await blobClient.DeleteAsync();
await blobClient.DeleteAsync().ConfigureAwait(false);
logger.LogInformation($"container: {cacheContainer.Name} blob: {blobName.Name} - cache file deleted");
}
}
}

public async Task<IEnumerable<Entity>> GetLatestFromCacheBlobAsync<Entity>(BlobContainerClient cacheContainer)
{
BlobClient blob = await GetFirstBlobAsync(cacheContainer);
BlobClient blob = await GetFirstBlobAsync(cacheContainer).ConfigureAwait(false);
if (blob != null)
{
var entities = await GetEntityBlobAsync<List<Entity>>(blob);
var entities = await GetEntityBlobAsync<List<Entity>>(blob).ConfigureAwait(false);
if (entities != null)
{
return entities;
Expand All @@ -94,14 +96,14 @@ public async Task<IEnumerable<Entity>> GetLatestFromCacheBlobAsync<Entity>(BlobC
public async Task<Entity> GetEntityBlobAsync<Entity>(string keyNotHashed, BlobContainerClient container) where Entity : class, new()
{
BlobClient blob = container.GetBlobClient(KeyGeneratorHelper.GenerateHashValue(keyNotHashed));
return await GetEntityBlobAsync<Entity>(blob);
return await GetEntityBlobAsync<Entity>(blob).ConfigureAwait(false);
}

public async Task<Entity> GetEntityBlobAsync<Entity>(BlobClient blobJson) where Entity : class, new()
{
try
{
var download = await blobJson.DownloadAsync();
var download = await blobJson.DownloadAsync().ConfigureAwait(false);
using (Stream s = download.Value.Content)
{
using (StreamReader sr = new StreamReader(s, Encoding.UTF8))
Expand All @@ -125,10 +127,42 @@ public async Task<IEnumerable<Entity>> GetLatestFromCacheBlobAsync<Entity>(BlobC
{
await foreach(var blobJson in GetAllBlobsAsync(container))
{
yield return await GetEntityBlobAsync<Entity>(blobJson);
Entity e = await GetEntityBlobAsync<Entity>(blobJson).ConfigureAwait(false);
if (e != null)
{
yield return e;
}
}
}

/// <summary>
/// Only gets blob entities that have no serialization errors
/// </summary>
/// <typeparam name="Entity"></typeparam>
/// <param name="container"></param>
/// <param name="logger"></param>
/// <returns></returns>
public async IAsyncEnumerable<Entity> SafeGetAllBlobEntitiesAsync<Entity>(BlobContainerClient container, ILogger logger) where Entity : class, new()
{
await foreach (var blobJson in GetAllBlobsAsync(container))
{
Entity e = null;
try
{
e = await GetEntityBlobAsync<Entity>(blobJson).ConfigureAwait(false);
}
catch(Exception ex)
{
// log and continue
logger.LogError(ex, $"{nameof(SafeGetAllBlobEntitiesAsync)}-{nameof(GetEntityBlobAsync)} error:");
continue;
}
if (e != null)
{
yield return e;
}
}
}


public async IAsyncEnumerable<BlobClient> GetAllBlobsAsync(BlobContainerClient container)
{
Expand Down Expand Up @@ -161,26 +195,26 @@ public async Task<BlobClient> GetFirstBlobAsync(BlobContainerClient container)
return null;
}

public async Task DeleteBlobAsync(string keyNotHashed, BlobContainerClient container)
public Task DeleteBlobAsync(string keyNotHashed, BlobContainerClient container)
{
BlobClient blob = container.GetBlobClient(KeyGeneratorHelper.GenerateHashValue(keyNotHashed));
await blob.DeleteIfExistsAsync();
return blob.DeleteIfExistsAsync();
}

public async Task SaveBlobWithHashedKeyAsync(string keyNotHashed, string jsonEntityContent, BlobContainerClient container)
public Task SaveBlobWithHashedKeyAsync(string keyNotHashed, string jsonEntityContent, BlobContainerClient container)
{
BlobClient blob = container.GetBlobClient(KeyGeneratorHelper.GenerateHashValue(keyNotHashed));

await blob.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes(jsonEntityContent)), new BlobHttpHeaders()
return blob.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes(jsonEntityContent)), new BlobHttpHeaders()
{
ContentType = "application/json"
});
}

public async Task SaveBlobAsync(string blobName, string jsonEntityContent, BlobContainerClient container)
public Task SaveBlobAsync(string blobName, string jsonEntityContent, BlobContainerClient container)
{
BlobClient blob = container.GetBlobClient(blobName);
await blob.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes(jsonEntityContent)), new BlobHttpHeaders()
return blob.UploadAsync(new MemoryStream(Encoding.UTF8.GetBytes(jsonEntityContent)), new BlobHttpHeaders()
{
ContentType = "application/json"
});
Expand All @@ -196,16 +230,17 @@ public async Task<Entity> GetEntityTableAsync<Entity>(string keyNotHashed, Cloud
where Entity : class, ITableEntity, new()
{
string hashedKey = KeyGeneratorHelper.GenerateHashValue(keyNotHashed);
var r = await table.ExecuteAsync(TableOperation.Retrieve<Entity>(hashedKey, hashedKey));
var r = await table.ExecuteAsync(TableOperation.Retrieve<Entity>(hashedKey, hashedKey))
.ConfigureAwait(false);
return r.Result as Entity;
}

public async Task GetAndDeleteTableEntityByKeysAsync(string partitionKey, string rowKey, CloudTable table)
{
var entity = (await table.ExecuteAsync(TableOperation.Retrieve(partitionKey, rowKey))).Result as ITableEntity;
var entity = (await table.ExecuteAsync(TableOperation.Retrieve(partitionKey, rowKey)).ConfigureAwait(false)).Result as ITableEntity;
if (entity != null)
{
await table.ExecuteAsync(TableOperation.Delete(entity));
await table.ExecuteAsync(TableOperation.Delete(entity)).ConfigureAwait(false);
}

}
Expand Down
Loading

0 comments on commit 05ee09c

Please sign in to comment.