feat(seeding): add parallalization to seeding

Refs: eclipse-tractusx/portal-iam#238
eclipse-tractusx · Dec 11, 2024 · fbd2597 · fbd2597
1 parent f74b7b9
commit fbd2597
Show file tree

Hide file tree

Showing 6 changed files with 177 additions and 146 deletions.
diff --git a/src/keycloak/Keycloak.Seeding/BusinessLogic/ClientScopeMapperUpdater.cs b/src/keycloak/Keycloak.Seeding/BusinessLogic/ClientScopeMapperUpdater.cs
@@ -37,27 +37,31 @@ public async Task UpdateClientScopeMapper(string instanceName, CancellationToken
         var seederConfig = seedDataHandler.GetSpecificConfiguration(ConfigurationKey.ClientScopes);
 
         var clients = await keycloak.GetClientsAsync(realm, null, true, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-        foreach (var (clientName, mappingModels) in seedDataHandler.ClientScopeMappings)
-        {
-            var client = clients.SingleOrDefault(x => x.ClientId == clientName);
-            if (client?.Id is null)
+        await Parallel.ForEachAsync(seedDataHandler.ClientScopeMappings,
+            new ParallelOptions { MaxDegreeOfParallelism = 3, CancellationToken = cancellationToken },
+            async (mappings, token) =>
             {
-                throw new ConflictException($"No client id found with name {clientName}");
-            }
-
-            var roles = await keycloak.GetRolesAsync(realm, client.Id, cancellationToken: cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-            foreach (var mappingModel in mappingModels)
-            {
-                var clientScope = clients.SingleOrDefault(x => x.ClientId == mappingModel.Client);
-                if (clientScope?.Id is null)
+                var (clientName, mappingModels) = mappings;
+                var client = clients.SingleOrDefault(x => x.ClientId == clientName);
+                if (client?.Id is null)
                 {
                     throw new ConflictException($"No client id found with name {clientName}");
                 }
-                var clientRoles = await keycloak.GetClientRolesScopeMappingsForClientAsync(realm, clientScope.Id, client.Id, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-                var mappingModelRoles = mappingModel.Roles?.Select(roleName => roles.SingleOrDefault(r => r.Name == roleName) ?? throw new ConflictException($"No role with name {roleName} found")) ?? Enumerable.Empty<Role>();
-                await AddAndDeleteRoles(keycloak, realm, clientScope.Id, client.Id, clientRoles, mappingModelRoles, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-            }
-        }
+
+                var roles = await keycloak.GetRolesAsync(realm, client.Id, cancellationToken: token).ConfigureAwait(ConfigureAwaitOptions.None);
+                foreach (var mappingModel in mappingModels)
+                {
+                    var clientScope = clients.SingleOrDefault(x => x.ClientId == mappingModel.Client);
+                    if (clientScope?.Id is null)
+                    {
+                        throw new ConflictException($"No client id found with name {clientName}");
+                    }
+
+                    var clientRoles = await keycloak.GetClientRolesScopeMappingsForClientAsync(realm, clientScope.Id, client.Id, token).ConfigureAwait(ConfigureAwaitOptions.None);
+                    var mappingModelRoles = mappingModel.Roles?.Select(roleName => roles.SingleOrDefault(r => r.Name == roleName) ?? throw new ConflictException($"No role with name {roleName} found")) ?? Enumerable.Empty<Role>();
+                    await AddAndDeleteRoles(keycloak, realm, clientScope.Id, client.Id, clientRoles, mappingModelRoles, seederConfig, token).ConfigureAwait(ConfigureAwaitOptions.None);
+                }
+            }).ConfigureAwait(ConfigureAwaitOptions.None);
     }
 
     private static async Task AddAndDeleteRoles(KeycloakClient keycloak, string realm, string clientScopeId, string clientId, IEnumerable<Role> roles, IEnumerable<Role> updateRoles, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)

diff --git a/src/keycloak/Keycloak.Seeding/BusinessLogic/ClientScopesUpdater.cs b/src/keycloak/Keycloak.Seeding/BusinessLogic/ClientScopesUpdater.cs
@@ -39,51 +39,59 @@ public async Task UpdateClientScopes(string instanceName, CancellationToken canc
         var clientScopes = await keycloak.GetClientScopesAsync(realm, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
         var seedClientScopes = seedDataHandler.ClientScopes;
 
-        await CheckAndExecute(ModificationType.Delete, keycloak, realm, clientScopes, seedClientScopes, seederConfig, cancellationToken, RemoveObsoleteClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
-        await CheckAndExecute(ModificationType.Create, keycloak, realm, clientScopes, seedClientScopes, seederConfig, cancellationToken, CreateMissingClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
-        await CheckAndExecute(ModificationType.Update, keycloak, realm, clientScopes, seedClientScopes, seederConfig, cancellationToken, UpdateExistingClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
+        var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = 3, CancellationToken = cancellationToken };
+        await CheckAndExecute(ModificationType.Delete, keycloak, realm, clientScopes, seedClientScopes, seederConfig, parallelOptions, RemoveObsoleteClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
+        await CheckAndExecute(ModificationType.Create, keycloak, realm, clientScopes, seedClientScopes, seederConfig, parallelOptions, CreateMissingClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
+        await CheckAndExecute(ModificationType.Update, keycloak, realm, clientScopes, seedClientScopes, seederConfig, parallelOptions, UpdateExistingClientScopes).ConfigureAwait(ConfigureAwaitOptions.None);
     }
 
-    private static Task CheckAndExecute(ModificationType modificationType, KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken, Func<KeycloakClient, string, IEnumerable<ClientScope>, IEnumerable<ClientScopeModel>, KeycloakSeederConfigModel, CancellationToken, Task> executeLogic) =>
+    private static Task CheckAndExecute(ModificationType modificationType, KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions, Func<KeycloakClient, string, IEnumerable<ClientScope>, IEnumerable<ClientScopeModel>, KeycloakSeederConfigModel, ParallelOptions, Task> executeLogic) =>
         seederConfig.ModificationAllowed(modificationType)
-            ? executeLogic(keycloak, realm, clientScopes, seedClientScopes, seederConfig, cancellationToken)
+            ? executeLogic(keycloak, realm, clientScopes, seedClientScopes, seederConfig, parallelOptions)
             : Task.CompletedTask;
 
-    private static async Task RemoveObsoleteClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
+    private static async Task RemoveObsoleteClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
     {
-        foreach (var deleteScope in clientScopes
-                     .Where(x => seederConfig.ModificationAllowed(ModificationType.Delete, x.Name))
-                     .ExceptBy(seedClientScopes.Select(x => x.Name), x => x.Name))
-        {
-            await keycloak.DeleteClientScopeAsync(
-                realm,
-                deleteScope.Id ?? throw new ConflictException($"clientScope.Id is null: {deleteScope.Name}"),
-                cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-        }
+        var deleteScopes = clientScopes
+            .Where(x => seederConfig.ModificationAllowed(ModificationType.Delete, x.Name))
+            .ExceptBy(seedClientScopes.Select(x => x.Name), x => x.Name);
+        await Parallel.ForEachAsync(deleteScopes, parallelOptions, async (scope, cancellationToken) =>
+            {
+                await keycloak.DeleteClientScopeAsync(
+                    realm,
+                    scope.Id ?? throw new ConflictException($"clientScope.Id is null: {scope.Name}"),
+                    cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
+            })
+            .ConfigureAwait(ConfigureAwaitOptions.None);
     }
 
-    private static async Task CreateMissingClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
+    private static async Task CreateMissingClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
     {
-        foreach (var addScope in seedClientScopes
-                     .Where(x => seederConfig.ModificationAllowed(ModificationType.Create, x.Name))
-                     .ExceptBy(clientScopes.Select(x => x.Name), x => x.Name))
-        {
-            await keycloak.CreateClientScopeAsync(realm, CreateClientScope(null, addScope, true), cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-        }
+        var clientScopeModels = seedClientScopes
+            .Where(x => seederConfig.ModificationAllowed(ModificationType.Create, x.Name))
+            .ExceptBy(clientScopes.Select(x => x.Name), x => x.Name);
+        await Parallel.ForEachAsync(clientScopeModels, parallelOptions, async (scope, cancellationToken) =>
+            {
+                await keycloak.CreateClientScopeAsync(realm, CreateClientScope(null, scope, true), cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
+            })
+            .ConfigureAwait(ConfigureAwaitOptions.None);
     }
 
-    private static async Task UpdateExistingClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
+    private static async Task UpdateExistingClientScopes(KeycloakClient keycloak, string realm, IEnumerable<ClientScope> clientScopes, IEnumerable<ClientScopeModel> seedClientScopes, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
     {
-        foreach (var (clientScope, update) in clientScopes
+        var clientScopeModels = clientScopes
             .Where(x => seederConfig.ModificationAllowed(ModificationType.Update, x.Name))
             .Join(
                 seedClientScopes,
                 x => x.Name,
                 x => x.Name,
-                (clientScope, update) => (ClientScope: clientScope, Update: update)))
-        {
-            await UpdateClientScopeWithProtocolMappers(keycloak, realm, clientScope, update, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-        }
+                (clientScope, update) => (ClientScope: clientScope, Update: update));
+        await Parallel.ForEachAsync(clientScopeModels, parallelOptions, async (scope, cancellationToken) =>
+            {
+                var (clientScope, update) = scope;
+                await UpdateClientScopeWithProtocolMappers(keycloak, realm, clientScope, update, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
+            })
+            .ConfigureAwait(ConfigureAwaitOptions.None);
     }
 
     private static async Task UpdateClientScopeWithProtocolMappers(KeycloakClient keycloak, string realm, ClientScope clientScope, ClientScopeModel update, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)

diff --git a/src/keycloak/Keycloak.Seeding/BusinessLogic/IdentityProvidersUpdater.cs b/src/keycloak/Keycloak.Seeding/BusinessLogic/IdentityProvidersUpdater.cs
@@ -36,7 +36,8 @@ public async Task UpdateIdentityProviders(string keycloakInstanceName, Cancellat
         var keycloak = keycloakFactory.CreateKeycloakClient(keycloakInstanceName);
         var realm = seedDataHandler.Realm;
         var seederConfig = seedDataHandler.GetSpecificConfiguration(ConfigurationKey.IdentityProviders);
-
+        var parallelOptions = new ParallelOptions { CancellationToken = cancellationToken, MaxDegreeOfParallelism = 3 };
+
         foreach (var updateIdentityProvider in seedDataHandler.IdentityProviders)
         {
             if (updateIdentityProvider.Alias == null)
@@ -64,17 +65,19 @@ public async Task UpdateIdentityProviders(string keycloakInstanceName, Cancellat
             var updateMappers = seedDataHandler.IdentityProviderMappers.Where(x => x.IdentityProviderAlias == updateIdentityProvider.Alias);
             var mappers = await keycloak.GetIdentityProviderMappersAsync(realm, updateIdentityProvider.Alias, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
 
-            await DeleteObsoleteIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-            await CreateMissingIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-            await UpdateExistingIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
+            await DeleteObsoleteIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, parallelOptions).ConfigureAwait(ConfigureAwaitOptions.None);
+            await CreateMissingIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, parallelOptions).ConfigureAwait(ConfigureAwaitOptions.None);
+            await UpdateExistingIdentityProviderMappers(keycloak, realm, updateIdentityProvider.Alias, mappers, updateMappers, seederConfig, parallelOptions).ConfigureAwait(ConfigureAwaitOptions.None);
         }
     }
 
-    private static async Task CreateMissingIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
+    private static async Task CreateMissingIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
     {
-        foreach (var mapper in updateMappers
-                     .Where(x => seederConfig.ModificationAllowed(alias, ConfigurationKey.IdentityProviderMappers, ModificationType.Create, x.Name))
-                     .ExceptBy(mappers.Select(x => x.Name), x => x.Name))
+        var addMappers = updateMappers
+            .Where(x => seederConfig.ModificationAllowed(alias, ConfigurationKey.IdentityProviderMappers,
+                ModificationType.Create, x.Name))
+            .ExceptBy(mappers.Select(x => x.Name), x => x.Name);
+        await Parallel.ForEachAsync(addMappers, parallelOptions, async (mapper, cancellationToken) => 
         {
             await keycloak.AddIdentityProviderMapperAsync(
                 realm,
@@ -87,44 +90,46 @@ await keycloak.AddIdentityProviderMapperAsync(
                     },
                     mapper),
                 cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-        }
+        }).ConfigureAwait(ConfigureAwaitOptions.None);
     }
 
-    private static async Task UpdateExistingIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
+    private static async Task UpdateExistingIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
     {
-        foreach (var (mapper, update) in mappers
+        var mappersToUpdate = mappers
             .Where(x => seederConfig.ModificationAllowed(alias, ConfigurationKey.IdentityProviderMappers, ModificationType.Update, x.Name))
             .Join(
                 updateMappers,
                 x => x.Name,
                 x => x.Name,
                 (mapper, update) => (Mapper: mapper, Update: update))
-            .Where(x => !CompareIdentityProviderMapper(x.Mapper, x.Update)))
+            .Where(x => !CompareIdentityProviderMapper(x.Mapper, x.Update));
+        await Parallel.ForEachAsync(mappersToUpdate, parallelOptions, async (mapperModel, cancellationToken) =>
         {
+            var (mapper, update) = mapperModel;
             await keycloak.UpdateIdentityProviderMapperAsync(
                 realm,
                 alias,
                 mapper.Id ?? throw new ConflictException($"identityProviderMapper.id must never be null {mapper.Name} {mapper.IdentityProviderAlias}"),
                 UpdateIdentityProviderMapper(mapper, update),
                 cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-        }
+        }).ConfigureAwait(ConfigureAwaitOptions.None);
     }
 
-    private static async Task DeleteObsoleteIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, CancellationToken cancellationToken)
+    private static async Task DeleteObsoleteIdentityProviderMappers(KeycloakClient keycloak, string realm, string alias, IEnumerable<IdentityProviderMapper> mappers, IEnumerable<IdentityProviderMapperModel> updateMappers, KeycloakSeederConfigModel seederConfig, ParallelOptions parallelOptions)
     {
         if (mappers
             .Where(x => seederConfig.ModificationAllowed(alias, ConfigurationKey.IdentityProviderMappers, ModificationType.Delete, x.Name))
             .ExceptBy(updateMappers.Select(x => x.Name), x => x.Name)
             .IfAny(async deleteMappers =>
                 {
-                    foreach (var mapper in deleteMappers)
+                    await Parallel.ForEachAsync(deleteMappers, parallelOptions, async (mapper, cancellationToken) => 
                     {
                         await keycloak.DeleteIdentityProviderMapperAsync(
                             realm,
                             alias,
                             mapper.Id ?? throw new ConflictException($"identityProviderMapper.id must never be null {mapper.Name} {mapper.IdentityProviderAlias}"),
                             cancellationToken).ConfigureAwait(ConfigureAwaitOptions.None);
-                    }
+                    }).ConfigureAwait(ConfigureAwaitOptions.None);
                 },
                 out var deleteMappersTask))
         {