diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContext.cs b/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContext.cs index 16f9849..735adab 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContext.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/IMaterializeDataContext.cs @@ -26,7 +26,9 @@ public interface IMaterializeDataContext TableNameTerm GetLoadingTableName(string tableName); TableNameTerm GetLoadingTableName(); TableNameTerm GetLoadingTableName(Type modelType); - void CancelMaterializationProcess(); bool IsCancelled { get; } + IMaterializeDataContext CancelMaterializationProcess(); + bool IsMaterializedLoadingTableCleanupEnabled { get; } + IMaterializeDataContext DisableMaterializedLoadingTableCleanup(); } } \ No newline at end of file diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs index c80d604..c894cbb 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataContext.cs @@ -14,6 +14,7 @@ public class MaterializeDataContext : IMaterializeDataContextCompletionSource, I protected ISqlBulkHelpersConfig BulkHelpersConfig { get; } protected bool IsDisposed { get; set; } = false; public bool IsCancelled { get; protected set; } = false; + public bool IsMaterializedLoadingTableCleanupEnabled { get; protected set; } = true; protected List TablesWithFullTextIndexesRemoved { get; set; } = new List(); @@ -25,7 +26,30 @@ public class MaterializeDataContext : IMaterializeDataContextCompletionSource, I public MaterializationTableInfo this[Type modelType] => FindMaterializationTableInfoCaseInsensitive(modelType); - public void CancelMaterializationProcess() => IsCancelled = true; + /// + /// Provides ability to manually control if materialization process is Cancelled for advanced validation and control flow support. + /// + /// + public IMaterializeDataContext CancelMaterializationProcess() + { + IsCancelled = true; + return this; + } + + /// + /// Provides ability to manually control if Materialized Loading tables are cleaned-up/removed when using `SchemaCopyMode.OutsideTransactionAvoidSchemaLocks` + /// for advanced debugging and control flow support; always enabled by default and throws an `InvalidOperationException` if if SchemaCopyMode.InsideTransactionAllowSchemaLocks is used. + /// + /// + /// + public IMaterializeDataContext DisableMaterializedLoadingTableCleanup() + { + if (BulkHelpersConfig.MaterializedDataSchemaCopyMode == SchemaCopyMode.InsideTransactionAllowSchemaLocks) + throw new InvalidOperationException("You cannot disable the cleanup of Materialized Loading tables when using SchemaCopyMode.InsideTransactionAllowSchemaLocks."); + + IsMaterializedLoadingTableCleanupEnabled = false; + return this; + } public TableNameTerm GetLoadingTableName(string tableName) { diff --git a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs index d1a1fb0..087879c 100644 --- a/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs +++ b/NetStandard.SqlBulkHelpers/MaterializedData/MaterializeDataHelper.cs @@ -72,10 +72,12 @@ public async Task CleanupMaterializeDataProcessAsync(Sq //Explicitly clean up all Loading/Discarding Tables (contains old data) to free resources -- this leaves us with only the (new) Live Table in place! foreach (var materializationTableInfo in materializationTables) { - switchScriptBuilder - //Finally cleanup the Loading and Discarding tables... - .DropTableIfExists(materializationTableInfo.LoadingTable) - .DropTableIfExists(materializationTableInfo.DiscardingTable); + //ALWAYS cleanup the Discarding tables... + switchScriptBuilder.DropTableIfExists(materializationTableInfo.DiscardingTable); + + //IF enabled (default is always Enabled) then cleanup up the Loading Tables + if (materializedDataContext.IsMaterializedLoadingTableCleanupEnabled) + switchScriptBuilder.DropTableIfExists(materializationTableInfo.LoadingTable); } await sqlTransaction.ExecuteMaterializedDataSqlScriptAsync( diff --git a/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj b/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj index 032c019..fe0a1d0 100644 --- a/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj +++ b/NetStandard.SqlBulkHelpers/NetStandard.SqlBulkHelpers.csproj @@ -8,18 +8,22 @@ MIT BBernard / CajunCoding CajunCoding - 2.4.1 + 2.4.2 https://github.com/cajuncoding/SqlBulkHelpers https://github.com/cajuncoding/SqlBulkHelpers A library for easy, efficient and high performance bulk insert and update of data, into a Sql Database, from .Net applications. By leveraging the power of the SqlBulkCopy classes with added support for Identity primary key table columns this library provides a greatly simplified interface to process Identity based Entities with Bulk Performance with the wide compatibility of .NetStandard 2.0. sql server database table bulk insert update identity column sqlbulkcopy orm dapper linq2sql materialization materialized data view materialized-data materialized-view sync replication replica readonly + - Add Support to manually control if Materialized Loading tables are cleaned-up/removed when using `SchemaCopyMode.OutsideTransactionAvoidSchemaLocks` via `materializeDataContext.DisableMaterializedStagingTableCleanup()`; + always enabled by default and throws an `InvalidOperationException` if if SchemaCopyMode.InsideTransactionAllowSchemaLocks is used. This provides support for advanced debugging and control flow support. + - Improved SqlBulkHelpers Configuration API to now provide Clone() and Configure() methods to more easily copy/clone existing configuration and change values is specific instances; + including copy/clone of the Defaults for unique exeuctions. - Added support to load Table Schema for Temp Tables (basic Schema details needed for BulkInsert or Update, etc. to allow Bulk Loading Temp Tables! - Improved Error message for when custom SQL Merge Match qualifiers are specified but DB Schema may have changed making them invalid or missing from Cached schema. + - Added new explicit CopyTableDataAsync() APIs which enable explicit copying of data between two tables on matching columns (automatically detected by column Name and Data Type). + - Added new Materialized Data Configuration value MaterializedDataLoadingTableDataCopyMode to control whether the materialized data process automatically copies data into the Loading Tables after cloning. This helps to greatly simplify new use cases where data must be merged (and preserved) during the materialization process. Prior Relese Notes: - -Added new explicit CopyTableDataAsync() APIs which enable explicit copying of data between two tables on matching columns (automatically detected by column Name and Data Type). - -Added new Materialized Data Configuration value MaterializedDataLoadingTableDataCopyMode to control whether the materialized data process automatically copies data into the Loading Tables after cloning. This helps to greatly simplify new use cases where data must be merged (and preserved) during the materialization process. - Fixed bug with Sql Bulk Insert/Update processing with Model Properties that have mapped database names via mapping attribute (e.g. [SqlBulkColumn("")], [Map("")], [Column("")], etc.). - Changed default behaviour to no longer clone tables/schema inside a Transaction which creates a full Schema Lock -- as this greatly impacts Schema aware ORMs such as SqlBulkHelpers, RepoDb, etc. - New separate methods is now added to handle the CleanupMaterializeDataProcessAsync() but must be explicitly called as it is no longer implicitly called with FinishMaterializeDataProcessAsync(). diff --git a/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs b/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs index 7d0f8cc..eb03eb8 100644 --- a/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs +++ b/NetStandard.SqlBulkHelpers/SqlBulkHelpersConfig.cs @@ -52,19 +52,54 @@ public interface ISqlBulkHelpersConfig int MaxConcurrentConnections { get; } bool IsConcurrentConnectionProcessingEnabled { get; } bool IsFullTextIndexHandlingEnabled { get; } + + ISqlBulkHelpersConfig Clone(); + ISqlBulkHelpersConfig Configure(Action configAction); } public class SqlBulkHelpersConfig : ISqlBulkHelpersConfig { public static ISqlBulkHelpersConfig DefaultConfig { get; private set; } = new SqlBulkHelpersConfig(); - public static SqlBulkHelpersConfig Create(Action configAction) + // Constructor for cloning + private SqlBulkHelpersConfig(ISqlBulkHelpersConfig otherConfigToClone = null) { - configAction.AssertArgumentIsNotNull(nameof(configAction)); + if (otherConfigToClone == null) return; + + // Copy property values from the other instance + this.SqlBulkBatchSize = otherConfigToClone.SqlBulkBatchSize; + this.SqlBulkPerBatchTimeoutSeconds = otherConfigToClone.SqlBulkPerBatchTimeoutSeconds; + this.IsSqlBulkTableLockEnabled = otherConfigToClone.IsSqlBulkTableLockEnabled; + this.SqlBulkCopyOptions = otherConfigToClone.SqlBulkCopyOptions; + this.DbSchemaLoaderQueryTimeoutSeconds = otherConfigToClone.DbSchemaLoaderQueryTimeoutSeconds; + this.MaterializeDataStructureProcessingTimeoutSeconds = otherConfigToClone.MaterializeDataStructureProcessingTimeoutSeconds; + this.MaterializedDataSwitchTableWaitTimeoutMinutes = otherConfigToClone.MaterializedDataSwitchTableWaitTimeoutMinutes; + this.MaterializedDataSwitchTimeoutAction = otherConfigToClone.MaterializedDataSwitchTimeoutAction; + this.MaterializedDataSchemaCopyMode = otherConfigToClone.MaterializedDataSchemaCopyMode; + this.MaterializedDataLoadingTableDataCopyMode = otherConfigToClone.MaterializedDataLoadingTableDataCopyMode; + this.MaterializedDataMakeSchemaCopyNamesUnique = otherConfigToClone.MaterializedDataMakeSchemaCopyNamesUnique; + this.MaterializedDataLoadingSchema = otherConfigToClone.MaterializedDataLoadingSchema; + this.MaterializedDataLoadingTablePrefix = otherConfigToClone.MaterializedDataLoadingTablePrefix; + this.MaterializedDataLoadingTableSuffix = otherConfigToClone.MaterializedDataLoadingTableSuffix; + this.MaterializedDataDiscardingSchema = otherConfigToClone.MaterializedDataDiscardingSchema; + this.MaterializedDataDiscardingTablePrefix = otherConfigToClone.MaterializedDataDiscardingTablePrefix; + this.MaterializedDataDiscardingTableSuffix = otherConfigToClone.MaterializedDataDiscardingTableSuffix; + this.IsCloningIdentitySeedValueEnabled = otherConfigToClone.IsCloningIdentitySeedValueEnabled; + this.ConcurrentConnectionFactory = otherConfigToClone.ConcurrentConnectionFactory; + this.MaxConcurrentConnections = otherConfigToClone.MaxConcurrentConnections; + this.IsFullTextIndexHandlingEnabled = otherConfigToClone.IsFullTextIndexHandlingEnabled; + } + + public static SqlBulkHelpersConfig Create(Action configAction, ISqlBulkHelpersConfig otherConfigToClone = null) + => (SqlBulkHelpersConfig)new SqlBulkHelpersConfig(otherConfigToClone).Configure(configAction); + + public ISqlBulkHelpersConfig Clone() => new SqlBulkHelpersConfig(this); - var newConfig = new SqlBulkHelpersConfig(); - configAction.Invoke(newConfig); - return newConfig; + public ISqlBulkHelpersConfig Configure(Action configAction) + { + configAction.AssertArgumentIsNotNull(nameof(configAction)); + configAction.Invoke(this); + return this; } /// @@ -125,6 +160,8 @@ public void EnableConcurrentSqlConnectionProcessing( this.IsFullTextIndexHandlingEnabled = IsFullTextIndexHandlingEnabled || enableFullTextIndexHandling; } + #region All Public Properties / Config Setttings... + public int SqlBulkBatchSize { get; set; } = 2000; //General guidance is that 2000-5000 is efficient enough. public int SqlBulkPerBatchTimeoutSeconds { get; set; } @@ -186,5 +223,7 @@ public int MaxConcurrentConnections /// Recommended to use the SqlBulkHelpersConfig.EnableConcurrentSqlConnectionProcessing() convenience method(s) to enable this more easily! /// public bool IsFullTextIndexHandlingEnabled { get; set; } = false; + + #endregion } } diff --git a/README.md b/README.md index ecbed45..75d9f70 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,21 @@ public class TestDataService ## Nuget Package To use in your project, add the [SqlBulkHelpers NuGet package](https://www.nuget.org/packages/SqlBulkHelpers/) to your project. +### v2.4.2 Release Notes: +- Add Support to manually control if Materialized Loading tables are cleaned-up/removed when using `SchemaCopyMode.OutsideTransactionAvoidSchemaLocks` via `materializeDataContext.DisableMaterializedStagingTableCleanup()`; + always enabled by default and throws an `InvalidOperationException` if if SchemaCopyMode.InsideTransactionAllowSchemaLocks is used. This provides support for advanced debugging and control flow support. +- Improved SqlBulkHelpers Configuration API to now provide Clone() and Configure() methods to more easily copy/clone existing configuration and change values is specific instances; + including copy/clone of the Defaults for unique exeuctions. + +### v2.4.1 Release Notes: +- Added support to load Table Schema for Temp Tables (basic Schema details needed for BulkInsert or Update, etc. to allow Bulk Loading Temp Tables! +- Improved Error message for when custom SQL Merge Match qualifiers are specified but DB Schema may have changed making them invalid or missing from Cached schema. + +### v2.4.0 Release Notes: +- Added new explicit CopyTableDataAsync() APIs which enable explicit copying of data between two tables on matching columns (automatically detected by column Name and Data Type). +- Added new Materialized Data Configuration value MaterializedDataLoadingTableDataCopyMode to control whether the materialized data process automatically copies data into the Loading Tables after cloning. + This helps to greatly simplify new use cases where data must be merged (and preserved) during the materialization process. + ## v2.3.1 Release Notes: - Fixed bug with Sql Bulk Insert/Update processing with Model Properties that have mapped database names via mapping attribute (e.g. [SqlBulkColumn("")], [Map("")], [Column("")], etc.). diff --git a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs index 3408caf..757aac5 100644 --- a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs +++ b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/CloneTablesTests.cs @@ -21,8 +21,8 @@ public async Task TestCloneTableStructureByAnnotationAsync() { var cloneInfo = await sqlTrans.CloneTableAsync().ConfigureAwait(false); - var sourceTableSchema = sqlTrans.GetTableSchemaDefinition(cloneInfo.SourceTable.FullyQualifiedTableName); - var clonedTableSchema = sqlTrans.GetTableSchemaDefinition(cloneInfo.TargetTable.FullyQualifiedTableName); + var sourceTableSchema = await sqlTrans.GetTableSchemaDefinitionAsync(cloneInfo.SourceTable.FullyQualifiedTableName).ConfigureAwait(false); + var clonedTableSchema = await sqlTrans.GetTableSchemaDefinitionAsync(cloneInfo.TargetTable.FullyQualifiedTableName).ConfigureAwait(false); await sqlTrans.RollbackAsync().ConfigureAwait(false); //await sqlTransaction.CommitAsync().ConfigureAwait(false); diff --git a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs index ccc0311..d77e745 100644 --- a/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs +++ b/SqlBulkHelpers.Tests/IntegrationTests/MaterializeDataTests/MaterializeIntoTests.cs @@ -5,6 +5,7 @@ using Microsoft.Data.SqlClient; using RepoDb; using SqlBulkHelpers.Utilities; +using SqlBulkHelpers.CustomExtensions; namespace SqlBulkHelpers.IntegrationTests { @@ -305,7 +306,7 @@ public async Task TestMaterializeDataContextRetrieveTableInfoAsync() //****************************************************************************************** //START the Materialize Data Process... //****************************************************************************************** - await sqlConn.ExecuteMaterializeDataProcessAsync(new[] { typeof(TestElementWithMappedNames) }, async (materializeDataContext, sqlTransaction) => + await sqlConn.ExecuteMaterializeDataProcessAsync(new[] { typeof(TestElementWithMappedNames) }, (materializeDataContext, sqlTransaction) => { var tableInfoByIndex = materializeDataContext[0]; Assert.IsNotNull(tableInfoByIndex); @@ -321,6 +322,8 @@ await sqlConn.ExecuteMaterializeDataProcessAsync(new[] { typeof(TestElementWithM //TEST Passive Cancellation process (no need to throw an exception in this advanced use case)... materializeDataContext.CancelMaterializationProcess(); + + return Task.CompletedTask; }).ConfigureAwait(false); @@ -329,6 +332,68 @@ await sqlConn.ExecuteMaterializeDataProcessAsync(new[] { typeof(TestElementWithM } } + [TestMethod] + public async Task TestMaterializeDataContextWithLoadingTableCleanupDisabledAsync() + { + var sqlConnectionProvider = SqlConnectionHelper.GetConnectionProvider(); + var timer = Stopwatch.StartNew(); + + string? finalLoadingTableName = null; + + //NOW Materialize Data into the Tables! + await using (var sqlConn = await sqlConnectionProvider.NewConnectionAsync().ConfigureAwait(false)) + { + var testDataCount = 1; + + var loadingTableSuffix = $"_{nameof(TestMaterializeDataContextWithLoadingTableCleanupDisabledAsync)}"; + + //****************************************************************************************** + //START the Materialize Data Process... + //****************************************************************************************** + var bulkHelpersConfig = SqlBulkHelpersConfig.DefaultConfig.Clone().Configure(c => + { + c.MaterializedDataLoadingTableSuffix = loadingTableSuffix; + c.MaterializedDataMakeSchemaCopyNamesUnique = false; + }); + + await sqlConn.ExecuteMaterializeDataProcessAsync((materializeDataContext, sqlTransaction) => + { + finalLoadingTableName = materializeDataContext.GetLoadingTableName(); + finalLoadingTableName.AssertArgumentIsNotNullOrWhiteSpace(nameof(finalLoadingTableName)); + TestContext.WriteLine($"Loading Table Name (that will not be cleaned up is: [{finalLoadingTableName}]"); + + //TEST Passive Cancellation process (no need to throw an exception in this advanced use case)... + materializeDataContext + .DisableMaterializedLoadingTableCleanup() + .CancelMaterializationProcess(); + TestContext.WriteLine($"Now cancelling/bailing out the Materialization Process but leaving Loading Table to exist!"); + + return Task.CompletedTask; + }, + bulkHelpersConfig + ).ConfigureAwait(false); + } + + await using (var sqlConn = await sqlConnectionProvider.NewConnectionAsync().ConfigureAwait(false)) + { + var originalTableDef = await sqlConn.GetTableSchemaDefinitionAsync(detailLevel: TableSchemaDetailLevel.BasicDetails).ConfigureAwait(false); + var testTableDef = await sqlConn.GetTableSchemaDefinitionAsync(finalLoadingTableName, TableSchemaDetailLevel.BasicDetails, forceCacheReload: true).ConfigureAwait(false); + + Assert.IsNotNull(testTableDef); + Assert.AreEqual(testTableDef.TableFullyQualifiedName, finalLoadingTableName); + Assert.IsTrue(testTableDef.TableName.Contains(originalTableDef.TableName)); + Assert.AreEqual(originalTableDef.TableColumns.Count, testTableDef.TableColumns.Count); + Assert.AreEqual(originalTableDef.TableIndexes.Count, testTableDef.TableIndexes.Count); + + await using var sqlTransaction = (SqlTransaction)await sqlConn.BeginTransactionAsync().ConfigureAwait(false); + await sqlTransaction.DropTableAsync(testTableDef.TableFullyQualifiedName); + await sqlTransaction.CommitAsync().ConfigureAwait(false); + + timer.Stop(); + TestContext.WriteLine($"Materialization Test Completed/Finished in [{timer.ElapsedMilliseconds}] millis..."); + } + } + [TestMethod] public async Task TestMaterializeDataWithFKeyConstraintFailedValidationAsync() {