From 997d9d8d92c3ed9599cc3dcd7d635f83eb19c6db Mon Sep 17 00:00:00 2001 From: Keshav Taurah Date: Sat, 26 Sep 2020 20:26:06 -0700 Subject: [PATCH 1/3] yuniql init creates new folder _drop --- yuniql-cli/Properties/launchSettings.json | 2 +- yuniql-core/LocalVersionService.cs | 293 ++++++++++++---------- 2 files changed, 158 insertions(+), 137 deletions(-) diff --git a/yuniql-cli/Properties/launchSettings.json b/yuniql-cli/Properties/launchSettings.json index 442a669c..94e0c6c0 100644 --- a/yuniql-cli/Properties/launchSettings.json +++ b/yuniql-cli/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "Yuniql.CLI": { "commandName": "Project", - "commandLineArgs": "platforms", + "commandLineArgs": "init", "environmentVariables": { "Key": "Value" }, diff --git a/yuniql-core/LocalVersionService.cs b/yuniql-core/LocalVersionService.cs index 880d6c6a..994dcfe1 100644 --- a/yuniql-core/LocalVersionService.cs +++ b/yuniql-core/LocalVersionService.cs @@ -1,88 +1,104 @@ -using Yuniql.Extensibility; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using Yuniql.Extensibility; -namespace Yuniql.Core +namespace Yuniql.Core { - /// - /// Service responsible for initializing and managing the local workspace. A local workspace is a directory where yuniql operations are executed from. - /// When user calls yuniql-init, a directory structure is created in the target workspace directory. + /// + /// Service responsible for initializing and managing the local workspace. A local workspace is a directory where + /// yuniql operations are executed from. When user calls yuniql-init, a directory structure is created in the target + /// workspace directory. /// - public class LocalVersionService : ILocalVersionService - { + public class LocalVersionService : ILocalVersionService + { private readonly ITraceService _traceService; /// - public LocalVersionService(ITraceService traceService) - { - _traceService = traceService; - } + public LocalVersionService(ITraceService traceService) { _traceService = traceService; } + /// - public void Init(string workingPath) - { - string initDirectoryPath = Path.Combine(workingPath, "_init"); - if (!Directory.Exists(initDirectoryPath)) - { - Directory.CreateDirectory(initDirectoryPath); - File.AppendAllText(Path.Combine(initDirectoryPath, "README.md"), @"# The `_init` directory -Initialization scripts. Executed once. This is called the first time you do `yuniql run`."); - _traceService.Info($"Created script directory {initDirectoryPath}"); + public void Init(string workingPath) + { + string initDirectoryPath = Path.Combine(workingPath, "_init"); + if(!Directory.Exists(initDirectoryPath)) + { + Directory.CreateDirectory(initDirectoryPath); + File.AppendAllText(Path.Combine(initDirectoryPath, "README.md"), + @"# The `_init` directory +Initialization scripts. Executed once. This is called the first time you do `yuniql run`."); + _traceService.Info($"Created script directory {initDirectoryPath}"); } - string preDirectoryPath = Path.Combine(workingPath, "_pre"); - if (!Directory.Exists(preDirectoryPath)) - { - Directory.CreateDirectory(preDirectoryPath); - File.AppendAllText(Path.Combine(preDirectoryPath, "README.md"), @"# The `_pre` directory + string preDirectoryPath = Path.Combine(workingPath, "_pre"); + if(!Directory.Exists(preDirectoryPath)) + { + Directory.CreateDirectory(preDirectoryPath); + File.AppendAllText(Path.Combine(preDirectoryPath, "README.md"), + @"# The `_pre` directory Pre migration scripts. Executed every time before any version. -"); - _traceService.Info($"Created script directory {preDirectoryPath}"); +"); + _traceService.Info($"Created script directory {preDirectoryPath}"); } - string defaultVersionDirectoryPath = Path.Combine(workingPath, "v0.00"); - if (!Directory.Exists(defaultVersionDirectoryPath)) - { - Directory.CreateDirectory(defaultVersionDirectoryPath); - File.AppendAllText(Path.Combine(defaultVersionDirectoryPath, "README.md"), @"# The `v0.00` directory -Baseline scripts. Executed once. This is called when you do `yuniql run`."); - _traceService.Info($"Created script directory {defaultVersionDirectoryPath}"); + string defaultVersionDirectoryPath = Path.Combine(workingPath, "v0.00"); + if(!Directory.Exists(defaultVersionDirectoryPath)) + { + Directory.CreateDirectory(defaultVersionDirectoryPath); + File.AppendAllText(Path.Combine(defaultVersionDirectoryPath, "README.md"), + @"# The `v0.00` directory +Baseline scripts. Executed once. This is called when you do `yuniql run`."); + _traceService.Info($"Created script directory {defaultVersionDirectoryPath}"); } - string draftDirectoryPath = Path.Combine(workingPath, "_draft"); - if (!Directory.Exists(draftDirectoryPath)) - { - Directory.CreateDirectory(draftDirectoryPath); - File.AppendAllText(Path.Combine(draftDirectoryPath, "README.md"), @"# The `_draft` directory -Scripts in progress. Scripts that you are currently working and have not moved to specific version directory yet. Executed every time after the latest version."); - _traceService.Info($"Created script directory {draftDirectoryPath}"); + string draftDirectoryPath = Path.Combine(workingPath, "_draft"); + if(!Directory.Exists(draftDirectoryPath)) + { + Directory.CreateDirectory(draftDirectoryPath); + File.AppendAllText(Path.Combine(draftDirectoryPath, "README.md"), + @"# The `_draft` directory +Scripts in progress. Scripts that you are currently working and have not moved to specific version directory yet. Executed every time after the latest version."); + _traceService.Info($"Created script directory {draftDirectoryPath}"); } - string postDirectoryPath = Path.Combine(workingPath, "_post"); - if (!Directory.Exists(postDirectoryPath)) - { - Directory.CreateDirectory(postDirectoryPath); - File.AppendAllText(Path.Combine(postDirectoryPath, "README.md"), @"# The `_post` directory -Post migration scripts. Executed every time and always the last batch to run."); - _traceService.Info($"Created script directory {postDirectoryPath}"); + string postDirectoryPath = Path.Combine(workingPath, "_post"); + if(!Directory.Exists(postDirectoryPath)) + { + Directory.CreateDirectory(postDirectoryPath); + File.AppendAllText(Path.Combine(postDirectoryPath, "README.md"), + @"# The `_post` directory +Post migration scripts. Executed every time and always the last batch to run."); + _traceService.Info($"Created script directory {postDirectoryPath}"); } - string eraseDirectoryPath = Path.Combine(workingPath, "_erase"); - if (!Directory.Exists(eraseDirectoryPath)) - { - Directory.CreateDirectory(eraseDirectoryPath); - File.AppendAllText(Path.Combine(eraseDirectoryPath, "README.md"), @"# The `_erase` directory -Database cleanup scripts. Executed once only when you do `yuniql erase`."); - _traceService.Info($"Created script directory {eraseDirectoryPath}"); + string eraseDirectoryPath = Path.Combine(workingPath, "_erase"); + if(!Directory.Exists(eraseDirectoryPath)) + { + Directory.CreateDirectory(eraseDirectoryPath); + File.AppendAllText(Path.Combine(eraseDirectoryPath, "README.md"), + @"# The `_erase` directory +Database cleanup scripts. Executed once only when you do `yuniql erase`."); + _traceService.Info($"Created script directory {eraseDirectoryPath}"); + } + + string dropDirectoryPath = Path.Combine(workingPath, "_drop"); + if(!Directory.Exists(dropDirectoryPath)) + { + Directory.CreateDirectory(dropDirectoryPath); + File.AppendAllText(Path.Combine(dropDirectoryPath, "README.md"), + @"# The `_drop` directory +Drop database scripts. Executed once only when you do `yuniql drop --force`."); + _traceService.Info($"Created script directory {dropDirectoryPath}"); } - var readMeFile = Path.Combine(workingPath, "README.md"); - if (!File.Exists(readMeFile)) - { - File.AppendAllText(readMeFile, @" + var readMeFile = Path.Combine(workingPath, "README.md"); + if(!File.Exists(readMeFile)) + { + File.AppendAllText(readMeFile, + @" ## Yuniql-based Database Migration Project This database migration project is created and to be executed thru `yuniql`. @@ -111,122 +127,127 @@ For running with token replacement ## Found bugs? Help us improve further please [create an issue](https://github.com/rdagumampan/yuniql/issues/new). -"); - _traceService.Info($"Created file {readMeFile}"); +"); + _traceService.Info($"Created file {readMeFile}"); } - var dockerFile = Path.Combine(workingPath, "Dockerfile"); - if (!File.Exists(dockerFile)) - { - File.AppendAllText(dockerFile, @" + var dockerFile = Path.Combine(workingPath, "Dockerfile"); + if(!File.Exists(dockerFile)) + { + File.AppendAllText(dockerFile, + @" FROM rdagumampan/yuniql:latest COPY . ./db -"); - _traceService.Info($"Created file {dockerFile}"); +"); + _traceService.Info($"Created file {dockerFile}"); } - var gitIgnoreFile = Path.Combine(workingPath, ".gitignore"); - if (!File.Exists(gitIgnoreFile)) - { - File.AppendAllText(gitIgnoreFile, @" + var gitIgnoreFile = Path.Combine(workingPath, ".gitignore"); + if(!File.Exists(gitIgnoreFile)) + { + File.AppendAllText(gitIgnoreFile, + @" .plugins yuniql.exe yuniql.pdb yuniqlx.exe -"); - _traceService.Info($"Created file {gitIgnoreFile}"); +"); + _traceService.Info($"Created file {gitIgnoreFile}"); } - - } - - private List GetLocalVersions(string workingPath) - { - var localVersions = Directory.GetDirectories(workingPath, "v*.*") - .Select(x => new DirectoryInfo(x).Name) - .Select(x => - { - var r = new LocalVersion(x); - return r; - }) - .OrderBy(x => x.SemVersion) - .Reverse() + } + + private List GetLocalVersions(string workingPath) + { + var localVersions = Directory.GetDirectories(workingPath, "v*.*") + .Select(x => new DirectoryInfo(x).Name) + .Select(x => + { + var r = new LocalVersion(x); + return r; + }) + .OrderBy(x => x.SemVersion) + .Reverse() .ToList(); - return localVersions; + return localVersions; } /// - public string GetLatestVersion(string workingPath) - { - return GetLocalVersions(workingPath).First().SemVersion; - } + public string GetLatestVersion(string workingPath) { return GetLocalVersions(workingPath).First().SemVersion; } /// - public string IncrementMajorVersion(string workingPath, string sqlFileName) - { + public string IncrementMajorVersion(string workingPath, string sqlFileName) + { var localVersions = GetLocalVersions(workingPath); - var nextMajorVersion = new LocalVersion { Major = localVersions.First().Major + 1, Minor = 0 }; + var nextMajorVersion = new LocalVersion { Major = localVersions.First().Major + 1, Minor = 0 }; localVersions.Add(nextMajorVersion); - string nextVersionPath = Path.Combine(workingPath, nextMajorVersion.SemVersion); - Directory.CreateDirectory(nextVersionPath); + string nextVersionPath = Path.Combine(workingPath, nextMajorVersion.SemVersion); + Directory.CreateDirectory(nextVersionPath); _traceService.Info($"Created script directory {nextVersionPath}"); - if (!string.IsNullOrEmpty(sqlFileName)) - { - var sqlFilePath = Path.Combine(nextVersionPath, sqlFileName); - File.AppendAllText(sqlFilePath, @""); - _traceService.Info($"Created file {sqlFilePath}"); + if(!string.IsNullOrEmpty(sqlFileName)) + { + var sqlFilePath = Path.Combine(nextVersionPath, sqlFileName); + File.AppendAllText(sqlFilePath, string.Empty); + _traceService.Info($"Created file {sqlFilePath}"); } - return nextMajorVersion.SemVersion; + return nextMajorVersion.SemVersion; } /// - public string IncrementMinorVersion(string workingPath, string sqlFileName) - { + public string IncrementMinorVersion(string workingPath, string sqlFileName) + { var localVersions = GetLocalVersions(workingPath); - var nextMinorVersion = new LocalVersion { Major = localVersions.First().Major, Minor = localVersions.First().Minor + 1 }; + var nextMinorVersion = new LocalVersion + { Major = localVersions.First().Major, Minor = localVersions.First().Minor + 1 }; localVersions.Add(nextMinorVersion); - string nextVersionPath = Path.Combine(workingPath, nextMinorVersion.SemVersion); - Directory.CreateDirectory(nextVersionPath); + string nextVersionPath = Path.Combine(workingPath, nextMinorVersion.SemVersion); + Directory.CreateDirectory(nextVersionPath); _traceService.Info($"Created script directory {nextVersionPath}"); - if (!string.IsNullOrEmpty(sqlFileName)) - { - var sqlFilePath = Path.Combine(nextVersionPath, sqlFileName); - File.AppendAllText(sqlFilePath, @""); - _traceService.Info($"Created file {sqlFilePath}"); + if(!string.IsNullOrEmpty(sqlFileName)) + { + var sqlFilePath = Path.Combine(nextVersionPath, sqlFileName); + File.AppendAllText(sqlFilePath, string.Empty); + _traceService.Info($"Created file {sqlFilePath}"); } - return nextMinorVersion.SemVersion; + return nextMinorVersion.SemVersion; } /// - public void Validate(string workingPath) - { + public void Validate(string workingPath) + { string versionZeroDirectory = Directory.GetDirectories(workingPath, "v0.00*").FirstOrDefault(); - var directories = new List> { - new KeyValuePair(Path.Combine(workingPath, "_init"), Directory.Exists(Path.Combine(workingPath, "_init"))), - new KeyValuePair(Path.Combine(workingPath, "_pre"), Directory.Exists(Path.Combine(workingPath, "_pre"))), - new KeyValuePair(Path.Combine(workingPath, "v0.00*"), versionZeroDirectory != null), - new KeyValuePair(Path.Combine(workingPath, "_draft"), Directory.Exists(Path.Combine(workingPath, "_draft"))), - new KeyValuePair(Path.Combine(workingPath, "_post"), Directory.Exists(Path.Combine(workingPath, "_post"))), - new KeyValuePair(Path.Combine(workingPath, "_erase"), Directory.Exists(Path.Combine(workingPath, "_erase"))), + var directories = new List> + { + new KeyValuePair(Path.Combine(workingPath, "_init"), + Directory.Exists(Path.Combine(workingPath, "_init"))), + new KeyValuePair(Path.Combine(workingPath, "_pre"), + Directory.Exists(Path.Combine(workingPath, "_pre"))), + new KeyValuePair(Path.Combine(workingPath, "v0.00*"), versionZeroDirectory != null), + new KeyValuePair(Path.Combine(workingPath, "_draft"), + Directory.Exists(Path.Combine(workingPath, "_draft"))), + new KeyValuePair(Path.Combine(workingPath, "_post"), + Directory.Exists(Path.Combine(workingPath, "_post"))), + new KeyValuePair(Path.Combine(workingPath, "_erase"), + Directory.Exists(Path.Combine(workingPath, "_erase"))), }; - if (directories.Any(t => !t.Value)) - { - var message = new StringBuilder(); + if(directories.Any(t => !t.Value)) + { + var message = new StringBuilder(); directories.ForEach(t => message.AppendLine($"{t.Key} / {(t.Value ? "Found" : "Missing!")}")); - throw new YuniqlMigrationException($"At least one Yuniql directory is missing in your project. " + - $"See validation results below.{Environment.NewLine}{message.ToString()}"); - } - } - } + throw new YuniqlMigrationException($"At least one Yuniql directory is missing in your project. " + + $"See validation results below.{Environment.NewLine}{message.ToString()}"); + } + } + } } From 9235abcb7924d1f6faf4965d683230db252e7cf4 Mon Sep 17 00:00:00 2001 From: Keshav Taurah Date: Sat, 26 Sep 2020 21:42:05 -0700 Subject: [PATCH 2/3] added new method RunNonVersionDropScripts which do has the transaction parameter as drop database command cannot be used inside a transaction --- yuniql-cli/CommandLineService.cs | 53 ++- yuniql-cli/DropOption.cs | 24 ++ yuniql-cli/Program.cs | 4 +- yuniql-cli/Properties/launchSettings.json | 2 +- yuniql-core/IMigrationService.cs | 196 +++++----- yuniql-core/MigrationServiceBase.cs | 438 +++++++++++++--------- 6 files changed, 427 insertions(+), 290 deletions(-) create mode 100644 yuniql-cli/DropOption.cs diff --git a/yuniql-cli/CommandLineService.cs b/yuniql-cli/CommandLineService.cs index 3dc580b4..b92a86bb 100644 --- a/yuniql-cli/CommandLineService.cs +++ b/yuniql-cli/CommandLineService.cs @@ -300,16 +300,16 @@ public int RunEraseOption(EraseOption opts) .Select(t => new KeyValuePair(t.Split("=")[0], t.Split("=")[1])) .ToList(); - //run all erase scripts + //run all drop scripts var migrationService = _migrationServiceFactory.Create(opts.Platform); migrationService.Initialize(opts.ConnectionString, opts.CommandTimeout); - migrationService.Erase(opts.Path, tokens, opts.CommandTimeout, opts.Environment); + migrationService.Drop(opts.Path, tokens, opts.CommandTimeout, opts.Environment); - _traceService.Success($"Schema erase completed successfuly on {opts.Path}."); + _traceService.Success($"Database drop completed successfuly on {opts.Path}."); return 0; } catch(Exception ex) { - return OnException(ex, "Failed to execute erase function", opts.Debug, _traceService); + return OnException(ex, "Failed to execute drop function", opts.Debug, _traceService); } } @@ -380,6 +380,51 @@ public int RunPlatformsOption(PlatformsOption opts) } } + public int RunDropOption(DropOption opts) + { + try + { + //if no path provided, we default into current directory + if(string.IsNullOrEmpty(opts.Path)) + { + var workingPath = _environmentService.GetCurrentDirectory(); + opts.Path = workingPath; + } + + //if no connection string provided, we default into environment variable or throw exception + if(string.IsNullOrEmpty(opts.ConnectionString)) + { + opts.ConnectionString = _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_CONNECTION_STRING); + } + + //if no target platform provided, we default into sqlserver + if(string.IsNullOrEmpty(opts.Platform)) + { + opts.Platform = _environmentService.GetEnvironmentVariable(ENVIRONMENT_VARIABLE.YUNIQL_TARGET_PLATFORM); + if(string.IsNullOrEmpty(opts.Platform)) + { + opts.Platform = SUPPORTED_DATABASES.SQLSERVER; + } + } + + //parse tokens + var tokens = opts.Tokens + .Select(t => new KeyValuePair(t.Split("=")[0], t.Split("=")[1])) + .ToList(); + + //run all drop scripts + var migrationService = _migrationServiceFactory.Create(opts.Platform); + migrationService.Initialize(opts.ConnectionString, opts.CommandTimeout); + migrationService.Drop(opts.Path, tokens, opts.CommandTimeout, opts.Environment); + + _traceService.Success($"Database drop completed successfuly on {opts.Path}."); + return 0; + } catch(Exception ex) + { + return OnException(ex, "Failed to execute drop function", opts.Debug, _traceService); + } + } + private int OnException(Exception exception, string headerMessage, bool debug, ITraceService traceService) { var userMessage = debug ? exception.ToString() : $"{exception.Message} {exception.InnerException?.Message}"; diff --git a/yuniql-cli/DropOption.cs b/yuniql-cli/DropOption.cs new file mode 100644 index 00000000..74ee2fe7 --- /dev/null +++ b/yuniql-cli/DropOption.cs @@ -0,0 +1,24 @@ +using CommandLine; +using System; +using System.Collections.Generic; + +namespace Yuniql.CLI +{ + //yuniql info + [Verb("drop", HelpText = "Drop database.")] + public class DropOption : BasePlatformOption + { + //yuniql erase -k "Token1=TokenValue1" -k "Token2=TokenValue2" -k "Token3=TokenValue3" | --token "..." --token "..." --token "..." + //yuniql erase -k "Token1=TokenValue1,Token2=TokenValue2,Token3=TokenValue3" | --token "...,...,..." + [Option('k', "token", Required = false, HelpText = "Replace tokens using the passed key-value pairs.", Separator = ',')] + public IEnumerable Tokens { get; set; } = new List(); + + //yuniql erase --force + [Option('f', "force", Required = true, HelpText = "Force execution of drop commands.")] + public bool Force { get; set; } + + //yuniql --environment "DEV" | --environment "PROD" + [Option("environment", Required = false, HelpText = "Environment code for environment-aware scripts.")] + public string Environment { get; set; } + } +} diff --git a/yuniql-cli/Program.cs b/yuniql-cli/Program.cs index 7ce5da3f..7bbbbaed 100644 --- a/yuniql-cli/Program.cs +++ b/yuniql-cli/Program.cs @@ -24,7 +24,7 @@ public static int Main(string[] args) traceService); var resultCode = Parser.Default - .ParseArguments(args) + .ParseArguments(args) .MapResult((InitOption opts) => Dispatch(commandLineService.RunInitOption, opts, traceService), (RunOption opts) => Dispatch(commandLineService.RunMigration, opts, traceService), (NextVersionOption opts) => Dispatch(commandLineService.IncrementVersion, opts, traceService), @@ -34,8 +34,8 @@ public static int Main(string[] args) (BaselineOption opts) => Dispatch(commandLineService.RunBaselineOption, opts, traceService), (RebaseOption opts) => Dispatch(commandLineService.RunRebaseOption, opts, traceService), (ArchiveOption opts) => Dispatch(commandLineService.RunArchiveOption, opts, traceService), + (DropOption opts) => Dispatch(commandLineService.RunDropOption, opts, traceService), (PlatformsOption opts) => Dispatch(commandLineService.RunPlatformsOption, opts, traceService), - errs => 1); return resultCode; diff --git a/yuniql-cli/Properties/launchSettings.json b/yuniql-cli/Properties/launchSettings.json index 94e0c6c0..cc40df3d 100644 --- a/yuniql-cli/Properties/launchSettings.json +++ b/yuniql-cli/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "Yuniql.CLI": { "commandName": "Project", - "commandLineArgs": "init", + "commandLineArgs": "drop --force", "environmentVariables": { "Key": "Value" }, diff --git a/yuniql-core/IMigrationService.cs b/yuniql-core/IMigrationService.cs index efbdd157..eacf8e0d 100644 --- a/yuniql-core/IMigrationService.cs +++ b/yuniql-core/IMigrationService.cs @@ -1,42 +1,45 @@ -using Yuniql.Extensibility; -using System.Collections.Generic; +using System.Collections.Generic; using System.Data; +using Yuniql.Extensibility; -namespace Yuniql.Core +namespace Yuniql.Core { - /// - /// Runs migrations by executing alls scripts in the workspace directory. + /// + /// Runs migrations by executing alls scripts in the workspace directory. /// - public interface IMigrationService + public interface IMigrationService { - /// - /// Initializes the current instance of + /// + /// Initializes the current instance of /// /// Connection string to target database server or instance. /// Command timeout in seconds. void Initialize(string connectionString, int? commandTimeout = null); - /// - /// Returns the current migration version applied in target database. + /// + /// Returns the current migration version applied in target database. /// string GetCurrentVersion(string schemaName = null, string tableName = null); - /// - /// Returns all migration versions applied in the target database + /// + /// Returns all migration versions applied in the target database /// List GetAllVersions(string schemaName = null, string tableName = null); - /// - /// Runs migrations by executing alls scripts in the workspace directory. - /// When CSV files are present also run bulk import operations to target database table having same file name. + /// + /// Runs migrations by executing alls scripts in the workspace directory. When CSV files are present also run bulk + /// import operations to target database table having same file name. /// /// The directory path to migration project. /// The maximum version to run to. When NULL, runs migration to the latest version found in the workspace path. /// When TRUE, creates the database in the target host. /// Token kev/value pairs to replace tokens in script files. - /// When TRUE, runs the migration in uncommitted mode. No changes are committed to target database. When NULL, runs migration in atomic mode. + /// + /// When TRUE, runs the migration in uncommitted mode. No changes are committed to target database. When NULL, runs + /// migration in atomic mode. + /// /// Bulk file values separator character in the CSV bulk import files. When NULL, uses comma. - /// Schema name for schema versions table. When empty, uses the default schema in the target data platform. + /// Schema name for schema versions table. When empty, uses the default schema in the target data platform. /// Table name for schema versions table. When empty, uses __yuniqldbversion. /// Command timeout in seconds. When NULL, it uses default provider command timeout. /// Batch rows to processed when performing bulk import. When NULL, it uses default provider batch size. @@ -45,94 +48,87 @@ public interface IMigrationService /// Environment code for environment-aware scripts. /// The resume from failure. /// When TRUE, migration will run without using transactions - void Run( - string workingPath, - string targetVersion = null, - bool? autoCreateDatabase = null, - List> tokens = null, - bool? verifyOnly = null, - string bulkSeparator = null, - string metaSchemaName = null, - string metaTableName = null, - int? commandTimeout = null, - int? bulkBatchSize = null, - string appliedByTool = null, - string appliedByToolVersion = null, - string environmentCode = null, - NonTransactionalResolvingOption? resumeFromFailure = null, - bool noTransaction = false - ); + void Run(string workingPath, + string targetVersion = null, + bool? autoCreateDatabase = null, + List> tokens = null, + bool? verifyOnly = null, + string bulkSeparator = null, + string metaSchemaName = null, + string metaTableName = null, + int? commandTimeout = null, + int? bulkBatchSize = null, + string appliedByTool = null, + string appliedByToolVersion = null, + string environmentCode = null, + NonTransactionalResolvingOption? resumeFromFailure = null, + bool noTransaction = false); - /// - /// Executes erase scripts presentin _erase directory and subdirectories. + /// + /// Executes erase scripts presentin _erase directory and subdirectories. /// /// The directory path to migration project. /// Token kev/value pairs to replace tokens in script files. /// Command timeout in seconds. /// Environment code for environment-aware scripts. - bool IsTargetDatabaseLatest(string targetVersion, string schemaName = null, string tableName = null); - - void RunNonVersionScripts( - IDbConnection connection, - IDbTransaction transaction, - string workingPath, - List> tokenKeyPairs = null, - string bulkSeparator = null, - int? commandTimeout = null, - string environmentCode = null - ); - - void RunVersionScripts( - IDbConnection connection, - IDbTransaction transaction, - List dbVersions, - string workingPath, - string targetVersion, - NonTransactionalContext nonTransactionalContext, - List> tokenKeyPairs = null, - string bulkSeparator = null, - string metaSchemaName = null, - string metaTableName = null, - int? commandTimeout = null, - int? bulkBatchSize = null, - string appliedByTool = null, - string appliedByToolVersion = null, - string environmentCode = null - ); - - void RunBulkImport( - IDbConnection connection, - IDbTransaction transaction, - string workingPath, - string scriptDirectory, - string bulkSeparator = null, - int? bulkBatchSize = null, - int? commandTimeout = null, - string environmentCode = null - ); - - void RunSqlScripts( - IDbConnection connection, - IDbTransaction transaction, - NonTransactionalContext nonTransactionalContext, - string version, - string workingPath, - string scriptDirectory, - string metaSchemaName, - string metaTableName, - List> tokenKeyPairs = null, - int? commandTimeout = null, - string environmentCode = null, - string appliedByTool = null, - string appliedByToolVersion = null - ); - - void Erase( - string workingPath, - List> tokens = null, - int? commandTimeout = null, - string environmentCode = null - ); - } + bool IsTargetDatabaseLatest(string targetVersion, string schemaName = null, string tableName = null); + + void RunNonVersionScripts(IDbConnection connection, + IDbTransaction transaction, + string workingPath, + List> tokenKeyPairs = null, + string bulkSeparator = null, + int? commandTimeout = null, + string environmentCode = null); + + void RunVersionScripts(IDbConnection connection, + IDbTransaction transaction, + List dbVersions, + string workingPath, + string targetVersion, + NonTransactionalContext nonTransactionalContext, + List> tokenKeyPairs = null, + string bulkSeparator = null, + string metaSchemaName = null, + string metaTableName = null, + int? commandTimeout = null, + int? bulkBatchSize = null, + string appliedByTool = null, + string appliedByToolVersion = null, + string environmentCode = null); + + void RunBulkImport(IDbConnection connection, + IDbTransaction transaction, + string workingPath, + string scriptDirectory, + string bulkSeparator = null, + int? bulkBatchSize = null, + int? commandTimeout = null, + string environmentCode = null); + + void RunSqlScripts(IDbConnection connection, + IDbTransaction transaction, + NonTransactionalContext nonTransactionalContext, + string version, + string workingPath, + string scriptDirectory, + string metaSchemaName, + string metaTableName, + List> tokenKeyPairs = null, + int? commandTimeout = null, + string environmentCode = null, + string appliedByTool = null, + string appliedByToolVersion = null); + + void Erase(string workingPath, + List> tokens = null, + int? commandTimeout = null, + string environmentCode = null); + + void Drop(string workingPath, + List> tokens = null, + int? commandTimeout = null, + string environmentCode = null); + } } \ No newline at end of file diff --git a/yuniql-core/MigrationServiceBase.cs b/yuniql-core/MigrationServiceBase.cs index 0e4ba053..66d8ac6c 100644 --- a/yuniql-core/MigrationServiceBase.cs +++ b/yuniql-core/MigrationServiceBase.cs @@ -1,244 +1,316 @@ -using Yuniql.Extensibility; -using System; -using System.Collections.Generic; -using System.Data; -using System.Linq; -using System.IO; - -namespace Yuniql.Core +using System; +using System.Collections.Generic; +using System.Data; +using System.IO; +using System.Linq; +using Yuniql.Extensibility; + +namespace Yuniql.Core { /// - public abstract class MigrationServiceBase : IMigrationService - { - private readonly ILocalVersionService _localVersionService; - private readonly IDataService _dataService; - private readonly IBulkImportService _bulkImportService; - private readonly ITokenReplacementService _tokenReplacementService; - private readonly IDirectoryService _directoryService; - private readonly IFileService _fileService; - private readonly ITraceService _traceService; + public abstract class MigrationServiceBase : IMigrationService + { + private readonly ILocalVersionService _localVersionService; + private readonly IDataService _dataService; + private readonly IBulkImportService _bulkImportService; + private readonly ITokenReplacementService _tokenReplacementService; + private readonly IDirectoryService _directoryService; + private readonly IFileService _fileService; + private readonly ITraceService _traceService; private readonly IConfigurationDataService _configurationDataService; /// - public MigrationServiceBase( - ILocalVersionService localVersionService, - IDataService dataService, - IBulkImportService bulkImportService, - IConfigurationDataService configurationDataService, - ITokenReplacementService tokenReplacementService, - IDirectoryService directoryService, - IFileService fileService, - ITraceService traceService) - { - this._localVersionService = localVersionService; - this._dataService = dataService; - this._bulkImportService = bulkImportService; - this._tokenReplacementService = tokenReplacementService; - this._directoryService = directoryService; - this._fileService = fileService; - this._traceService = traceService; - this._configurationDataService = configurationDataService; + public MigrationServiceBase(ILocalVersionService localVersionService, + IDataService dataService, + IBulkImportService bulkImportService, + IConfigurationDataService configurationDataService, + ITokenReplacementService tokenReplacementService, + IDirectoryService directoryService, + IFileService fileService, + ITraceService traceService) + { + this._localVersionService = localVersionService; + this._dataService = dataService; + this._bulkImportService = bulkImportService; + this._tokenReplacementService = tokenReplacementService; + this._directoryService = directoryService; + this._fileService = fileService; + this._traceService = traceService; + this._configurationDataService = configurationDataService; } - /// - public virtual void Initialize( - string connectionString, - int? commandTimeout = null) + /// + public virtual void Initialize(string connectionString, int? commandTimeout = null) { - //initialize dependencies - _dataService.Initialize(connectionString); - _bulkImportService.Initialize(connectionString); + //initialize dependencies + _dataService.Initialize(connectionString); + _bulkImportService.Initialize(connectionString); } - /// - public virtual string GetCurrentVersion(string metaSchemaName = null, string metaTableName = null) - { - return _configurationDataService.GetCurrentVersion(metaSchemaName, metaTableName); - } + /// + public virtual string GetCurrentVersion(string metaSchemaName = null, string metaTableName = null) + { return _configurationDataService.GetCurrentVersion(metaSchemaName, metaTableName); } - /// + /// //TODO: Move this to MigrationServiceBase - public virtual List GetAllVersions(string metaSchemaName = null, string metaTableName = null) - { - return _configurationDataService.GetAllAppliedVersions(metaSchemaName, metaTableName); - } + public virtual List GetAllVersions(string metaSchemaName = null, string metaTableName = null) + { return _configurationDataService.GetAllAppliedVersions(metaSchemaName, metaTableName); } - /// - public abstract void Run( - string workingPath, - string targetVersion = null, - bool? autoCreateDatabase = false, - List> tokenKeyPairs = null, - bool? verifyOnly = false, - string bulkSeparator = null, - string metaSchemaName = null, - string metaTableName = null, - int? commandTimeout = null, - int? bulkBatchSize = null, - string appliedByTool = null, - string appliedByToolVersion = null, - string environmentCode = null, - NonTransactionalResolvingOption? nonTransactionalResolvingOption = null, - bool noTransaction = false - ); + /// + public abstract void Run(string workingPath, + string targetVersion = null, + bool? autoCreateDatabase = false, + List> tokenKeyPairs = null, + bool? verifyOnly = false, + string bulkSeparator = null, + string metaSchemaName = null, + string metaTableName = null, + int? commandTimeout = null, + int? bulkBatchSize = null, + string appliedByTool = null, + string appliedByToolVersion = null, + string environmentCode = null, + NonTransactionalResolvingOption? nonTransactionalResolvingOption = null, + bool noTransaction = false); - /// - public virtual bool IsTargetDatabaseLatest(string targetVersion, string metaSchemaName = null, string metaTableName = null) + /// + public virtual bool IsTargetDatabaseLatest(string targetVersion, + string metaSchemaName = null, + string metaTableName = null) { - //get the current version stored in database - var remoteCurrentVersion = _configurationDataService.GetCurrentVersion(metaSchemaName, metaTableName); - if (string.IsNullOrEmpty(remoteCurrentVersion)) return false; + //get the current version stored in database + var remoteCurrentVersion = _configurationDataService.GetCurrentVersion(metaSchemaName, metaTableName); + if(string.IsNullOrEmpty(remoteCurrentVersion)) + return false; //compare version applied in db vs versions available locally - var localCurrentVersion = new LocalVersion(remoteCurrentVersion); - var localTargetVersion = new LocalVersion(targetVersion); - return string.Compare(localCurrentVersion.SemVersion, localTargetVersion.SemVersion) == 1 || //db has more updated than local version - string.Compare(localCurrentVersion.SemVersion, localTargetVersion.SemVersion) == 0; //db has the same version as local version + var localCurrentVersion = new LocalVersion(remoteCurrentVersion); + var localTargetVersion = new LocalVersion(targetVersion); + return string.Compare(localCurrentVersion.SemVersion, localTargetVersion.SemVersion) == 1 || //db has more updated than local version + string.Compare(localCurrentVersion.SemVersion, localTargetVersion.SemVersion) == 0; //db has the same version as local version } - /// - public virtual void RunNonVersionScripts( - IDbConnection connection, - IDbTransaction transaction, - string workingPath, - List> tokenKeyPairs = null, - string bulkSeparator = null, - int? commandTimeout = null, - string environmentCode = null - ) + /// + public virtual void RunNonVersionScripts(IDbConnection connection, + IDbTransaction transaction, + string workingPath, + List> tokenKeyPairs = null, + string bulkSeparator = null, + int? commandTimeout = null, + string environmentCode = null) { - //extract and filter out scripts when environment code is used - var sqlScriptFiles = _directoryService.GetAllFiles(workingPath, "*.sql").ToList(); - sqlScriptFiles = _directoryService.FilterFiles(workingPath, environmentCode, sqlScriptFiles).ToList(); - _traceService.Info($"Found {sqlScriptFiles.Count} script files on {workingPath}" + (sqlScriptFiles.Count > 0 ? Environment.NewLine: string.Empty) + - $"{string.Join(Environment.NewLine, sqlScriptFiles.Select(s => " + " + new FileInfo(s).Name))}"); + //extract and filter out scripts when environment code is used + var sqlScriptFiles = _directoryService.GetAllFiles(workingPath, "*.sql").ToList(); + sqlScriptFiles = _directoryService.FilterFiles(workingPath, environmentCode, sqlScriptFiles).ToList(); + _traceService.Info($"Found {sqlScriptFiles.Count} script files on {workingPath}" + + (sqlScriptFiles.Count > 0 ? Environment.NewLine : string.Empty) + + $"{string.Join(Environment.NewLine, sqlScriptFiles.Select(s => " + " + new FileInfo(s).Name))}"); //execute all script files in the target folder - sqlScriptFiles.Sort(); - sqlScriptFiles.ForEach(scriptFile => - { - var sqlStatementRaw = _fileService.ReadAllText(scriptFile); - var sqlStatements = _dataService.BreakStatements(sqlStatementRaw) - .Where(s => !string.IsNullOrWhiteSpace(s)) + sqlScriptFiles.Sort(); + sqlScriptFiles.ForEach(scriptFile => + { + var sqlStatementRaw = _fileService.ReadAllText(scriptFile); + var sqlStatements = _dataService.BreakStatements(sqlStatementRaw) + .Where(s => !string.IsNullOrWhiteSpace(s)) .ToList(); - sqlStatements.ForEach(sqlStatement => - { + sqlStatements.ForEach(sqlStatement => + { try { sqlStatement = _tokenReplacementService.Replace(tokenKeyPairs, sqlStatement); _traceService.Debug($"Executing sql statement as part of : {scriptFile}"); - _configurationDataService.ExecuteSql( - connection: connection, - commandText: sqlStatement, - transaction: transaction, - commandTimeout: commandTimeout, - traceService: _traceService); - } - catch (Exception) + _configurationDataService.ExecuteSql(connection: connection, + commandText: sqlStatement, + transaction: transaction, + commandTimeout: commandTimeout, + traceService: _traceService); + } catch(Exception) { _traceService.Error($"Failed to execute sql statements in script file {scriptFile}.{Environment.NewLine}" + $"The failing statement starts here --------------------------{Environment.NewLine}" + $"{sqlStatement} {Environment.NewLine}" + $"The failing statement ends here --------------------------"); throw; - } + } }); - _traceService.Info($"Executed script file {scriptFile}."); - }); - } + _traceService.Info($"Executed script file {scriptFile}."); + }); + } + + public virtual void RunNonVersionDropScripts(IDbConnection connection, + string workingPath, + List> tokenKeyPairs = null, + string bulkSeparator = null, + int? commandTimeout = null, + string environmentCode = null) + { + //extract and filter out scripts when environment code is used + var sqlScriptFiles = _directoryService.GetAllFiles(workingPath, "*.sql").ToList(); + sqlScriptFiles = _directoryService.FilterFiles(workingPath, environmentCode, sqlScriptFiles).ToList(); + _traceService.Info($"Found {sqlScriptFiles.Count} script files on {workingPath}" + + (sqlScriptFiles.Count > 0 ? Environment.NewLine : string.Empty) + + $"{string.Join(Environment.NewLine, sqlScriptFiles.Select(s => " + " + new FileInfo(s).Name))}"); + + //execute all script files in the target folder + sqlScriptFiles.Sort(); + sqlScriptFiles.ForEach(scriptFile => + { + var sqlStatementRaw = _fileService.ReadAllText(scriptFile); + var sqlStatements = _dataService.BreakStatements(sqlStatementRaw) + .Where(s => !string.IsNullOrWhiteSpace(s)) + .ToList(); - public abstract void RunVersionScripts( - IDbConnection connection, - IDbTransaction transaction, - List dbVersions, - string workingPath, - string targetVersion, - NonTransactionalContext nonTransactionalContext, - List> tokenKeyPairs = null, - string bulkSeparator = null, - string metaschemaName = null, - string metaTableName = null, - int? commandTimeout = null, - int? bulkBatchSize = null, - string appliedByTool = null, - string appliedByToolVersion = null, - string environmentCode = null - ); + sqlStatements.ForEach(sqlStatement => + { + try + { + sqlStatement = _tokenReplacementService.Replace(tokenKeyPairs, sqlStatement); + _traceService.Debug($"Executing sql statement as part of : {scriptFile}"); + + _configurationDataService.ExecuteSql(connection: connection, + commandText: sqlStatement, + commandTimeout: commandTimeout, + traceService: _traceService); + } catch(Exception) + { + _traceService.Error($"Failed to execute sql statements in script file {scriptFile}.{Environment.NewLine}" + + $"The failing statement starts here --------------------------{Environment.NewLine}" + + $"{sqlStatement} {Environment.NewLine}" + + $"The failing statement ends here --------------------------"); + throw; + } + }); - public virtual void RunBulkImport( - IDbConnection connection, - IDbTransaction transaction, - string workingPath, - string scriptDirectory, - string bulkSeparator = null, - int? bulkBatchSize = null, - int? commandTimeout = null, - string environmentCode = null - ) + _traceService.Info($"Executed script file {scriptFile}."); + }); + } + + public abstract void RunVersionScripts(IDbConnection connection, + IDbTransaction transaction, + List dbVersions, + string workingPath, + string targetVersion, + NonTransactionalContext nonTransactionalContext, + List> tokenKeyPairs = null, + string bulkSeparator = null, + string metaschemaName = null, + string metaTableName = null, + int? commandTimeout = null, + int? bulkBatchSize = null, + string appliedByTool = null, + string appliedByToolVersion = null, + string environmentCode = null); + + public virtual void RunBulkImport(IDbConnection connection, + IDbTransaction transaction, + string workingPath, + string scriptDirectory, + string bulkSeparator = null, + int? bulkBatchSize = null, + int? commandTimeout = null, + string environmentCode = null) { - //extract and filter out scripts when environment code is used - var bulkFiles = _directoryService.GetFiles(scriptDirectory, "*.csv").ToList(); + //extract and filter out scripts when environment code is used + var bulkFiles = _directoryService.GetFiles(scriptDirectory, "*.csv").ToList(); bulkFiles = _directoryService.FilterFiles(workingPath, environmentCode, bulkFiles).ToList(); - _traceService.Info($"Found {bulkFiles.Count} script files on {scriptDirectory}" + (bulkFiles.Count > 0 ? Environment.NewLine : string.Empty) + - $"{string.Join(Environment.NewLine, bulkFiles.Select(s => " + " + new FileInfo(s).Name))}"); - bulkFiles.Sort(); - bulkFiles.ForEach(csvFile => - { - _bulkImportService.Run(connection, transaction, csvFile, bulkSeparator, bulkBatchSize: bulkBatchSize, commandTimeout: commandTimeout); - _traceService.Info($"Imported bulk file {csvFile}."); - }); - } - - public abstract void RunSqlScripts( - IDbConnection connection, - IDbTransaction transaction, - NonTransactionalContext nonTransactionalContext, - string version, - string workingPath, - string scriptDirectory, - string metaSchemaName, - string metaTableName, - List> tokenKeyPairs = null, - int? commandTimeout = null, - string environmentCode = null, - string appliedByTool = null, - string appliedByToolVersion = null - ); - - /// - public virtual void Erase( - string workingPath, - List> tokenKeyPairs = null, - int? commandTimeout = null, - string environmentCode = null - ) + _traceService.Info($"Found {bulkFiles.Count} script files on {scriptDirectory}" + + (bulkFiles.Count > 0 ? Environment.NewLine : string.Empty) + + $"{string.Join(Environment.NewLine, bulkFiles.Select(s => " + " + new FileInfo(s).Name))}"); + bulkFiles.Sort(); + bulkFiles.ForEach(csvFile => + { + _bulkImportService.Run(connection, + transaction, + csvFile, + bulkSeparator, + bulkBatchSize: bulkBatchSize, + commandTimeout: commandTimeout); + _traceService.Info($"Imported bulk file {csvFile}."); + }); + } + + public abstract void RunSqlScripts(IDbConnection connection, + IDbTransaction transaction, + NonTransactionalContext nonTransactionalContext, + string version, + string workingPath, + string scriptDirectory, + string metaSchemaName, + string metaTableName, + List> tokenKeyPairs = null, + int? commandTimeout = null, + string environmentCode = null, + string appliedByTool = null, + string appliedByToolVersion = null); + + /// + public virtual void Erase(string workingPath, + List> tokenKeyPairs = null, + int? commandTimeout = null, + string environmentCode = null) { //create a shared open connection to entire migration run - using (var connection = _dataService.CreateConnection()) + using(var connection = _dataService.CreateConnection()) { connection.KeepOpen(); //enclose all executions in a single transaction in case platform supports it - using (var transaction = connection.BeginTransaction()) + using(var transaction = connection.BeginTransaction()) { try { //runs all scripts in the _erase folder - RunNonVersionScripts(connection, transaction, Path.Combine(workingPath, "_erase"), tokenKeyPairs: tokenKeyPairs, bulkSeparator: DEFAULT_CONSTANTS.BULK_SEPARATOR, commandTimeout: commandTimeout, environmentCode: environmentCode); + RunNonVersionScripts(connection, + transaction, + Path.Combine(workingPath, "_erase"), + tokenKeyPairs: tokenKeyPairs, + bulkSeparator: DEFAULT_CONSTANTS.BULK_SEPARATOR, + commandTimeout: commandTimeout, + environmentCode: environmentCode); _traceService.Info($"Executed script files on {Path.Combine(workingPath, "_erase")}"); transaction.Commit(); - } - catch (Exception) + } catch(Exception) { transaction.Rollback(); throw; } } } - } - } + } + + + public virtual void Drop(string workingPath, + List> tokenKeyPairs = null, + int? commandTimeout = null, + string environmentCode = null) + { + //create a shared open connection to entire migration run + using(var connection = _dataService.CreateConnection()) + { + connection.KeepOpen(); + + //enclose all executions in a single transaction in case platform supports it + + try + { + //runs all scripts in the _drop folder + RunNonVersionDropScripts(connection, + Path.Combine(workingPath, "_drop"), + tokenKeyPairs: tokenKeyPairs, + bulkSeparator: DEFAULT_CONSTANTS.BULK_SEPARATOR, + commandTimeout: commandTimeout, + environmentCode: environmentCode); + _traceService.Info($"Executed script files on {Path.Combine(workingPath, "_drop")}"); + } catch(Exception) + { + throw; + } + } + } + } } From a80f5c68b37a6c434e13d2cdc07ac072e8a1dea5 Mon Sep 17 00:00:00 2001 From: Keshav Taurah Date: Sat, 26 Sep 2020 22:02:30 -0700 Subject: [PATCH 3/3] Solve bug in RunEraseOption method --- yuniql-cli/CommandLineService.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/yuniql-cli/CommandLineService.cs b/yuniql-cli/CommandLineService.cs index b92a86bb..50194ca1 100644 --- a/yuniql-cli/CommandLineService.cs +++ b/yuniql-cli/CommandLineService.cs @@ -303,13 +303,13 @@ public int RunEraseOption(EraseOption opts) //run all drop scripts var migrationService = _migrationServiceFactory.Create(opts.Platform); migrationService.Initialize(opts.ConnectionString, opts.CommandTimeout); - migrationService.Drop(opts.Path, tokens, opts.CommandTimeout, opts.Environment); - - _traceService.Success($"Database drop completed successfuly on {opts.Path}."); + migrationService.Erase(opts.Path, tokens, opts.CommandTimeout, opts.Environment); + + _traceService.Success($"Schema erase completed successfuly on {opts.Path}."); return 0; } catch(Exception ex) { - return OnException(ex, "Failed to execute drop function", opts.Debug, _traceService); + return OnException(ex, "Failed to execute erase function", opts.Debug, _traceService); } }