diff --git a/azure-pipelines.yml b/azure-pipelines.yml index bb48cb6..351d196 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -166,5 +166,5 @@ steps: action: 'create' target: '$(Build.SourceVersion)' tagSource: 'gitTag' - title: '$(SemVersion) Release of Enbrea Cli' + title: '$(SemVersion) Release of Enbrea CLI' addChangeLog: false diff --git a/src/Enbrea.Cli.DaVinci/Export/ExportManager.cs b/src/Enbrea.Cli.DaVinci/Export/ExportManager.cs index 490437e..ad88bf3 100644 --- a/src/Enbrea.Cli.DaVinci/Export/ExportManager.cs +++ b/src/Enbrea.Cli.DaVinci/Export/ExportManager.cs @@ -47,14 +47,14 @@ public async override Task Execute() // Create command for DAVINCI CONSOLE var cmd = _config.DataProvider == DataProvider.File ? CliWrap.Cli.Wrap(ConsoleUtils.GetConsolePath()) - .WithArguments(new[] { + .WithArguments([ "export", "-minimal-display", "-s", "file", "-o", GetEcfFolderName(), "-fn", _config.DataFile - }) : + ]) : CliWrap.Cli.Wrap(ConsoleUtils.GetConsolePath()) - .WithArguments(new[] { + .WithArguments([ "export", "-minimal-display", "-s", "server", "-o", GetEcfFolderName(), @@ -63,7 +63,7 @@ public async override Task Execute() "-un", _config.ServerUserName, "-up", _config.ServerPassword, "-sf", $"{{{_config.ServerFileId}}}" - }); + ]); // Excecute DAVINCI CONSOLE await foreach (var cmdEvent in cmd.ListenAsync(_cancellationToken)) diff --git a/src/Enbrea.Cli.DaVinci/Import/ImportManager.cs b/src/Enbrea.Cli.DaVinci/Import/ImportManager.cs index 6ee3bac..7a43570 100644 --- a/src/Enbrea.Cli.DaVinci/Import/ImportManager.cs +++ b/src/Enbrea.Cli.DaVinci/Import/ImportManager.cs @@ -47,14 +47,14 @@ public async override Task Execute() // Create command for DAVINCI CONSOLE var cmd = _config.DataProvider == DataProvider.File ? CliWrap.Cli.Wrap(ConsoleUtils.GetConsolePath()) - .WithArguments(new[] { + .WithArguments([ "import", "-minimal-display", "-s", "file", "-i", GetEcfFolderName(), "-fn", _config.DataFile - }) : + ]) : CliWrap.Cli.Wrap(ConsoleUtils.GetConsolePath()) - .WithArguments(new[] { + .WithArguments([ "import", "-minimal-display", "-s", "server", "-i", GetEcfFolderName(), @@ -63,7 +63,7 @@ public async override Task Execute() "-un", _config.ServerUserName, "-up", _config.ServerPassword, "-sf", $"{{{_config.ServerFileId}}}" - }); + ]); // Excecute DAVINCI CONSOLE await foreach (var cmdEvent in cmd.ListenAsync(_cancellationToken)) diff --git a/src/Enbrea.Cli.Untis/Utils/ConsoleUtils.cs b/src/Enbrea.Cli.Untis/Utils/ConsoleUtils.cs index 938af01..eeec6ed 100644 --- a/src/Enbrea.Cli.Untis/Utils/ConsoleUtils.cs +++ b/src/Enbrea.Cli.Untis/Utils/ConsoleUtils.cs @@ -53,12 +53,12 @@ static public async Task RunUntisBackup( // Execute Untis var result = await CliWrap.Cli.Wrap(GetUntisPath()) - .WithArguments(new[] { + .WithArguments([ $"DB~{schoolNo}~{schoolYear}~{version}", $"/backup={outputFile}", $"/user={userName}", $"/pwd={password}" - }) + ]) .WithValidation(CommandResultValidation.None) .ExecuteAsync(cancellationToken); @@ -148,10 +148,10 @@ static public async Task RunUntisSingleGpuExport( // Execute Untis var result = await CliWrap.Cli.Wrap(GetUntisPath()) - .WithArguments(new[] { + .WithArguments([ $"{untisFile}", $"/exp{outputType}={outputFile}" - }) + ]) .WithValidation(CommandResultValidation.None) .ExecuteAsync(cancellationToken); @@ -193,10 +193,10 @@ static public async Task RunUntisXmlExport( // Execute Untis var result = await CliWrap.Cli.Wrap(GetUntisPath()) - .WithArguments(new[] { + .WithArguments([ $"{untisFile}", $"/xml={outputFile}" - }) + ]) .WithValidation(CommandResultValidation.None) .ExecuteAsync(cancellationToken); diff --git a/src/Enbrea.Cli/CommandDefinitions.cs b/src/Enbrea.Cli/CommandDefinitions.cs index 0de3da1..e77c7bb 100644 --- a/src/Enbrea.Cli/CommandDefinitions.cs +++ b/src/Enbrea.Cli/CommandDefinitions.cs @@ -31,11 +31,11 @@ public static Command Backup() { var command = new Command("backup-offline", "Creates and downloads an Enbrea database backup for offline use") { - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true }, - new Option(new[] { "--out", "-o" }, "Path to output file") + new Option(["--out", "-o"], "Path to output file") { IsRequired = true } @@ -53,15 +53,15 @@ public static Command CreateExportTask() { var command = new Command("create-export-task", "Schedules a task for a data export from an Enbrea instance") { - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true }, - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true }, - new Option(new[] { "--interval", "-i" }, delegate() { return 10; }, "Time interval in minutes") + new Option(["--interval", "-i"], delegate() { return 10; }, "Time interval in minutes") { IsRequired = true } @@ -79,15 +79,15 @@ public static Command CreateImportTask() { var command = new Command("create-import-task", "Schedules a task for a data import to an Enbrea instance") { - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true }, - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true }, - new Option(new[] { "--interval", "-i" }, delegate() { return 10; }, "Time interval in minutes") + new Option(["--interval", "-i"], delegate() { return 10; }, "Time interval in minutes") { IsRequired = true } @@ -105,7 +105,7 @@ public static Command CreateSnaphot() { var command = new Command("create-snapshot", "Create a Enbrea database snapshot") { - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true } @@ -131,7 +131,7 @@ public static Command DeleteExportTask() { var command = new Command("delete-export-task", "Deletes a scheduled export task for Enbrea") { - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true } @@ -147,7 +147,7 @@ public static Command DeleteImportTask() { var command = new Command("delete-import-task", "Deletes a scheduled import task for Enbrea") { - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true } @@ -162,11 +162,11 @@ public static Command DeleteImportTask() public static Command DeleteSnaphot() { var command = new Command("delete-snapshot", "Delete an Enbrea database snapshot"){ - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true }, - new Option(new[] { "--id", "-id" }, "Unique ID of the snapshot") + new Option(["--id", "-id"], "Unique ID of the snapshot") { IsRequired = true } @@ -184,7 +184,7 @@ public static Command DisableExportTask() { var command = new Command("disable-export-task", "Deactivates a scheduled export task for Enbrea") { - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true } @@ -200,7 +200,7 @@ public static Command DisableImportTask() { var command = new Command("disable-import-task", "Deactivates a scheduled import task for Enbrea") { - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true } @@ -216,7 +216,7 @@ public static Command EnableExportTask() { var command = new Command("enable-export-task", "Activates a scheduled export task for Enbrea") { - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true } @@ -232,7 +232,7 @@ public static Command EnableImportTask() { var command = new Command("enable-import-task", "Activates a scheduled import task for Enbrea") { - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true } @@ -248,23 +248,23 @@ public static Command Export() { var command = new Command("export", "Exports data from an Enbrea instance to an external provider") { - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true }, - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true }, - new Option(new[] { "--skip-enbrea" }, delegate() { return false; }, "Skip export of ECF files from Enbrea") + new Option(["--skip-enbrea"], delegate() { return false; }, "Skip export of ECF files from Enbrea") { IsRequired = false }, - new Option(new[] { "--skip-provider" }, delegate() { return false; }, "Skip import of ECF files to external provider") + new Option(["--skip-provider"], delegate() { return false; }, "Skip import of ECF files to external provider") { IsRequired = false }, - new Option(new[] { "--log", "-l" }, delegate () { return null; }, "Log file folder") + new Option(["--log", "-l"], delegate () { return null; }, "Log file folder") { IsRequired = false } @@ -284,27 +284,31 @@ public static Command Import() { var command = new Command("import", "Imports data from external provider to an Enbrea instance") { - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true }, - new Option(new[] { "--provider", "-p" }, "Name of external data provider") + new Option(["--provider", "-p"], "Name of external data provider") { IsRequired = true }, - new Option(new[] { "--skip-provider" }, delegate() { return false; }, "Skip import of ECF files from external provider") + new Option(["--behaviour", "-b"], delegate() { return ImportBehaviour.diff; }, "Import behaviour") { IsRequired = false }, - new Option(new[] { "--skip-enbrea" }, delegate() { return false; }, "Skip import of ECF files to Enbrea") + new Option(["--skip-provider"], delegate() { return false; }, "Skip import of ECF files from external provider") { IsRequired = false }, - new Option(new[] { "--skip-snapshot" }, delegate() { return false; }, "Skip creating of a snapshot") + new Option(["--skip-enbrea"], delegate() { return false; }, "Skip import of ECF files to Enbrea") { IsRequired = false }, - new Option(new[] { "--log", "-l" }, delegate () { return null; }, "Log file folder") + new Option(["--skip-snapshot"], delegate() { return false; }, "Skip creating of a snapshot") + { + IsRequired = false + }, + new Option(["--log", "-l"], delegate () { return null; }, "Log file folder") { IsRequired = false } @@ -313,10 +317,11 @@ public static Command Import() command.SetHandler(CommandHandlers.Import, command.Options[0] as Option, command.Options[1] as Option, - command.Options[2] as Option, + command.Options[2] as Option, command.Options[3] as Option, command.Options[4] as Option, - command.Options[5] as Option); + command.Options[5] as Option, + command.Options[6] as Option); return command; } @@ -325,7 +330,7 @@ public static Command Init() { var command = new Command("init", "Create an Enbrea configuration file template") { - new Option(new[] { "--config", "-c" }, "Path for new JSON configuration file") + new Option(["--config", "-c"], "Path for new JSON configuration file") { IsRequired = true } @@ -350,7 +355,7 @@ public static Command ListAllTasks() public static Command ListSchoolTerms() { var command = new Command("list-schoolterms", "Get list of Enbrea school terms"){ - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true } @@ -366,7 +371,7 @@ command.Options[0] as Option public static Command ListSnaphots() { var command = new Command("list-snapshots", "Get list of Enbrea database snapshots"){ - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true } @@ -382,11 +387,11 @@ command.Options[0] as Option public static Command RestoreSnaphot() { var command = new Command("restore-snapshot", "Restore an Enbrea database from an Enbrea snapshot"){ - new Option(new[] { "--config", "-c" }, "Path to existing JSON configuration file") + new Option(["--config", "-c"], "Path to existing JSON configuration file") { IsRequired = true }, - new Option(new[] { "--id", "-id" }, "Unique ID of the snapshot") + new Option(["--id", "-id"], "Unique ID of the snapshot") { IsRequired = true } diff --git a/src/Enbrea.Cli/CommandHandlers.cs b/src/Enbrea.Cli/CommandHandlers.cs index f7d1013..96cc0fd 100644 --- a/src/Enbrea.Cli/CommandHandlers.cs +++ b/src/Enbrea.Cli/CommandHandlers.cs @@ -359,7 +359,7 @@ await Execute(async (cancellationEvent, cancellationToken) => }); } - public static async Task Import(FileInfo configFile, ImportProvider provider, bool skipProvider, bool skipEnbrea, bool skipSnapshot, string logFile) + public static async Task Import(FileInfo configFile, ImportProvider provider, ImportBehaviour behaviour, bool skipProvider, bool skipEnbrea, bool skipSnapshot, string logFile) { await Execute(async (cancellationEvent, cancellationToken) => { @@ -379,7 +379,7 @@ await Execute(async (cancellationEvent, cancellationToken) => if (!skipEnbrea) { - var importManager = ImportManagerFactory.CreateImportToEnbreaManager(provider, config, skipSnapshot, consoleWriter, cancellationEvent, cancellationToken); + var importManager = ImportManagerFactory.CreateImportToEnbreaManager(provider, config, behaviour, skipSnapshot, consoleWriter, cancellationEvent, cancellationToken); if (importManager != null) { await importManager.Execute(); diff --git a/src/Enbrea.Cli/Enbrea.Cli.csproj b/src/Enbrea.Cli/Enbrea.Cli.csproj index d368c49..0acddad 100644 --- a/src/Enbrea.Cli/Enbrea.Cli.csproj +++ b/src/Enbrea.Cli/Enbrea.Cli.csproj @@ -20,10 +20,10 @@ - + - + diff --git a/src/Enbrea.Cli/Export/ExportManager.cs b/src/Enbrea.Cli/Export/ExportManager.cs index 8c79a5f..904cdd1 100644 --- a/src/Enbrea.Cli/Export/ExportManager.cs +++ b/src/Enbrea.Cli/Export/ExportManager.cs @@ -307,10 +307,10 @@ protected async Task Extract(Guid jobId) await ThrowExportException("Start extracting failed.", response); } - WaitHandle.WaitAny(new[] - { + WaitHandle.WaitAny( + [ finishEvent, _cancellationEvent - }); + ]); if (!successfullExtract) { diff --git a/src/Enbrea.Cli/Import/ImportBehaviour.cs b/src/Enbrea.Cli/Import/ImportBehaviour.cs new file mode 100644 index 0000000..dd25d31 --- /dev/null +++ b/src/Enbrea.Cli/Import/ImportBehaviour.cs @@ -0,0 +1,28 @@ +#region Enbrea - Copyright (c) STÜBER SYSTEMS GmbH +/* + * Enbrea + * + * Copyright (c) STÜBER SYSTEMS GmbH + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ +#endregion + +namespace Enbrea.Cli +{ + public enum ImportBehaviour + { + diff, full, smartfull + } +} diff --git a/src/Enbrea.Cli/Import/ImportManager.cs b/src/Enbrea.Cli/Import/ImportManager.cs index f6a3b90..869626d 100644 --- a/src/Enbrea.Cli/Import/ImportManager.cs +++ b/src/Enbrea.Cli/Import/ImportManager.cs @@ -19,6 +19,7 @@ */ #endregion +using DocumentFormat.OpenXml.Wordprocessing; using Enbrea.Cli.Common; using Enbrea.Csv; using Enbrea.Ecf; @@ -43,6 +44,7 @@ namespace Enbrea.Cli /// public class ImportManager : CustomManager { + private readonly ImportBehaviour _behaviour; private readonly ImportProvider _provider; private readonly ProviderEcfMapping _providerEcfMapping; private readonly bool _skipSnapshot; @@ -52,6 +54,7 @@ public ImportManager( ProviderEcfMapping providerEcfMapping, string dataFolderName, Configuration config, + ImportBehaviour behaviour, bool skipSnapshot, ConsoleWriter consoleWriter, EventWaitHandle cancellationEvent, @@ -61,6 +64,7 @@ public ImportManager( _provider = provider; _providerEcfMapping = providerEcfMapping; _skipSnapshot = skipSnapshot; + _behaviour = behaviour; } public override async Task Execute() @@ -88,7 +92,7 @@ public override async Task Execute() // Has context changed? if ((previousContext.SchoolTerm != _config.SchoolTerm) || (previousContext.ValidFrom != manifest.ValidFrom) || - (previousContext.ValidTo != manifest.ValidTo)) + (previousContext.ValidTo != manifest.ValidTo) || _behaviour == ImportBehaviour.full) { // Console output _consoleWriter.Caption("Delete previous ECF files"); @@ -108,8 +112,8 @@ public override async Task Execute() var changedOnlyFileCount = await GenerateChangedOnlyFiles(files); var deletedOnlyFileCount = await GenerateDeletedOnlyFiles(files); - // Do we have something to import? - newImportDataAvailable = changedOnlyFileCount > 0 || deletedOnlyFileCount > 0; + // Do we have something to import? + newImportDataAvailable = changedOnlyFileCount > 0 || deletedOnlyFileCount > 0; } // Do we have data to import? @@ -359,28 +363,36 @@ protected async Task GenerateChangedOnlyFiles(IEnumerable files { if (File.Exists(file.FullNameForPreviousRows)) { - var numberOfAffectedRows = 0; - - using var ecfTextReader = File.OpenText(file.FullName); - using var ecfTextReaderForPreviousRows = File.OpenText(file.FullNameForPreviousRows); - using (var ecfTextWriterForChangedOnlyRows = File.CreateText(file.FullNameForChangedOnlyRows)) + if (_behaviour == ImportBehaviour.smartfull) { - var ecfTableReader = new EcfTableReader(ecfTextReader); - var ecfTableReaderForPreviousRows = new EcfTableReader(ecfTextReaderForPreviousRows); - var ecfTableDiff = new CsvDiff(ecfTextWriterForChangedOnlyRows, new EcfConfiguration(), EcfHeaders.Id); - - numberOfAffectedRows = await ecfTableDiff.GenerateAsync(CsvDiffType.AddedOrUpdatedOnly, ecfTableReaderForPreviousRows, ecfTableReader, _cancellationToken); - } + File.Copy(file.FullName, file.FullNameForChangedOnlyRows, overwrite: true); - if (numberOfAffectedRows == 0) - { - File.Delete(file.FullNameForChangedOnlyRows); + _consoleWriter.ContinueProgress(++fileCount); } else { - _consoleWriter.ContinueProgress(++fileCount); + var numberOfAffectedRows = 0; + + using var ecfTextReader = File.OpenText(file.FullName); + using var ecfTextReaderForPreviousRows = File.OpenText(file.FullNameForPreviousRows); + using (var ecfTextWriterForChangedOnlyRows = File.CreateText(file.FullNameForChangedOnlyRows)) + { + var ecfTableReader = new EcfTableReader(ecfTextReader); + var ecfTableReaderForPreviousRows = new EcfTableReader(ecfTextReaderForPreviousRows); + var ecfTableDiff = new CsvDiff(ecfTextWriterForChangedOnlyRows, new EcfConfiguration(), EcfHeaders.Id); + + numberOfAffectedRows = await ecfTableDiff.GenerateAsync(CsvDiffType.AddedOrUpdatedOnly, ecfTableReaderForPreviousRows, ecfTableReader, _cancellationToken); + } + + if (numberOfAffectedRows == 0) + { + File.Delete(file.FullNameForChangedOnlyRows); + } + else + { + _consoleWriter.ContinueProgress(++fileCount); + } } - } else { @@ -558,10 +570,7 @@ protected async Task Merge(Guid jobId) await ThrowImportException("Start merging failed", response); } - WaitHandle.WaitAny(new[] - { - finishEvent, _cancellationEvent - }); + WaitHandle.WaitAny([finishEvent, _cancellationEvent]); if (!successfullMerge) { diff --git a/src/Enbrea.Cli/Import/ImportManagerFactory.cs b/src/Enbrea.Cli/Import/ImportManagerFactory.cs index 3b80801..ec0fb9c 100644 --- a/src/Enbrea.Cli/Import/ImportManagerFactory.cs +++ b/src/Enbrea.Cli/Import/ImportManagerFactory.cs @@ -27,20 +27,23 @@ namespace Enbrea.Cli { public static class ImportManagerFactory { - public static EcfCustomManager CreateImportToEnbreaManager(ImportProvider provider, Configuration config, bool skipSnapshot, ConsoleWriter consoleWriter, EventWaitHandle cancellationEvent, CancellationToken cancellationToken) + public static EcfCustomManager CreateImportToEnbreaManager(ImportProvider provider, Configuration config, ImportBehaviour behaviour, bool skipSnapshot, + ConsoleWriter consoleWriter, EventWaitHandle cancellationEvent, CancellationToken cancellationToken) { return new ImportManager( provider, GetProviderEcfMapping(provider, config), GetEnbreaEcfTarget(provider, config), - config, - skipSnapshot, + config, + behaviour, + skipSnapshot, consoleWriter, cancellationEvent, cancellationToken); } - public static EcfCustomManager CreateImportToProviderManager(ExportProvider provider, Configuration config, ConsoleWriter consoleWriter, CancellationToken cancellationToken) + public static EcfCustomManager CreateImportToProviderManager(ExportProvider provider, Configuration config, + ConsoleWriter consoleWriter, CancellationToken cancellationToken) { switch (provider) {