Added DatFileDeletor as PostProcessingStep and removed all old refere…

…nces to it
FZJ-IEK3-VSA · Oct 12, 2021 · d4e98c5 · d4e98c5
1 parent c1e5433
commit d4e98c5
Show file tree

Hide file tree

Showing 21 changed files with 52 additions and 82 deletions.
diff --git a/Automation/CalcOption.cs b/Automation/CalcOption.cs
@@ -59,6 +59,7 @@ public enum CalcOption {
         AffordanceDefinitions,
         JsonHouseholdSumFilesNoFlex,
         HouseholdSumProfilesCsvNoFlex,
-        FlexibilityEvents
+        FlexibilityEvents,
+        DeleteDatFiles
     }
 }
diff --git a/Automation/JsonCalcSpecification.cs b/Automation/JsonCalcSpecification.cs
@@ -47,7 +47,6 @@ public JsonCalcSpecification([NotNull] JsonCalcSpecification o)
             }
 
             DeleteAllButPDF = o.DeleteAllButPDF;
-            DeleteDAT = o.DeleteDAT;
             DeviceSelection = o.DeviceSelection;
             StartDate = o.StartDate;
             EndDate = o.EndDate;
@@ -102,9 +101,6 @@ public JsonCalcSpecification([NotNull] JsonCalcSpecification o)
             "to generate a full set of PDFs for all households to get a detailed view of the results for each household. Default: false")]
         public bool DeleteAllButPDF { get; set; }
 
-        [Comment("This option make the LPG delete all the DAT-files after the calculation. Default=true")]
-        public bool DeleteDAT { get; set; } = true;
-
         [Comment("If you want the people to use certain devices, for example if you want to make sure that " +
                  "the people really use incandescent light bulbs, then you can set up a device selection to ensure that " +
                  "this type of device will always be selected.")]

diff --git a/CalcPostProcessor/GeneralSteps/DatFileDeletor.cs b/CalcPostProcessor/GeneralSteps/DatFileDeletor.cs
@@ -0,0 +1,33 @@
+using System.Collections.Generic;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using Automation;
+using Automation.ResultFiles;
+using CalcPostProcessor.Steps;
+using Common;
+using Common.SQLResultLogging;
+
+namespace CalcPostProcessor.GeneralSteps
+{
+    [SuppressMessage("ReSharper", "RedundantNameQualifier")]
+    public class DatFileDeletor: GeneralStepBase
+    {
+        [JetBrains.Annotations.NotNull]
+        private readonly IFileFactoryAndTracker _fft;
+
+        public DatFileDeletor([JetBrains.Annotations.NotNull] CalcDataRepository repository, [JetBrains.Annotations.NotNull] ICalculationProfiler calculationProfiler,
+                                    [JetBrains.Annotations.NotNull] IFileFactoryAndTracker fft)
+            : base(repository, AutomationUtili.GetOptionList(CalcOption.HouseholdContents), calculationProfiler, "Delete .dat files", 0)
+        {
+            _fft = fft;
+        }
+
+        protected override void PerformActualStep(IStepParameters parameters)
+        {
+            // insert deleting function here
+        }
+
+        [JetBrains.Annotations.NotNull]
+        public override List<CalcOption> NeededOptions => new List<CalcOption>() {CalcOption.DeleteDatFiles};
+    }
+}
diff --git a/CalcPostProcessor/PostProcessingManager.cs b/CalcPostProcessor/PostProcessingManager.cs
@@ -56,6 +56,7 @@ public static  IContainer RegisterEverything(string resultPath, [JetBrains.Annot
 
             //general processing steps
             builder.RegisterType<AffordanceTagsWriter>().As<IGeneralStep>().SingleInstance();
+            builder.RegisterType<DatFileDeletor>().As<IGeneralStep>().SingleInstance();
 
             //general household steps
             builder.RegisterType<MakeActivationsPerFrequencies>().As<IGeneralHouseholdStep>().SingleInstance();

diff --git a/CalcPostProcessor/PostProcessor.cs b/CalcPostProcessor/PostProcessor.cs
@@ -190,14 +190,6 @@ private void ActualFunctionCaller([CanBeNull] [ItemNotNull] List<string> loadTyp
                     CalcOptionHelper.CalcOptionDictionary[CalcOption.OverallSum]);
                 _calculationProfiler.StopPart(Utili.GetCurrentMethodAndClass() + " - Overall Sum File");
             }*/
-            {
-                var gsp = new GeneralStepParameters();
-                foreach (var generalPostProcessingStep in _generalPostProcessingSteps) {
-                    if (generalPostProcessingStep.IsEnabled()) {
-                        generalPostProcessingStep.Run(gsp);
-                    }
-                }
-            }
 
             foreach (var keyEntry in _repository.HouseholdKeys) {
                 var gsp = new HouseholdStepParameters(keyEntry);
@@ -236,6 +228,16 @@ private void ActualFunctionCaller([CanBeNull] [ItemNotNull] List<string> loadTyp
                 //repository.efc, devices, calcLocations, autoDevices, loadTypes, deviceTaggingSets, allResults,persons, deviceNameToDeviceCategory, householdNamesByKey,carpetPlotColumnWidth);
                 _calculationProfiler.StopPart(Utili.GetCurrentMethodAndClass() + " - Load Type Dependend");
             }
+            {
+                var gsp = new GeneralStepParameters();
+                foreach (var generalPostProcessingStep in _generalPostProcessingSteps)
+                {
+                    if (generalPostProcessingStep.IsEnabled())
+                    {
+                        generalPostProcessingStep.Run(gsp);
+                    }
+                }
+            }
         }
 
         private static void CheckTotalsForChange([JetBrains.Annotations.NotNull] [ItemNotNull]

diff --git a/CalculationController/DtoFactories/CalcParametersFactory.cs b/CalculationController/DtoFactories/CalcParametersFactory.cs
@@ -18,7 +18,6 @@ public CalcParameters MakeCalculationParametersFromConfig([JetBrains.Annotations
         {
             CalcParameters cp = CalcParameters.GetNew();
             cp.LoadtypesToPostprocess = csps.LoadTypesToProcess;
-            cp.SetDeleteDatFiles(csps.DeleteDatFiles);
             cp.SetWriteExcelColumn (csps.WriteExcelColumn);
             cp.SetManyOptionsWithClear(csps.CalcOptions);
             FileFactoryAndTrackerDummy fftd = new FileFactoryAndTrackerDummy();

diff --git a/CalculationController/InputLoggers/CalcStartParameterSetLogger.cs b/CalculationController/InputLoggers/CalcStartParameterSetLogger.cs
@@ -26,7 +26,6 @@ public override void Run(HouseholdKey key,object o)
             se.AddRow(RowBuilder.Start("Name", "AffordanceRepetitionCount").Add("Value", csps.AffordanceRepetitionCount.ToString(CultureInfo.InvariantCulture)).ToDictionary());
             se.AddRow(RowBuilder.Start("Name", "CalcOptions").Add("Value", JsonConvert.SerializeObject(csps.CalcOptions,Formatting.Indented)).ToDictionary());
             se.AddRow(RowBuilder.Start("Name", "CalcTarget").Add("Value", csps.CalcTarget.Name).ToDictionary());
-            se.AddRow(RowBuilder.Start("Name", "DeleteDatFiles").Add("Value", csps.DeleteDatFiles.ToString()).ToDictionary());
             se.AddRow(RowBuilder.Start("Name", "DeviceSelection").Add("Value", csps.DeviceSelection?.Name??"None").ToDictionary());
             se.AddRow(RowBuilder.Start("Name", "EnergyIntensity").Add("Value", csps.EnergyIntensity.ToString()).ToDictionary());
             se.AddRow(RowBuilder.Start("Name", "ExternalTimeResolution").Add("Value", csps.ExternalTimeResolution.ToString()).ToDictionary());

diff --git a/CalculationController/Queue/CalcQueueRunner.cs b/CalculationController/Queue/CalcQueueRunner.cs
@@ -161,8 +161,6 @@ private static void SaveRun(bool forceRandom,
 #pragma warning restore 162
             if (allgood) {
                 Logger.Info("Starting final cleanup");
-                var cpp = new DatFileDeletor( csps.DeleteDatFiles,csps.ResultPath, csps.CalcTarget.Name);
-                cpp.ProcessResults();
                 SaveCallFunction(csps.ReportFinishFuncForHousehold,csps);
                 Logger.Info("Finished final cleanup");
             }

diff --git a/CalculationController/Queue/CalcStartParameterSet.cs b/CalculationController/Queue/CalcStartParameterSet.cs
@@ -80,7 +80,6 @@ public CalcStartParameterSet(
             CsvCharacter = csvCharacter;
             SelectedRandomSeed = selectedRandomSeed;
             ExternalTimeResolution = externalTimeResolution;
-            DeleteDatFiles = deleteDatFiles;
             WriteExcelColumn = writeExcelColumn;
             ShowSettlingPeriod = showSettlingPeriod;
             SettlingDays = settlingDays;
@@ -139,7 +138,6 @@ [NotNull] Func<bool, string, ObservableCollection<ResultFileEntry>, bool>
             ResultPath = resultPath;
             LoadTypesToProcess = loadTypesToProcess;
             ExternalTimeResolution = externalTimeResolution;
-            DeleteDatFiles = deleteDatFiles;
             WriteExcelColumn = writeExcelColumn;
             ShowSettlingPeriod = showSettlingPeriod;
             SettlingDays = settlingDays;
@@ -193,8 +191,6 @@ [NotNull] Func<bool, string, ObservableCollection<ResultFileEntry>, bool>
         [NotNull]
         public string DecimalSeperator { get; }
 
-        public bool DeleteDatFiles { get; }
-
         [CanBeNull]
         public DeviceSelection DeviceSelection { get; }
 

diff --git a/Common/Enums/CalcOptionHelper.cs b/Common/Enums/CalcOptionHelper.cs
@@ -65,7 +65,7 @@ static CalcOptionHelper()
             CalcOptionDictionary.Add(CalcOption.JsonHouseholdSumFilesNoFlex, "Household sums with the flexible devices excluded as JSON");
             CalcOptionDictionary.Add(CalcOption.HouseholdSumProfilesCsvNoFlex, "Household sums with the flexible devices excluded as CSV");
             CalcOptionDictionary.Add(CalcOption.FlexibilityEvents, "Flexibility Events describing shiftable devices such as washing machines");
-
+            CalcOptionDictionary.Add(CalcOption.DeleteDatFiles, "Delete .dat files");
         }
 
         [SuppressMessage("Microsoft.Security", "CA2104:DoNotDeclareReadOnlyMutableReferenceTypes")]

diff --git a/Common/JSON/CalcParameters.cs b/Common/JSON/CalcParameters.cs
@@ -290,12 +290,6 @@ public CalcParameters SetDecimalSeperator([NotNull] string decimalSeperator)
             DecimalSeperator = decimalSeperator;
             return this;
         }
-        [NotNull]
-        public CalcParameters SetDeleteDatFiles(bool deleteDatFiles)
-        {
-            DeleteDatFiles = deleteDatFiles;
-            return this;
-        }
 
         [NotNull]
         public CalcParameters SetDummyTimeSteps(int timesteps)

diff --git a/Database/Tables/GeneralConfig.cs b/Database/Tables/GeneralConfig.cs
@@ -112,16 +112,6 @@ public DeviceProfileHeaderMode DeviceProfileHeaderMode {
             }
         }
 
-        [UsedImplicitly]
-        public bool DeleteDatFilesBool => DeleteDatFiles=="TRUE";
-
-        [JetBrains.Annotations.NotNull]
-        [UsedImplicitly]
-        public string DeleteDatFiles {
-            get => _settings[nameof(DeleteDatFiles)].SettingValue;
-            set => UpdateValue(nameof(DeleteDatFiles), value);
-        }
-
         [JetBrains.Annotations.NotNull]
         [UsedImplicitly]
         public string DestinationPath {
@@ -537,7 +527,6 @@ public static GeneralConfig LoadFromDatabase([JetBrains.Annotations.NotNull] str
             gc.CheckExistence(nameof(LastSelectedCalcType), "Household", ignoreMissing);
             gc.CheckExistence(nameof(SelectedLoadTypePriority), "Mandatory", ignoreMissing);
 
-            gc.CheckExistence(nameof(DeleteDatFiles), "False", ignoreMissing);
             gc.CheckExistence(nameof(EnableFlexibility), "False", ignoreMissing);
             gc.CheckExistence(nameof(CarpetPlotWidth), "7", ignoreMissing);
             gc.CheckExistence(nameof(SelectedEnergyIntensity), "0", ignoreMissing);

diff --git a/Database/Tables/Houses/Settlement.cs b/Database/Tables/Houses/Settlement.cs
@@ -171,19 +171,6 @@ public string DstPath {
             }
         }*/
 
-        public bool DeleteDatFiles {
-            get => _calcSpecification.DeleteDAT;
-            [UsedImplicitly]
-            set {
-                if (_calcSpecification.DeleteDAT == value) {
-                    return;
-                }
-
-                _calcSpecification.DeleteDAT = value;
-                OnPropertyChanged(nameof(DeleteDatFiles));
-            }
-        }
-
         public bool DeleteSqliteFiles {
             get => _calcSpecification.DeleteSqlite;
             [UsedImplicitly]

diff --git a/SimulationEngine.Tests/CalculationTests.cs b/SimulationEngine.Tests/CalculationTests.cs
@@ -58,7 +58,6 @@ public static HouseCreationAndCalculationJob PrepareNewHouseForHouseholdTesting(
             hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
             SetEndDate(duration, hj);
 
-    hj.CalcSpec.DeleteDAT = false;
             hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.NoFiles;
             hj.CalcSpec.DeleteSqlite = false;
             hj.CalcSpec.ExternalTimeResolution = "00:15:00";
@@ -109,7 +108,6 @@ public static HouseCreationAndCalculationJob PrepareNewHouseForHouseholdTestingW
             };
             hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
             SetEndDate(duration, hj);
-            hj.CalcSpec.DeleteDAT = false;
             hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.NoFiles;
             hj.CalcSpec.DeleteSqlite = false;
             hj.CalcSpec.ExternalTimeResolution = "00:15:00";
@@ -141,7 +139,6 @@ public static HouseCreationAndCalculationJob PrepareNewHouseForHousetypeTesting(
             };
             hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
             hj.CalcSpec.EndDate = new DateTime(2020, 1, 3);
-            hj.CalcSpec.DeleteDAT = false;
             hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.NoFiles;
             hj.CalcSpec.DeleteSqlite = false;
             hj.CalcSpec.ExternalTimeResolution = "00:15:00";
@@ -173,7 +170,6 @@ public static HouseCreationAndCalculationJob PrepareNewHouseForOutputFileTesting
             };
             hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
             hj.CalcSpec.EndDate = new DateTime(2020, 1, 3);
-            hj.CalcSpec.DeleteDAT = false;
             hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.NoFiles;
             hj.CalcSpec.DeleteSqlite = false;
             hj.CalcSpec.ExternalTimeResolution = "00:15:00";
@@ -569,7 +565,6 @@ static HouseCreationAndCalculationJob PrepareHousejob(Simulator sim)
                 hj.CalcSpec = JsonCalcSpecification.MakeDefaultsForTesting();
                 hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
                 hj.CalcSpec.EndDate = new DateTime(2020, 1, 3);
-                hj.CalcSpec.DeleteDAT = false;
                 hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.Reasonable;
                 hj.CalcSpec.DeleteSqlite = false;
                 hj.CalcSpec.ExternalTimeResolution = "00:15:00";
@@ -932,13 +927,11 @@ private static HouseCreationAndCalculationJob MakeKenishHouseJob(Simulator sim,
             };
             hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
             hj.CalcSpec.EndDate = new DateTime(2020, 1, 3);
-            hj.CalcSpec.DeleteDAT = false;
             hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.Reasonable;
             hj.CalcSpec.DeleteSqlite = false;
             hj.CalcSpec.ExternalTimeResolution = "00:15:00";
             hj.CalcSpec.EnableTransportation = true;
             hj.CalcSpec.GeographicLocation = sim.GeographicLocations.FindFirstByNameNotNull("Berlin", FindMode.Partial).GetJsonReference();
-            hj.CalcSpec.DeleteDAT = true;
             hj.CalcSpec.OutputDirectory = "TestingData_" + idx;
             if (hj.CalcSpec.CalcOptions == null) {
                 throw new LPGException("was null");
@@ -1094,7 +1087,6 @@ public static HouseCreationAndCalculationJob PrepareNewHouseholdWithTemplateForT
             };
             hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
             hj.CalcSpec.EndDate = new DateTime(2020, 1, 3);
-            hj.CalcSpec.DeleteDAT = false;
             hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.NoFiles;
             hj.CalcSpec.DeleteSqlite = false;
             //hj.CalcSpec.EnergyIntensityType = EnergyIntensityType.EnergyIntensive;

diff --git a/SimulationEngine.Tests/MakeHomeOfficeHouseholds.cs b/SimulationEngine.Tests/MakeHomeOfficeHouseholds.cs
@@ -86,7 +86,6 @@ public static HouseCreationAndCalculationJob PrepareNewHouseForHouseholdTemplate
             };
             hj.CalcSpec.StartDate = new DateTime(2020, 1, 1);
             hj.CalcSpec.EndDate = new DateTime(2020, 7, 1);
-            hj.CalcSpec.DeleteDAT = false;
             hj.CalcSpec.DefaultForOutputFiles = OutputFileDefault.NoFiles;
             hj.CalcSpec.DeleteSqlite = false;
             hj.CalcSpec.ExternalTimeResolution = "00:15:00";

diff --git a/SimulationEngine.Tests/SimZukunftProcessor/HouseGeneratorJobTests.cs b/SimulationEngine.Tests/SimZukunftProcessor/HouseGeneratorJobTests.cs
@@ -369,8 +369,7 @@ public void HouseJobForHouseTypes()
                                 CalcOption.EnergyStorageFile,
                                 //CalcOption.EnergyCarpetPlot,
                                 CalcOption.HouseholdContents, CalcOption.TotalsPerLoadtype
-                            },
-                            DeleteDAT = false
+                            }
                         };
                         StartHouseJob(houseJob, wd, htcode);
                         SqlResultLoggingService srls = new SqlResultLoggingService(houseJob.CalcSpec.OutputDirectory);

diff --git a/SimulationEngineLib/HouseJobProcessor/HouseGenerator.cs b/SimulationEngineLib/HouseJobProcessor/HouseGenerator.cs
@@ -145,7 +145,6 @@ public static void CreateExampleHouseJob([NotNull] string connectionString)
             var calculationSettings = new JsonCalcSpecification {
                 StartDate = new DateTime(2019, 1, 1),
                 EndDate = new DateTime(2019, 1, 3),
-                DeleteDAT = true,
                 DefaultForOutputFiles = OutputFileDefault.OnlySums
             };
             if (calculationSettings.CalcOptions == null) {

diff --git a/SimulationEngineLib/HouseJobProcessor/JsonCalculator.cs b/SimulationEngineLib/HouseJobProcessor/JsonCalculator.cs
@@ -187,13 +187,6 @@ public void StartHousehold([JetBrains.Annotations.NotNull] Simulator sim, [JetBr
                 }
             }
 
-            if (jcs.DeleteDAT) {
-                sim.MyGeneralConfig.DeleteDatFiles = "TRUE";
-            }
-            else {
-                sim.MyGeneralConfig.DeleteDatFiles = "FALSE";
-            }
-
             if (jcs.ExternalTimeResolution == null) {
                 sim.MyGeneralConfig.ExternalTimeResolution = sim.MyGeneralConfig.InternalTimeResolution;
             }
@@ -267,7 +260,7 @@ public void StartHousehold([JetBrains.Annotations.NotNull] Simulator sim, [JetBr
                 sim.MyGeneralConfig.CSVCharacter,
                 jcs.RandomSeed,
                 sim.MyGeneralConfig.ExternalStepSize,
-                sim.MyGeneralConfig.DeleteDatFilesBool,
+                false /*DELETE DAT FILES*/,
                 sim.MyGeneralConfig.WriteExcelColumnBool,
                 sim.MyGeneralConfig.ShowSettlingPeriodBool,
                 3,
@@ -456,4 +449,4 @@ private OutputFileDefault GetOutputFileDefaultsFromString([CanBeNull] string def
             throw new LPGException("Unrecognized default for output files value");
         }*/
     }
-}
+}
diff --git a/SimulationEngineLib/HouseJobProcessor/ParallelJsonLauncher.cs b/SimulationEngineLib/HouseJobProcessor/ParallelJsonLauncher.cs
@@ -443,13 +443,6 @@ private void SingleThreadExecutor(int index)
                                         completedCalc.Status = "No folder and no finished.flag found";
                                         Logger.Info("Thread " + index + ": Not enqueued for moving due to missing finished flag: " + outputFolder.FullName + progress);
                                     }
-
-                                    if (jcs.CalcSpec.DeleteDAT) {
-                                        var dats = outputFolder.GetFiles("*.dat", SearchOption.AllDirectories);
-                                        foreach (var info in dats) {
-                                            info.Delete();
-                                        }
-                                    }
                                 }
                                 else {
                                     completedCalc.Status = "No folder found";

diff --git a/WpfApplication1/Presenters/SpecialViews/CalculationOutcomesPresenter.cs b/WpfApplication1/Presenters/SpecialViews/CalculationOutcomesPresenter.cs
@@ -590,7 +590,7 @@ public static void StartOneCalculation([JetBrains.Annotations.NotNull] ICalcObje
                     ";",
                     -1,
                     sim.MyGeneralConfig.ExternalStepSize,
-                    sim.MyGeneralConfig.DeleteDatFilesBool,
+                    false /*DELETE DAT FILES*/,
                     sim.MyGeneralConfig.WriteExcelColumnBool,
                     sim.MyGeneralConfig.ShowSettlingPeriodBool,
                     3,

diff --git a/WpfApplication1/Presenters/SpecialViews/CalculationPresenter.cs b/WpfApplication1/Presenters/SpecialViews/CalculationPresenter.cs
@@ -594,7 +594,7 @@ public void RunSimulation([CanBeNull] string resultpath)
                 // ReSharper disable once AssignNullToNotNullAttribute
                 _selectedCalcObject, EnergyIntensity.EnergyIntensityType, ReportCancel, ResumeSettlement,  null, Sim.MyGeneralConfig.SelectedLoadTypePriority, tds, trs,
                 Sim.MyGeneralConfig.AllEnabledOptions(), Sim.MyGeneralConfig.StartDateDateTime, Sim.MyGeneralConfig.EndDateDateTime, Sim.MyGeneralConfig.InternalStepSize,
-                Sim.MyGeneralConfig.CSVCharacter, Sim.MyGeneralConfig.RandomSeed, Sim.MyGeneralConfig.ExternalStepSize, Sim.MyGeneralConfig.DeleteDatFilesBool,
+                Sim.MyGeneralConfig.CSVCharacter, Sim.MyGeneralConfig.RandomSeed, Sim.MyGeneralConfig.ExternalStepSize, false /*DELETE DAT FILES*/,
                 Sim.MyGeneralConfig.WriteExcelColumnBool, Sim.MyGeneralConfig.ShowSettlingPeriodBool, 3,
                 Sim.MyGeneralConfig.RepetitionCount, calculationProfiler, SelectedChargingStationSet,null,
                 Sim.MyGeneralConfig.DeviceProfileHeaderMode,false,resultpath,_calculateTransportation,