From f24baad5e1873c0f4600c215fa531f2a5a2a680a Mon Sep 17 00:00:00 2001 From: Christian Hansen <120550591+cenh-halfspace@users.noreply.github.com> Date: Thu, 23 Jan 2025 09:13:05 +0100 Subject: [PATCH] feat: Removed Sett Repo Subsystem Test (#3095) --- .../Fixtures/Databricks/FileInfo.cs | 17 -- .../Fixtures/Databricks/FilesApiClient.cs | 71 ------- .../Databricks/FilesDatabricksClient.cs | 65 ------ .../Fixtures/Databricks/IFilesApi.cs | 38 ---- .../Fixtures/SettlementReportJobName.cs | 24 --- ...ettlementReportJobScenarioConfiguration.cs | 54 ----- .../SettlementReportJobScenarioFixture.cs | 187 ------------------ ...saleCalculationsJobCollectionDefinition.cs | 28 --- ...ortBalanceFixingJobGeneratesZipScenario.cs | 141 ------------- ...saleCalculationsJobGeneratesZipScenario.cs | 146 -------------- .../States/GeneratesZipScenarioState.cs | 43 ---- .../States/SettlementReportJobState.cs | 25 --- 12 files changed, 839 deletions(-) delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FileInfo.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FilesApiClient.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FilesDatabricksClient.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/IFilesApi.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobName.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobScenarioConfiguration.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobScenarioFixture.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportWholesaleCalculationsJobCollectionDefinition.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/SettlementReportBalanceFixingJobGeneratesZipScenario.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/SettlementReportWholesaleCalculationsJobGeneratesZipScenario.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/States/GeneratesZipScenarioState.cs delete mode 100644 source/dotnet/subsystem-tests/Features/SettlementReports/States/SettlementReportJobState.cs diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FileInfo.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FileInfo.cs deleted file mode 100644 index dc769f68a6..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FileInfo.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures.Databricks; - -public record FileInfo(string ContentType, long ContentLength, DateTimeOffset? LastModified); diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FilesApiClient.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FilesApiClient.cs deleted file mode 100644 index fff097636f..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FilesApiClient.cs +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Microsoft.Azure.Databricks.Client; - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures.Databricks; - -/// -/// Extend the Databricks Client with operations from the Files REST API: https://docs.databricks.com/api/azure/workspace/files -/// Inspired by the design of the Databricks Client library: https://github.com/Azure/azure-databricks-client -/// -public sealed class FilesApiClient : ApiClient, IFilesApi -{ - /// - /// Initializes a new instance of the class. - /// - /// The HTTP client. - public FilesApiClient(HttpClient httpClient) - : base(httpClient) - { - } - - /// - public async Task GetFileInfoAsync(string filePath, CancellationToken cancellationToken = default) - { - var request = new HttpRequestMessage(HttpMethod.Head, GetUrl(filePath)); - using var response = await HttpClient.SendAsync(request, cancellationToken); - - if (!response.IsSuccessStatusCode) - { - throw CreateApiException(response); - } - - return new FileInfo( - ContentType: response.Content.Headers.ContentType?.ToString() ?? string.Empty, - ContentLength: response.Content.Headers.ContentLength ?? -1, - LastModified: response.Content.Headers.LastModified); - } - - /// - /// Gets a stream to the file. - /// - /// The absolute path of the file. - /// Cancellation token. - /// A stream to the file. - public async Task GetFileStreamAsync(string filePath, CancellationToken cancellationToken = default) - { - var request = new HttpRequestMessage(HttpMethod.Get, GetUrl(filePath)); - var response = await HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); - - if (!response.IsSuccessStatusCode) - { - throw CreateApiException(response); - } - - return await response.Content.ReadAsStreamAsync(cancellationToken); - } - - private string GetUrl(string filePath) => $"{ApiVersion}/fs/files{filePath}"; -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FilesDatabricksClient.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FilesDatabricksClient.cs deleted file mode 100644 index a800fbea1d..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/FilesDatabricksClient.cs +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Net; -using System.Net.Http.Headers; -using Microsoft.Azure.Databricks.Client; - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures.Databricks; - -/// -/// Inspired by the implementation of the : -/// https://github.com/Azure/azure-databricks-client/blob/f7ffd8a2b4835351c000bf8a46797fd54addc7e5/csharp/Microsoft.Azure.Databricks.Client/DatabricksClient.cs -/// -public sealed class FilesDatabricksClient : IDisposable -{ - public FilesDatabricksClient(string baseUrl, string token) - { - Files = new FilesApiClient(CreateHttpClient(baseUrl, token)); - } - - public IFilesApi Files { get; } - - public void Dispose() - { - Files.Dispose(); - GC.SuppressFinalize(this); - } - - private static HttpClient CreateHttpClient(string baseUrl, string bearerToken, long timeoutSeconds = 30) - { - var apiUrl = new Uri(new Uri(baseUrl), "api/"); - - var handler = new HttpClientHandler - { - AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate, - }; - - var httpClient = new HttpClient(handler, false) - { - BaseAddress = apiUrl, - Timeout = TimeSpan.FromSeconds(timeoutSeconds), - }; - - SetDefaultHttpHeaders(httpClient); - httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", bearerToken); - return httpClient; - } - - private static void SetDefaultHttpHeaders(HttpClient httpClient) - { - httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - httpClient.DefaultRequestHeaders.AcceptEncoding.Add(new StringWithQualityHeaderValue("gzip")); - } -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/IFilesApi.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/IFilesApi.cs deleted file mode 100644 index b909f3da90..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/Databricks/IFilesApi.cs +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures.Databricks; - -/// -/// Extend the Databricks Client with operations from the Files REST API: https://docs.databricks.com/api/azure/workspace/files -/// Inspired by the design of the Databricks Client library: https://github.com/Azure/azure-databricks-client -/// -public interface IFilesApi : IDisposable -{ - /// - /// Wrapping a call to: https://docs.databricks.com/api/azure/workspace/files/getmetadata - /// - /// The absolute path of the file. Example: "/Volumes/my-catalog/my-schema/my-volume/directory/file.txt" - /// - /// File information if we can get metadata for the file; otherwise throws an exception. - Task GetFileInfoAsync(string filePath, CancellationToken cancellationToken = default); - - /// - /// Wrapping a call to: https://docs.databricks.com/api/azure/workspace/files/download - /// - /// The absolute path of the file. Example: "/Volumes/my-catalog/my-schema/my-volume/directory/file.txt" - /// - /// A stream to the file if it exists; otherwise throws an exception. - Task GetFileStreamAsync(string filePath, CancellationToken cancellationToken = default); -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobName.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobName.cs deleted file mode 100644 index f0f1e861b7..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobName.cs +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures; - -/// -/// Settlement report job names in Databricks. -/// -public enum SettlementReportJobName -{ - SettlementReportBalanceFixing, - SettlementReportWholesaleCalculations, -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobScenarioConfiguration.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobScenarioConfiguration.cs deleted file mode 100644 index 164e602a85..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobScenarioConfiguration.cs +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.Configuration; -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.Extensions; -using Microsoft.Extensions.Configuration; - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Performance.Fixtures; - -/// -/// Responsible for retrieving environment specific settings necessary for performing tests of 'SettlementReportJob' in Databricks. -/// -/// On developer machines we use the 'subsystemtest.local.settings.json' to set values. -/// On hosted agents we must set these using environment variables. -/// -public class SettlementReportJobScenarioConfiguration : SubsystemTestConfiguration -{ - public SettlementReportJobScenarioConfiguration() - { - InputCalculationId = Root.GetValue("SETTLEMENT_REPORT_CALCULATION_ID")!; - - var databricksCatalogName = Root.GetValue("DATABRICKS_CATALOG_NAME")!; - DatabricksCatalogRoot = $"/Volumes/{databricksCatalogName}"; - - var secretsConfiguration = Root.BuildSecretsConfiguration(); - DatabricksWorkspace = DatabricksWorkspaceConfiguration.CreateFromConfiguration(secretsConfiguration); - } - - /// - /// Calculation ID used for input parameter when starting Settlement Report Job. - /// - public string InputCalculationId { get; } - - /// - /// Databricks catalog root. - /// - public string DatabricksCatalogRoot { get; } - - /// - /// Settings necessary to manage the Databricks workspace. - /// - public DatabricksWorkspaceConfiguration DatabricksWorkspace { get; } -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobScenarioFixture.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobScenarioFixture.cs deleted file mode 100644 index 24d26c9574..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportJobScenarioFixture.cs +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Energinet.DataHub.Core.TestCommon; -using Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures.Databricks; -using Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.States; -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures; -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.Extensions; -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.LazyFixture; -using Energinet.DataHub.Wholesale.SubsystemTests.Performance.Fixtures; -using Microsoft.Azure.Databricks.Client; -using Microsoft.Azure.Databricks.Client.Models; -using Xunit.Abstractions; -using FileInfo = Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures.Databricks.FileInfo; - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures; - -public sealed class SettlementReportJobScenarioFixture : LazyFixtureBase - where TScenarioState : new() -{ - public SettlementReportJobScenarioFixture(IMessageSink diagnosticMessageSink) - : base(diagnosticMessageSink) - { - Configuration = new SettlementReportJobScenarioConfiguration(); - ScenarioState = new TScenarioState(); - } - - public TScenarioState ScenarioState { get; } - - public SettlementReportJobScenarioConfiguration Configuration { get; } - - /// - /// The actual client is not created until has been called by the base class. - /// - private DatabricksClient DatabricksClient { get; set; } = null!; - - /// - /// The actual client is not created until has been called by the base class. - /// - private FilesDatabricksClient FilesDatabricksClient { get; set; } = null!; - - public async Task CancelSettlementReportJobRunsAsync(IReadOnlyCollection jobRunIds) - { - foreach (var jobRunId in jobRunIds) - { - try - { - await DatabricksClient.Jobs.RunsCancel(jobRunId); - } - catch - { - } - } - } - - public async Task StartSettlementReportJobRunAsync(Guid reportId, SettlementReportJobName jobName, IReadOnlyCollection jobParameters) - { - var jobNameAsString = jobName.ToString(); - var settlementReportJobId = await DatabricksClient.GetJobIdAsync(jobNameAsString); - var runParameters = RunParameters.CreatePythonParams(jobParameters); - - var runId = await DatabricksClient - .Jobs - .RunNow(settlementReportJobId, runParameters); - - DiagnosticMessageSink.WriteDiagnosticMessage($"'{jobNameAsString}' for '{reportId}' with run id '{runId}' started."); - - return runId; - } - - public async Task<(bool IsCompleted, Run? Run)> WaitForSettlementReportJobRunCompletedAsync( - long runId, - TimeSpan waitTimeLimit) - { - var delay = TimeSpan.FromMinutes(1); - - (Run, RepairHistory) runState = default; - SettlementReportJobState? settlementReportJobState = SettlementReportJobState.Pending; - var isCondition = await Awaiter.TryWaitUntilConditionAsync( - async () => - { - runState = await DatabricksClient.Jobs.RunsGet(runId); - settlementReportJobState = ConvertToSettlementReportJobState(runState.Item1); - - return - settlementReportJobState is SettlementReportJobState.Completed - or SettlementReportJobState.Failed - or SettlementReportJobState.Canceled; - }, - waitTimeLimit, - delay); - - DiagnosticMessageSink.WriteDiagnosticMessage($"Wait for 'SettlementReportJob' with run id '{runId}' completed with '{nameof(isCondition)}={isCondition}' and '{nameof(settlementReportJobState)}={settlementReportJobState}'."); - - return (settlementReportJobState == SettlementReportJobState.Completed, runState.Item1); - } - - /// - /// Get file information for at file in the Databricks Catalogue. - /// - /// File path relative to the Databricks Catalogue root configured per environment. - /// File information if file exists; otherwise null. - public async Task GetFileInfoAsync(string relativeFilePath) - { - try - { - return await FilesDatabricksClient.Files.GetFileInfoAsync(GetAbsolutePath(relativeFilePath)); - } - catch (Exception ex) - { - DiagnosticMessageSink.WriteDiagnosticMessage($"File exists failed with exception: {ex}."); - return null; - } - } - - /// - /// Get a file stream for a file in the Databricks Catalogue. - /// - /// File path relative to the Databricks Catalogue root configured per environment. - /// A stream to the file if it exists; otherwise null. - public async Task GetFileStreamAsync(string relativeFilePath) - { - try - { - return await FilesDatabricksClient.Files.GetFileStreamAsync(GetAbsolutePath(relativeFilePath)); - } - catch (Exception ex) - { - DiagnosticMessageSink.WriteDiagnosticMessage($"Get file stream failed with exception: {ex}."); - return null; - } - } - - protected override Task OnInitializeAsync() - { - DatabricksClient = DatabricksClient.CreateClient(Configuration.DatabricksWorkspace.BaseUrl, Configuration.DatabricksWorkspace.Token); - FilesDatabricksClient = new FilesDatabricksClient(Configuration.DatabricksWorkspace.BaseUrl, Configuration.DatabricksWorkspace.Token); - - return Task.CompletedTask; - } - - protected override Task OnDisposeAsync() - { - DatabricksClient.Dispose(); - FilesDatabricksClient.Dispose(); - - return Task.CompletedTask; - } - - private static SettlementReportJobState ConvertToSettlementReportJobState(Run run) - { - return run.State.LifeCycleState switch - { - RunLifeCycleState.PENDING => SettlementReportJobState.Pending, - RunLifeCycleState.QUEUED => SettlementReportJobState.Queued, - RunLifeCycleState.RUNNING => SettlementReportJobState.Running, - RunLifeCycleState.TERMINATING => SettlementReportJobState.Running, - RunLifeCycleState.SKIPPED => SettlementReportJobState.Canceled, - RunLifeCycleState.INTERNAL_ERROR => SettlementReportJobState.Failed, - RunLifeCycleState.TERMINATED => run.State.ResultState switch - { - RunResultState.SUCCESS => SettlementReportJobState.Completed, - RunResultState.FAILED => SettlementReportJobState.Failed, - RunResultState.CANCELED => SettlementReportJobState.Canceled, - RunResultState.TIMEDOUT => SettlementReportJobState.Canceled, - _ => throw new ArgumentOutOfRangeException(nameof(run.State)), - }, - _ => throw new ArgumentOutOfRangeException(nameof(run.State)), - }; - } - - private string GetAbsolutePath(string relativeFilePath) - { - return $"{Configuration.DatabricksCatalogRoot}{relativeFilePath}"; - } -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportWholesaleCalculationsJobCollectionDefinition.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportWholesaleCalculationsJobCollectionDefinition.cs deleted file mode 100644 index ff6f3e2318..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/Fixtures/SettlementReportWholesaleCalculationsJobCollectionDefinition.cs +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Xunit; - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures; - -/// -/// A xUnit collection fixture for ensuring tests using the "Settlement Report Wholesale Calculations Job" don't run in parallel. -/// -/// xUnit documentation of collection fixtures: -/// * https://xunit.net/docs/shared-context#collection-fixture -/// -[CollectionDefinition(nameof(SettlementReportWholesaleCalculationsJobCollectionDefinition))] -public sealed class SettlementReportWholesaleCalculationsJobCollectionDefinition -{ -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/SettlementReportBalanceFixingJobGeneratesZipScenario.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/SettlementReportBalanceFixingJobGeneratesZipScenario.cs deleted file mode 100644 index 5c8857c098..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/SettlementReportBalanceFixingJobGeneratesZipScenario.cs +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures; -using Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.States; -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.Attributes; -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.LazyFixture; -using FluentAssertions; -using FluentAssertions.Execution; -using Xunit; - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports; - -[ExecutionContext(AzureEnvironment.AllDev)] -[TestCaseOrderer( - ordererTypeName: "Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.Orderers.ScenarioStepOrderer", - ordererAssemblyName: "Energinet.DataHub.Wholesale.SubsystemTests")] -public class SettlementReportBalanceFixingJobGeneratesZipScenario : SubsystemTestsBase> -{ - public SettlementReportBalanceFixingJobGeneratesZipScenario(LazyFixtureFactory> lazyFixtureFactory) - : base(lazyFixtureFactory) - { - } - - [ScenarioStep(0)] - [SubsystemFact] - public void Given_ScenarioSetup() - { - // Input - Fixture.ScenarioState.ReportId = Guid.NewGuid(); - Fixture.ScenarioState.JobName = SettlementReportJobName.SettlementReportBalanceFixing; - Fixture.ScenarioState.JobParameters = new[] - { - $"--report-id={Fixture.ScenarioState.ReportId}", - "--period-start=2022-01-11T23:00:00Z", - "--period-end=2022-01-12T23:00:00Z", - "--calculation-type=balance_fixing", - "--requesting-actor-market-role=datahub_administrator", - "--requesting-actor-id=1234567890123", - "--include-basis-data", - "--grid-area-codes=[543]", - }; - - // Expectations - Fixture.ScenarioState.ExpectedJobTimeLimit = TimeSpan.FromMinutes(20); - Fixture.ScenarioState.ExpectedRelativeOutputFilePath = - $"/wholesale_settlement_report_output/settlement_reports/{Fixture.ScenarioState.ReportId}.zip"; - } - - [ScenarioStep(1)] - [SubsystemFact] - public async Task When_JobIsStarted() - { - Fixture.ScenarioState.JobRunId = await Fixture.StartSettlementReportJobRunAsync( - Fixture.ScenarioState.ReportId, - Fixture.ScenarioState.JobName, - Fixture.ScenarioState.JobParameters); - - // Assert - Fixture.ScenarioState.JobRunId.Should().BePositive(); - } - - /// - /// In this step we focus on completing the job with a certain 'wait time'. - /// This is not an exact time for how long it took to perform the job, - /// but the time it took for our retry loop to determine that the job has completed. - /// - [ScenarioStep(2)] - [SubsystemFact] - public async Task Then_JobIsCompletedWithinWaitTime() - { - var (isCompleted, run) = await Fixture.WaitForSettlementReportJobRunCompletedAsync( - Fixture.ScenarioState.JobRunId, - waitTimeLimit: Fixture.ScenarioState.ExpectedJobTimeLimit.Add(TimeSpan.FromMinutes(5))); - - Fixture.ScenarioState.Run = run; - - // Assert - using var assertionScope = new AssertionScope(); - isCompleted.Should().BeTrue(); - run.Should().NotBeNull(); - } - - [ScenarioStep(3)] - [SubsystemFact] - public async Task AndThen_OutputFileIsGeneratedAtExpectedLocation() - { - var outputFileInfo = await Fixture.GetFileInfoAsync(Fixture.ScenarioState.ExpectedRelativeOutputFilePath); - - // Assert - outputFileInfo.Should().NotBeNull($"because we expected the file (relative path) '{Fixture.ScenarioState.ExpectedRelativeOutputFilePath}' to exists."); - } - - [ScenarioStep(4)] - [SubsystemFact] - public async Task AndThen_ZipFileContainsCsvFilesWithExpectedPrefix() - { - var expectedFilePrefixes = new[] - { - "TSSD60", - "TSSD15", - "MDMP", - "RESULTENERGY", - }; - await using var zipStream = await Fixture.GetFileStreamAsync(Fixture.ScenarioState.ExpectedRelativeOutputFilePath); - - zipStream.Should().NotBeNull(); - - using var archive = new System.IO.Compression.ZipArchive(zipStream!, System.IO.Compression.ZipArchiveMode.Read); - - var entryFound = expectedFilePrefixes.Any(prefix => archive.Entries.Any(e => e.Name.StartsWith(prefix))); - entryFound.Should().BeTrue($"because we expected the zip file to contain a file with a name starting with one of the specified prefixes: {string.Join(", ", expectedFilePrefixes)}"); - } - - /// - /// In this step we verify the 'duration' of the job is within our 'performance goal'. - /// - [ScenarioStep(5)] - [SubsystemFact] - public void AndThen_JobDurationIsLessThanOrEqualToTimeLimit() - { - var actualCalculationJobDuration = - Fixture.ScenarioState.Run.EndTime - Fixture.ScenarioState.Run.StartTime; - - // Assert - using var assertionScope = new AssertionScope(); - actualCalculationJobDuration.Should().BeGreaterThan(TimeSpan.Zero); - actualCalculationJobDuration.Should().BeLessThanOrEqualTo(Fixture.ScenarioState.ExpectedJobTimeLimit); - } -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/SettlementReportWholesaleCalculationsJobGeneratesZipScenario.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/SettlementReportWholesaleCalculationsJobGeneratesZipScenario.cs deleted file mode 100644 index c02329c249..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/SettlementReportWholesaleCalculationsJobGeneratesZipScenario.cs +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures; -using Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.States; -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.Attributes; -using Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.LazyFixture; -using FluentAssertions; -using FluentAssertions.Execution; -using Xunit; - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports; - -[Collection(nameof(SettlementReportWholesaleCalculationsJobCollectionDefinition))] -[ExecutionContext(AzureEnvironment.AllDev)] -[TestCaseOrderer( - ordererTypeName: "Energinet.DataHub.Wholesale.SubsystemTests.Fixtures.Orderers.ScenarioStepOrderer", - ordererAssemblyName: "Energinet.DataHub.Wholesale.SubsystemTests")] -public class SettlementReportWholesaleCalculationsJobGeneratesZipScenario : SubsystemTestsBase> -{ - public SettlementReportWholesaleCalculationsJobGeneratesZipScenario(LazyFixtureFactory> lazyFixtureFactory) - : base(lazyFixtureFactory) - { - } - - [ScenarioStep(0)] - [SubsystemFact] - public void Given_ScenarioSetup() - { - // Input - Fixture.ScenarioState.ReportId = Guid.NewGuid(); - Fixture.ScenarioState.JobName = SettlementReportJobName.SettlementReportWholesaleCalculations; - Fixture.ScenarioState.JobParameters = new[] - { - $"--report-id={Fixture.ScenarioState.ReportId}", - "--period-start=2023-01-31T23:00:00Z", - "--period-end=2023-02-28T23:00:00Z", - "--calculation-type=wholesale_fixing", - "--requesting-actor-market-role=datahub_administrator", - "--requesting-actor-id=1234567890123", - "--include-basis-data", - $"--calculation-id-by-grid-area={{\"804\": \"{Fixture.Configuration.InputCalculationId}\"}}", - }; - - // Expectations - Fixture.ScenarioState.ExpectedJobTimeLimit = TimeSpan.FromMinutes(20); - Fixture.ScenarioState.ExpectedRelativeOutputFilePath = - $"/wholesale_settlement_report_output/settlement_reports/{Fixture.ScenarioState.ReportId}.zip"; - } - - [ScenarioStep(1)] - [SubsystemFact] - public async Task When_JobIsStarted() - { - Fixture.ScenarioState.JobRunId = await Fixture.StartSettlementReportJobRunAsync( - Fixture.ScenarioState.ReportId, - Fixture.ScenarioState.JobName, - Fixture.ScenarioState.JobParameters); - - // Assert - Fixture.ScenarioState.JobRunId.Should().BePositive(); - } - - /// - /// In this step we focus on completing the job with a certain 'wait time'. - /// This is not an exact time for how long it took to perform the job, - /// but the time it took for our retry loop to determine that the job has completed. - /// - [ScenarioStep(2)] - [SubsystemFact] - public async Task Then_JobIsCompletedWithinWaitTime() - { - var (isCompleted, run) = await Fixture.WaitForSettlementReportJobRunCompletedAsync( - Fixture.ScenarioState.JobRunId, - waitTimeLimit: Fixture.ScenarioState.ExpectedJobTimeLimit.Add(TimeSpan.FromMinutes(5))); - - Fixture.ScenarioState.Run = run; - - // Assert - using var assertionScope = new AssertionScope(); - isCompleted.Should().BeTrue(); - run.Should().NotBeNull(); - } - - [ScenarioStep(3)] - [SubsystemFact] - public async Task AndThen_OutputFileIsGeneratedAtExpectedLocation() - { - var outputFileInfo = await Fixture.GetFileInfoAsync(Fixture.ScenarioState.ExpectedRelativeOutputFilePath); - - // Assert - outputFileInfo.Should().NotBeNull($"because we expected the file (relative path) '{Fixture.ScenarioState.ExpectedRelativeOutputFilePath}' to exists."); - } - - [ScenarioStep(4)] - [SubsystemFact] - public async Task AndThen_ZipFileContainsCsvFilesWithExpectedPrefix() - { - var expectedFilePrefixes = new[] - { - "TSSD60", - "TSSD15", - "MDMP", - "CHARGELINK", - "CHARGEPRICE", - "RESULTENERGY", - "RESULTWHOLESALE", - "RESULTMONTHLY", - }; - await using var zipStream = await Fixture.GetFileStreamAsync(Fixture.ScenarioState.ExpectedRelativeOutputFilePath); - - zipStream.Should().NotBeNull(); - - using var archive = new System.IO.Compression.ZipArchive(zipStream!, System.IO.Compression.ZipArchiveMode.Read); - - var entryFound = expectedFilePrefixes.Any(prefix => archive.Entries.Any(e => e.Name.StartsWith(prefix))); - entryFound.Should().BeTrue($"because we expected the zip file to contain a file with a name starting with one of the specified prefixes: {string.Join(", ", expectedFilePrefixes)}"); - } - - /// - /// In this step we verify the 'duration' of the job is within our 'performance goal'. - /// - [ScenarioStep(5)] - [SubsystemFact] - public void AndThen_JobDurationIsLessThanOrEqualToTimeLimit() - { - var actualCalculationJobDuration = - Fixture.ScenarioState.Run.EndTime - Fixture.ScenarioState.Run.StartTime; - - // Assert - using var assertionScope = new AssertionScope(); - actualCalculationJobDuration.Should().BeGreaterThan(TimeSpan.Zero); - actualCalculationJobDuration.Should().BeLessThanOrEqualTo(Fixture.ScenarioState.ExpectedJobTimeLimit); - } -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/States/GeneratesZipScenarioState.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/States/GeneratesZipScenarioState.cs deleted file mode 100644 index 54df01d569..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/States/GeneratesZipScenarioState.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System.Diagnostics.CodeAnalysis; -using Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.Fixtures; -using Microsoft.Azure.Databricks.Client.Models; - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.States; - -public class GeneratesZipScenarioState -{ - public Guid ReportId { get; set; } - - public SettlementReportJobName JobName { get; set; } - - [NotNull] - public IReadOnlyCollection? JobParameters { get; set; } - - /// - /// The expected max. duration of the job. - /// Use this to monitor (set expectations for) the performance of the job. - /// - public TimeSpan ExpectedJobTimeLimit { get; set; } - - [NotNull] - public string? ExpectedRelativeOutputFilePath { get; set; } - - public long JobRunId { get; set; } - - [NotNull] - public Run? Run { get; set; } -} diff --git a/source/dotnet/subsystem-tests/Features/SettlementReports/States/SettlementReportJobState.cs b/source/dotnet/subsystem-tests/Features/SettlementReports/States/SettlementReportJobState.cs deleted file mode 100644 index d604035a43..0000000000 --- a/source/dotnet/subsystem-tests/Features/SettlementReports/States/SettlementReportJobState.cs +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2020 Energinet DataHub A/S -// -// Licensed under the Apache License, Version 2.0 (the "License2"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -namespace Energinet.DataHub.Wholesale.SubsystemTests.Features.SettlementReports.States; - -public enum SettlementReportJobState -{ - Pending, - Queued, - Running, - Completed, - Canceled, - Failed, -}