From 7fdba58708b4edc7fee79fc2e82f0a4ecdbf1f2d Mon Sep 17 00:00:00 2001 From: Steffen Froehlich Date: Tue, 16 Jul 2024 11:46:52 -0700 Subject: [PATCH] Release/4.2.0 (#28) * Release 4.2.0 * release 4.2.0 * fixing workflow --- .github/workflows/dotnet-build.yml | 9 - .github/workflows/python-test.yml | 9 - .github/workflows/sdk-workflow.yml | 2 + .gitignore | 1 + Directory.Build.props | 2 +- Migration SDK.sln | 21 +- .../MyMigrationApplication.cs | 51 +++- .../Python.ExampleApplication.py | 39 +++- .../requirements.txt | 1 + .../articles/configuration/basic.md | 40 ---- src/Documentation/articles/troubleshooting.md | 21 ++ src/Python/Python.pyproj | 1 - src/Python/pytest.ini | 4 +- src/Python/requirements.txt | 2 + src/Python/scripts/build_tests.py | 9 - src/Python/src/tableau_migration/__init__.py | 2 + src/Python/src/tableau_migration/migration.py | 2 +- .../migration_engine_manifest.py | 51 ++++ src/Python/tests/test_classes.py | 6 +- .../tests/test_migration_engine_manifest.py | 35 +++ src/Python/tests/test_other.py | 14 +- .../Api/DataSourcesApiClient.cs | 14 +- .../Api/IDataSourcesApiClient.cs | 4 +- .../Api/IWorkbooksApiClient.cs | 4 +- .../Api/Models/FailedJobException.cs | 44 +++- src/Tableau.Migration/Api/Models/IJob.cs | 3 +- src/Tableau.Migration/Api/Models/Job.cs | 52 ++++- .../Api/Models/StatusNote.cs | 3 +- .../Api/Models/TimeoutJobException.cs | 42 +++- .../Api/Rest/RestException.cs | 60 ++++- .../Net/Responses/BuildResponseException.cs | 27 ++- .../Api/WorkbooksApiClient.cs | 11 +- .../Config/ContentTypesOptions.cs | 17 ++ .../Engine/IServiceCollectionExtensions.cs | 3 + .../Engine/Manifest/MigrationManifest.cs | 10 +- .../Engine/Manifest/MigrationManifestEntry.cs | 29 +-- .../Manifest/MigrationManifestSerializer.cs | 64 +++-- src/Tableau.Migration/ExceptionComparer.cs | 74 ++++++ .../IEnumerableExtensions.cs | 1 + .../IServiceCollectionExtensions.cs | 1 + .../BuildResponseExceptionJsonConverter.cs | 97 ++++++++ .../JsonConverters/Constants.cs | 4 +- .../JsonConverters/ExceptionJsonConverter.cs | 87 +++++++ .../ExceptionJsonConverterFactory.cs | 42 ++++ .../Exceptions/MismatchException.cs | 8 +- .../FailedJobExceptionJsonConverter.cs | 73 ++++++ .../JsonConverters/JobJsonConverter.cs | 175 ++++++++++++++ .../JsonConverters/JsonReaderUtils.cs | 104 +++++++++ .../JsonConverters/JsonWriterUtils.cs | 48 ++++ .../PythonExceptionConverter.cs | 106 +++++++++ .../RestExceptionJsonConverter.cs | 145 ++++++++++++ .../SerializableContentLocation.cs | 58 ++++- .../SerializableContentReference.cs | 93 ++++++++ .../SerializableEntryCollection.cs | 50 ++++ .../SerializableException.cs | 47 ++++ .../SerializableManifestEntry.cs | 147 ++++++++++++ .../SerializableMigrationManifest.cs | 129 ++++++++++ .../SerializedExceptionJsonConverter.cs | 152 ++++++++++++ .../TimeoutJobExceptionJsonConverter.cs | 83 +++++++ .../Tableau.Migration.csproj | 7 +- .../Python.ExampleApplication.Tests.pyproj | 60 +++++ .../Python.ExampleApplication.Tests/README.md | 11 + .../pyproject.toml | 35 +++ .../pytest.ini | 6 + .../requirements.txt | 2 + .../tests/__init__.py | 61 +++++ .../tests/test_default_project_filter.py | 41 ++++ .../tests/test_log_migration_batches_hook.py | 24 ++ .../Python.TestApplication.pyproj | 6 - tests/Python.TestApplication/build.py | 2 - tests/Python.TestApplication/helper.py | 3 - tests/Python.TestApplication/main.py | 13 +- ...igration_testcomponents_engine_manifest.py | 78 ------- .../migration_testcomponents_hooks.py | 4 +- tests/Python.TestApplication/pyproject.toml | 2 +- .../testapplication_tests/__init__.py | 36 --- .../testapplication_tests/test_classes.py | 221 ------------------ ...anifestAfterBatchMigrationCompletedHook.cs | 2 +- .../Program.cs | 2 - .../Tableau.Migration.TestApplication.csproj | 1 - .../TestApplication.cs | 13 +- .../AutoFixtureTestBase.cs | 157 ------------- .../MockServiceProvider.cs | 31 --- ...leau.Migration.TestComponents.Tests.csproj | 36 --- .../Usings.cs | 18 -- .../IServiceCollectionExtensions.cs | 39 ---- .../JsonConverters/ExceptionJsonConverter.cs | 99 -------- .../JsonObjects/JsonContentReference.cs | 59 ----- .../JsonObjects/JsonManifestEntry.cs | 72 ------ .../MigrationManifestEntryCollectionReader.cs | 177 -------------- .../MigrationManifestEntryCollectionWriter.cs | 61 ----- .../SerializeableMigrationManifest.cs | 35 --- .../Tableau.Migration.TestComponents.csproj | 18 -- .../AutoFixtureTestBase.cs | 6 + .../AutoFixtureTestBaseTests.cs | 72 ++++++ .../ExceptionComparerTests.cs | 123 ++++++++++ .../Tableau.Migration.Tests/FixtureFactory.cs | 168 +++++++++++++ .../ObjectExtensions.cs | 10 + .../Tests/Api/WorkbooksApiClientTests.cs | 2 +- .../Tests/IncrementalMigrationTests.cs | 2 +- .../Tableau.Migration.Tests.csproj | 8 +- .../Unit/Api/DataSourcesApiClientTests.cs | 15 +- .../Unit/Api/WorkbooksApiClientTests.cs | 15 +- .../Unit/Config/ConfigReaderTests.cs | 28 ++- .../Content/Schedules/VolatileCacheTests.cs | 15 +- ...udScheduleCompatibilityTransformerTests.cs | 2 +- .../MigrationManifestEntryCollectionTests.cs | 6 +- .../Manifest/MigrationManifestEntryTests.cs | 85 ++++--- .../Manifest/MigrationManifestFactoryTests.cs | 5 + .../TestMigrationManifestSerializer.cs | 22 +- .../TestSerializableContentLocation.cs} | 26 ++- .../TestSerializableContentReference.cs} | 19 +- .../TestSerializableManifestEntry.cs} | 25 +- .../SerializedExceptionJsonConverterTests.cs | 122 ++++++++++ ...LoggingServiceCollectionExtensionsTests.cs | 2 +- 115 files changed, 3196 insertions(+), 1447 deletions(-) delete mode 100644 src/Documentation/articles/configuration/basic.md delete mode 100644 src/Python/scripts/build_tests.py rename {tests/Tableau.Migration.TestComponents => src/Tableau.Migration}/Engine/Manifest/MigrationManifestSerializer.cs (66%) create mode 100644 src/Tableau.Migration/ExceptionComparer.cs create mode 100644 src/Tableau.Migration/JsonConverters/BuildResponseExceptionJsonConverter.cs rename {tests/Tableau.Migration.TestComponents => src/Tableau.Migration}/JsonConverters/Constants.cs (84%) create mode 100644 src/Tableau.Migration/JsonConverters/ExceptionJsonConverter.cs create mode 100644 src/Tableau.Migration/JsonConverters/ExceptionJsonConverterFactory.cs rename {tests/Tableau.Migration.TestComponents => src/Tableau.Migration}/JsonConverters/Exceptions/MismatchException.cs (84%) create mode 100644 src/Tableau.Migration/JsonConverters/FailedJobExceptionJsonConverter.cs create mode 100644 src/Tableau.Migration/JsonConverters/JobJsonConverter.cs create mode 100644 src/Tableau.Migration/JsonConverters/JsonReaderUtils.cs create mode 100644 src/Tableau.Migration/JsonConverters/JsonWriterUtils.cs create mode 100644 src/Tableau.Migration/JsonConverters/PythonExceptionConverter.cs create mode 100644 src/Tableau.Migration/JsonConverters/RestExceptionJsonConverter.cs rename tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonContentLocation.cs => src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentLocation.cs (52%) create mode 100644 src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentReference.cs create mode 100644 src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableEntryCollection.cs create mode 100644 src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableException.cs create mode 100644 src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableManifestEntry.cs create mode 100644 src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableMigrationManifest.cs create mode 100644 src/Tableau.Migration/JsonConverters/SerializedExceptionJsonConverter.cs create mode 100644 src/Tableau.Migration/JsonConverters/TimeoutJobExceptionJsonConverter.cs create mode 100644 tests/Python.ExampleApplication.Tests/Python.ExampleApplication.Tests.pyproj create mode 100644 tests/Python.ExampleApplication.Tests/README.md create mode 100644 tests/Python.ExampleApplication.Tests/pyproject.toml create mode 100644 tests/Python.ExampleApplication.Tests/pytest.ini create mode 100644 tests/Python.ExampleApplication.Tests/requirements.txt create mode 100644 tests/Python.ExampleApplication.Tests/tests/__init__.py create mode 100644 tests/Python.ExampleApplication.Tests/tests/test_default_project_filter.py create mode 100644 tests/Python.ExampleApplication.Tests/tests/test_log_migration_batches_hook.py delete mode 100644 tests/Python.TestApplication/migration_testcomponents_engine_manifest.py delete mode 100644 tests/Python.TestApplication/testapplication_tests/__init__.py delete mode 100644 tests/Python.TestApplication/testapplication_tests/test_classes.py delete mode 100644 tests/Tableau.Migration.TestComponents.Tests/AutoFixtureTestBase.cs delete mode 100644 tests/Tableau.Migration.TestComponents.Tests/MockServiceProvider.cs delete mode 100644 tests/Tableau.Migration.TestComponents.Tests/Tableau.Migration.TestComponents.Tests.csproj delete mode 100644 tests/Tableau.Migration.TestComponents.Tests/Usings.cs delete mode 100644 tests/Tableau.Migration.TestComponents/IServiceCollectionExtensions.cs delete mode 100644 tests/Tableau.Migration.TestComponents/JsonConverters/ExceptionJsonConverter.cs delete mode 100644 tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonContentReference.cs delete mode 100644 tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonManifestEntry.cs delete mode 100644 tests/Tableau.Migration.TestComponents/JsonConverters/MigrationManifestEntryCollectionReader.cs delete mode 100644 tests/Tableau.Migration.TestComponents/JsonConverters/MigrationManifestEntryCollectionWriter.cs delete mode 100644 tests/Tableau.Migration.TestComponents/JsonConverters/SerializeableMigrationManifest.cs delete mode 100644 tests/Tableau.Migration.TestComponents/Tableau.Migration.TestComponents.csproj create mode 100644 tests/Tableau.Migration.Tests/AutoFixtureTestBaseTests.cs create mode 100644 tests/Tableau.Migration.Tests/ExceptionComparerTests.cs rename tests/{Tableau.Migration.TestComponents.Tests => Tableau.Migration.Tests/Unit}/Engine/Manifest/TestMigrationManifestSerializer.cs (73%) rename tests/{Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonContentLocation.cs => Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentLocation.cs} (77%) rename tests/{Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonContentReference.cs => Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentReference.cs} (77%) rename tests/{Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonManifestEntry.cs => Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableManifestEntry.cs} (74%) create mode 100644 tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializedExceptionJsonConverterTests.cs diff --git a/.github/workflows/dotnet-build.yml b/.github/workflows/dotnet-build.yml index 3818c61..8a912e4 100644 --- a/.github/workflows/dotnet-build.yml +++ b/.github/workflows/dotnet-build.yml @@ -39,9 +39,6 @@ jobs: - name: Net Publish Tests ${{ matrix.config }} if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }} run: dotnet publish --no-build -p:DebugType=None -p:DebugSymbols=false -c ${{ matrix.config }} -f ${{ vars.PYTHON_NETPACKAGE_FRAMEWORK }} -o './dist/tests/' './tests/Tableau.Migration.Tests/Tableau.Migration.Tests.csproj' - - name: Net Publish TestComponents ${{ matrix.config }} - if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }} - run: dotnet publish --no-build -p:DebugType=None -p:DebugSymbols=false -c ${{ matrix.config }} -f ${{ vars.PYTHON_NETPACKAGE_FRAMEWORK }} -o './dist/testcomponents/' './tests/Tableau.Migration.TestComponents/Tableau.Migration.TestComponents.csproj' - name: Upload Published Artifacts uses: actions/upload-artifact@v4 if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }} @@ -54,12 +51,6 @@ jobs: with: name: tests-published-${{ matrix.config }} path: './dist/tests/**' - - name: Upload TestComponents Artifacts - uses: actions/upload-artifact@v4 - if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }} - with: - name: testcomponents-published-${{ matrix.config }} - path: './dist/testcomponents/**' - name: Upload Nupkg Artifact uses: actions/upload-artifact@v4 if: ${{ matrix.os == vars.PUBLISH_OS && matrix.config == 'Release' }} diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index afd36f9..18ab369 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -34,18 +34,9 @@ jobs: with: name: tests-published-${{ matrix.config }} path: ./src/Python/src/tableau_migration/bin/ - - uses: actions/download-artifact@v4 - with: - name: testcomponents-published-${{ matrix.config }} - path: ./tests/Python.TestApplication/bin/ - name: Lint with ruff run: python -m hatch run lint:lint - name: Test with pytest run: | python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:testcov - - - name: Test TestApplication with pytest - run: | - python -m hatch --data-dir=.hatch --cache-dir=.hatch_cache run test:test - working-directory: ./tests/Python.TestApplication diff --git a/.github/workflows/sdk-workflow.yml b/.github/workflows/sdk-workflow.yml index 948a822..a94aff9 100644 --- a/.github/workflows/sdk-workflow.yml +++ b/.github/workflows/sdk-workflow.yml @@ -7,6 +7,8 @@ on: branches: - main - 'release/**' + - 'staging/**' + - 'feature/**' paths-ignore: - README.md - '**/README.md' diff --git a/.gitignore b/.gitignore index c0e14ea..8543492 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ *.csproj.user .idea/ /.vscode +*.pyproj.user # Python stuff # Source: https://github.com/github/gitignore/blob/main/Python.gitignore diff --git a/Directory.Build.props b/Directory.Build.props index a5c1c23..b170f62 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,7 +4,7 @@ enable true true - 4.1.0 + 4.2.0 Salesforce, Inc. Salesforce, Inc. Copyright (c) 2024, Salesforce, Inc. and its licensors diff --git a/Migration SDK.sln b/Migration SDK.sln index 66d2025..f2d424c 100644 --- a/Migration SDK.sln +++ b/Migration SDK.sln @@ -37,16 +37,12 @@ Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "Python", "src\Python\Python EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tableau.Migration.TestApplication", "tests\Tableau.Migration.TestApplication\Tableau.Migration.TestApplication.csproj", "{9B20E6B0-E733-4280-9D63-0BAFA0A23276}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tableau.Migration.TestComponents", "tests\Tableau.Migration.TestComponents\Tableau.Migration.TestComponents.csproj", "{0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}" -EndProject Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "Python.TestApplication", "tests\Python.TestApplication\Python.TestApplication.pyproj", "{BCA30617-0CAF-405C-A26B-522CF20DB027}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Examples", "Examples", "{00B78B56-BF89-4F26-AF75-435A4B88D43F}" EndProject Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "Python.ExampleApplication", "examples\Python.ExampleApplication\Python.ExampleApplication.pyproj", "{9C94FBC9-AE67-4A26-BDDA-EB2CE9FE5C25}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tableau.Migration.TestComponents.Tests", "tests\Tableau.Migration.TestComponents.Tests\Tableau.Migration.TestComponents.Tests.csproj", "{A00DC50C-4184-4658-A437-00AD2A07412C}" -EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Csharp.ExampleApplication", "examples\Csharp.ExampleApplication\Csharp.ExampleApplication.csproj", "{B0D16449-BC58-4873-8B95-F62E805EE39C}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{92044843-B4D7-4062-B1D5-DE7596072E33}" @@ -85,7 +81,9 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "setup-dotnet", "setup-dotne .github\actions\setup-dotnet\action.yml = .github\actions\setup-dotnet\action.yml EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tableau.Migration.PythonGenerator", "src\Tableau.Migration.PythonGenerator\Tableau.Migration.PythonGenerator.csproj", "{F20029C7-4514-4668-8941-B2C3BC245CCB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tableau.Migration.PythonGenerator", "src\Tableau.Migration.PythonGenerator\Tableau.Migration.PythonGenerator.csproj", "{F20029C7-4514-4668-8941-B2C3BC245CCB}" +EndProject +Project("{888888A0-9F3D-457C-B088-3A5042F75D52}") = "Python.ExampleApplication.Tests", "tests\Python.ExampleApplication.Tests\Python.ExampleApplication.Tests.pyproj", "{B1884017-8E25-4A26-8C89-D9D880CFA392}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -109,18 +107,10 @@ Global {9B20E6B0-E733-4280-9D63-0BAFA0A23276}.Debug|Any CPU.Build.0 = Debug|Any CPU {9B20E6B0-E733-4280-9D63-0BAFA0A23276}.Release|Any CPU.ActiveCfg = Release|Any CPU {9B20E6B0-E733-4280-9D63-0BAFA0A23276}.Release|Any CPU.Build.0 = Release|Any CPU - {0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}.Debug|Any CPU.Build.0 = Debug|Any CPU - {0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}.Release|Any CPU.ActiveCfg = Release|Any CPU - {0454E8DE-D973-4E8F-95C7-9F6F02EF26FD}.Release|Any CPU.Build.0 = Release|Any CPU {BCA30617-0CAF-405C-A26B-522CF20DB027}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {BCA30617-0CAF-405C-A26B-522CF20DB027}.Release|Any CPU.ActiveCfg = Release|Any CPU {9C94FBC9-AE67-4A26-BDDA-EB2CE9FE5C25}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {9C94FBC9-AE67-4A26-BDDA-EB2CE9FE5C25}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A00DC50C-4184-4658-A437-00AD2A07412C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {A00DC50C-4184-4658-A437-00AD2A07412C}.Debug|Any CPU.Build.0 = Debug|Any CPU - {A00DC50C-4184-4658-A437-00AD2A07412C}.Release|Any CPU.ActiveCfg = Release|Any CPU - {A00DC50C-4184-4658-A437-00AD2A07412C}.Release|Any CPU.Build.0 = Release|Any CPU {B0D16449-BC58-4873-8B95-F62E805EE39C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {B0D16449-BC58-4873-8B95-F62E805EE39C}.Debug|Any CPU.Build.0 = Debug|Any CPU {B0D16449-BC58-4873-8B95-F62E805EE39C}.Release|Any CPU.ActiveCfg = Release|Any CPU @@ -133,6 +123,8 @@ Global {F20029C7-4514-4668-8941-B2C3BC245CCB}.Debug|Any CPU.Build.0 = Debug|Any CPU {F20029C7-4514-4668-8941-B2C3BC245CCB}.Release|Any CPU.ActiveCfg = Release|Any CPU {F20029C7-4514-4668-8941-B2C3BC245CCB}.Release|Any CPU.Build.0 = Release|Any CPU + {B1884017-8E25-4A26-8C89-D9D880CFA392}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B1884017-8E25-4A26-8C89-D9D880CFA392}.Release|Any CPU.ActiveCfg = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -141,10 +133,8 @@ Global {E71B1084-8907-4070-AD26-2189C80A2C34} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11} {1D53209C-9F3D-4F64-ABDF-FEA30F0C3D5D} = {C5BD8316-60C9-4C96-9B15-820E8BA4DF7F} {9B20E6B0-E733-4280-9D63-0BAFA0A23276} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11} - {0454E8DE-D973-4E8F-95C7-9F6F02EF26FD} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11} {BCA30617-0CAF-405C-A26B-522CF20DB027} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11} {9C94FBC9-AE67-4A26-BDDA-EB2CE9FE5C25} = {00B78B56-BF89-4F26-AF75-435A4B88D43F} - {A00DC50C-4184-4658-A437-00AD2A07412C} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11} {B0D16449-BC58-4873-8B95-F62E805EE39C} = {00B78B56-BF89-4F26-AF75-435A4B88D43F} {92044843-B4D7-4062-B1D5-DE7596072E33} = {C5BD8316-60C9-4C96-9B15-820E8BA4DF7F} {99DA12FB-BB16-4EE1-9C9C-047755210255} = {00B78B56-BF89-4F26-AF75-435A4B88D43F} @@ -152,6 +142,7 @@ Global {6B735E6E-1FFB-4C37-8CF6-BD979B4F8D9B} = {90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4} {EDD52CC0-7289-4167-A01E-E88B015FC67F} = {90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4} {454EF272-D967-4668-A20D-AD6B3EE96C1A} = {6B735E6E-1FFB-4C37-8CF6-BD979B4F8D9B} + {B1884017-8E25-4A26-8C89-D9D880CFA392} = {9BF466A2-E00F-4F2C-AD23-591E9159AD11} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {2C9E9FF4-E825-47A4-90BE-5499D5EDF3CC} diff --git a/examples/Csharp.ExampleApplication/MyMigrationApplication.cs b/examples/Csharp.ExampleApplication/MyMigrationApplication.cs index 840adcd..b2cb0bb 100644 --- a/examples/Csharp.ExampleApplication/MyMigrationApplication.cs +++ b/examples/Csharp.ExampleApplication/MyMigrationApplication.cs @@ -1,6 +1,8 @@ using System; using System.Diagnostics; +using System.IO; using System.Linq; +using System.Reflection; using System.Threading; using System.Threading.Tasks; using Csharp.ExampleApplication.Config; @@ -16,6 +18,7 @@ using Tableau.Migration; using Tableau.Migration.Content; using Tableau.Migration.Content.Schedules.Cloud; +using Tableau.Migration.Engine.Manifest; using Tableau.Migration.Engine.Pipelines; #region namespace @@ -30,13 +33,15 @@ internal sealed class MyMigrationApplication : IHostedService private readonly IMigrator _migrator; private readonly MyMigrationApplicationOptions _options; private readonly ILogger _logger; + private readonly MigrationManifestSerializer _manifestSerializer; public MyMigrationApplication( IHostApplicationLifetime appLifetime, IMigrationPlanBuilder planBuilder, IMigrator migrator, IOptions options, - ILogger logger) + ILogger logger, + MigrationManifestSerializer manifestSerializer) { _timer = new Stopwatch(); @@ -49,10 +54,19 @@ public MyMigrationApplication( _migrator = migrator; _options = options.Value; _logger = logger; + _manifestSerializer = manifestSerializer; } public async Task StartAsync(CancellationToken cancel) { + var executablePath = Assembly.GetExecutingAssembly().Location; + var currentFolder = Path.GetDirectoryName(executablePath); + if (currentFolder is null) + { + throw new Exception("Could not get the current folder path."); + } + var manifestPath = $"{currentFolder}/manifest.json"; + var startTime = DateTime.UtcNow; _timer.Start(); @@ -130,7 +144,7 @@ public async Task StartAsync(CancellationToken cancel) _planBuilder.Hooks.Add(); #endregion - //Add batch migration completed hooks + // Add batch migration completed hooks #region LogMigrationBatchesHook-Registration _planBuilder.Hooks.Add>(); _planBuilder.Hooks.Add>(); @@ -139,14 +153,20 @@ public async Task StartAsync(CancellationToken cancel) _planBuilder.Hooks.Add>(); #endregion + // Load the previous manifest if possible + var prevManifest = await LoadManifest(manifestPath, cancel); + // Build the plan var plan = _planBuilder.Build(); // Execute the migration - var result = await _migrator.ExecuteAsync(plan, cancel); + var result = await _migrator.ExecuteAsync(plan, prevManifest, cancel); _timer.Stop(); + // Save the manifest + await _manifestSerializer.SaveAsync(result.Manifest, manifestPath); + PrintResult(result); _logger.LogInformation($"Migration Started: {startTime}"); @@ -206,6 +226,31 @@ private void PrintResult(MigrationResult result) } } } + + private async Task LoadManifest(string manifestFilepath, CancellationToken cancel) + { + var manifest = await _manifestSerializer.LoadAsync(manifestFilepath, cancel); + if (manifest is not null) + { + ConsoleKey key; + do + { + Console.Write($"Existing Manifest found at {manifestFilepath}. Should it be used? [Y/n] "); + key = Console.ReadKey().Key; + Console.WriteLine(); // make Console logs prettier + } while (key is not ConsoleKey.Enter && key is not ConsoleKey.Y && key is not ConsoleKey.N); + + if (key is ConsoleKey.N) + { + return null; + } + + _logger.LogInformation($"Using previous manifest from {manifestFilepath}"); + return manifest; + } + + return null; + } } } diff --git a/examples/Python.ExampleApplication/Python.ExampleApplication.py b/examples/Python.ExampleApplication/Python.ExampleApplication.py index a98fcd4..56e5183 100644 --- a/examples/Python.ExampleApplication/Python.ExampleApplication.py +++ b/examples/Python.ExampleApplication/Python.ExampleApplication.py @@ -2,6 +2,10 @@ # By default all supported content will be migrated, but can be modified to your specific needs. # The application assumes you have already installed the Tableau Migration SDK Python package. +from dotenv import load_dotenv + +load_dotenv() + import configparser # configuration parser import os # environment variables import sys # system utility @@ -10,9 +14,20 @@ from threading import Thread # threading +from tableau_migration import ( + MigrationManifestSerializer, + MigrationManifest + ) + +serializer = MigrationManifestSerializer() + def migrate(): """Performs a migration using Tableau Migration SDK.""" + # Get the absolute path of the current file + current_file_path = os.path.abspath(__file__) + manifest_path = os.path.join(os.path.dirname(current_file_path), 'manifest.json') + plan_builder = tableau_migration.MigrationPlanBuilder() migration = tableau_migration.Migrator() @@ -37,6 +52,10 @@ def migrate(): # TODO: add filters, mappings, transformers, etc. here. + + # Load the previous manifest file if it exists. + prev_manifest = load_manifest(f'{manifest_path}') + # Validate the migration plan. validation_result = plan_builder.validate() @@ -45,13 +64,31 @@ def migrate(): plan = plan_builder.build() # Run the migration. - results = migration.execute(plan) + results = migration.execute(plan, prev_manifest) + # Save the manifest file. + serializer.save(results.manifest, f'{manifest_path}') + # TODO: Handle results here. print_result(results) print("All done.") +def load_manifest(manifest_path: str) -> MigrationManifest | None: + """Loads a manifest if requested.""" + manifest = serializer.load(manifest_path) + + if manifest is not None: + while True: + answer = input(f'Existing Manifest found at {manifest_path}. Should it be used? [Y/n] ').upper() + + if answer == 'N': + return None + elif answer == 'Y' or answer == '': + return manifest + + return None + if __name__ == '__main__': diff --git a/examples/Python.ExampleApplication/requirements.txt b/examples/Python.ExampleApplication/requirements.txt index b93e298..b7b0300 100644 --- a/examples/Python.ExampleApplication/requirements.txt +++ b/examples/Python.ExampleApplication/requirements.txt @@ -5,3 +5,4 @@ cffi==1.16.0 pycparser==2.22 pythonnet==3.0.3 typing_extensions==4.12.2 +python-dotenv==1.0.1 diff --git a/src/Documentation/articles/configuration/basic.md b/src/Documentation/articles/configuration/basic.md deleted file mode 100644 index 883267a..0000000 --- a/src/Documentation/articles/configuration/basic.md +++ /dev/null @@ -1,40 +0,0 @@ -# Basic Configuration - -## Migration Plan - -The migration plan is a required input in the migration process. It defines the Source and Destination servers and the hooks executed during the migration. Consider the Migration Plan as the steps the Migration SDK follows to migrate the information from a given source to a given destination. - -The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines the Migration Plan structure. The easiest way to generate a new Migration Plan is using [`MigrationPlanBuilder`](xref:Tableau.Migration.Engine.MigrationPlanBuilder). Following are the steps needed before [building](#build) a new plan: - -- [Define the required Source server](#source). -- [Define the required Destination server](#destination). -- [Define the required Migration Type](#migration-type). - -### Source - - The method [`MigrationPlanBuilder.FromSourceTableauServer`](xref:Tableau.Migration.Engine.MigrationPlanBuilder.FromSourceTableauServer(System.Uri,System.String,System.String,System.String,System.Boolean)) defines the source server by instantiating a new [`TableauSiteConnectionConfiguration`](xref:Tableau.Migration.Api.TableauSiteConnectionConfiguration) with the following parameters: - -- **serverUrl** -- **siteContentUrl:**(optional) -- **accessTokenName** -- **accessToken** - -### Destination - -The method [`MigrationPlanBuilder.ToDestinationTableauCloud`](xref:Tableau.Migration.Engine.MigrationPlanBuilder.ToDestinationTableauCloud(System.Uri,System.String,System.String,System.String,System.Boolean)) defines the destination server by instantiating a new [`TableauSiteConnectionConfiguration`](xref:Tableau.Migration.Api.TableauSiteConnectionConfiguration) with the following parameters: - -- **podUrl** -- **siteContentUrl** The site name on Tableau Cloud. -- **accessTokenName** -- **accessToken** - -> [!Important] -> Personal access tokens (PATs) are long-lived authentication tokens that allow you to sign in to the Tableau REST API without requiring hard-coded credentials or interactive sign-in. Revoke and generate a new PAT every day to keep your server secure. Access tokens should not be stored in plain text in application configuration files, and should instead use secure alternatives such as encryption or a secrets management system. If the source and destination sites are on the same server, separate PATs should be used. - -### Migration Type - -The method [`MigrationPlanBuilder.ForServerToCloud`](xref:Tableau.Migration.Engine.MigrationPlanBuilder.ForServerToCloud) will define the migration type and load all default hooks for a **Server to Cloud** migration. - -### Build - -The method [`MigrationPlanBuilder.Build`](xref:Tableau.Migration.Engine.MigrationPlanBuilder.Build) generates a Migration Plan ready to be used as an input to a migration process. \ No newline at end of file diff --git a/src/Documentation/articles/troubleshooting.md b/src/Documentation/articles/troubleshooting.md index ff35f23..5aa0a6a 100644 --- a/src/Documentation/articles/troubleshooting.md +++ b/src/Documentation/articles/troubleshooting.md @@ -61,6 +61,27 @@ Python _logger.LogInformation(f"{content_type.Name} {entry.Source.Location} migrated to {entry.Destination.Location}") ``` +### Python - The SDK isn't loading the *.env* configuration + +Environment variables must be set in the system the Python application runs in. This can be done through the OS itself, or by 3rd party libraries. The SDK will load the environment configuration on its **\_\_init\_\_** process. + +For the case of the library [dotenv](https://pypi.org/project/python-dotenv/), it is required to execute the command **load_dotenv()** before referring to any **tableau_migration** code. + +``` +# Used to load environment variables +from dotenv import load_dotenv + +# Load the environment variables before importing tableau_migration +load_dotenv() + +# first tableau_migration reference +import tableau_migration + +# The SDK will not recognize the .env file values +# Don't load the values here +# load_dotenv() +``` + ## Errors and Warnings This section provides a list of potential error and warning log messages that you may encounter in the logs. Each entry includes a description to assist you in debugging. diff --git a/src/Python/Python.pyproj b/src/Python/Python.pyproj index 63c2223..a073df7 100644 --- a/src/Python/Python.pyproj +++ b/src/Python/Python.pyproj @@ -27,7 +27,6 @@ - diff --git a/src/Python/pytest.ini b/src/Python/pytest.ini index 7463554..386edc6 100644 --- a/src/Python/pytest.ini +++ b/src/Python/pytest.ini @@ -7,4 +7,6 @@ pythonpath = env = MigrationSDK__ContentTypes__0__Type = User - MigrationSDK__ContentTypes__0__BatchSize = 102 \ No newline at end of file + MigrationSDK__ContentTypes__0__BatchSize = 102 + MigrationSDK__ContentTypes__1__Type = Workbook + MigrationSDK__ContentTypes__1__IncludeExtractEnabled = False \ No newline at end of file diff --git a/src/Python/requirements.txt b/src/Python/requirements.txt index 95014b7..2185e35 100644 --- a/src/Python/requirements.txt +++ b/src/Python/requirements.txt @@ -2,3 +2,5 @@ pycparser==2.22 pythonnet==3.0.3 typing_extensions==4.12.2 +pytest-env==1.1.3 +build=1.2.1 \ No newline at end of file diff --git a/src/Python/scripts/build_tests.py b/src/Python/scripts/build_tests.py deleted file mode 100644 index a48c793..0000000 --- a/src/Python/scripts/build_tests.py +++ /dev/null @@ -1,9 +0,0 @@ -# noqa: D100 -# This is a testing script to check that nothing is really wrong. -import subprocess -from os.path import abspath -import build_binaries - -testcomponent_project = abspath("../../tests/Tableau.Migration.TestComponents/Tableau.Migration.TestComponents.csproj") - -subprocess.run(["dotnet", "publish", testcomponent_project, "-o", build_binaries.bin_path, "-f", "net6.0"]) \ No newline at end of file diff --git a/src/Python/src/tableau_migration/__init__.py b/src/Python/src/tableau_migration/__init__.py index 806aab3..80e7fc4 100644 --- a/src/Python/src/tableau_migration/__init__.py +++ b/src/Python/src/tableau_migration/__init__.py @@ -75,6 +75,8 @@ from tableau_migration.migration_engine_endpoints_search import PySourceContentReferenceFinderFactory as ISourceContentReferenceFinderFactory # noqa: E402, F401 from tableau_migration.migration_content_schedules_cloud import PyCloudExtractRefreshTask as ICloudExtractRefreshTask # noqa: E402, F401 from tableau_migration.migration_content_schedules_server import PyServerExtractRefreshTask as IServerExtractRefreshTask # noqa: E402, F401 +from tableau_migration.migration_engine_manifest import PyMigrationManifest as MigrationManifest # noqa: E402, F401 +from tableau_migration.migration_engine_manifest import PyMigrationManifestSerializer as MigrationManifestSerializer # noqa: E402, F401 # region _generated diff --git a/src/Python/src/tableau_migration/migration.py b/src/Python/src/tableau_migration/migration.py index 70c0935..de56b98 100644 --- a/src/Python/src/tableau_migration/migration.py +++ b/src/Python/src/tableau_migration/migration.py @@ -199,7 +199,7 @@ def add_errors(self, errors) -> Self: self._migration_manifest.AddErrors(marshalled_error) return - # If somethign else is passed in, let dotnet handle it. + # If something else is passed in, let dotnet handle it. # It's either valid and it works # or an exception will be thrown self._migration_manifest.AddErrors(errors) diff --git a/src/Python/src/tableau_migration/migration_engine_manifest.py b/src/Python/src/tableau_migration/migration_engine_manifest.py index 2cbcabe..ffc0292 100644 --- a/src/Python/src/tableau_migration/migration_engine_manifest.py +++ b/src/Python/src/tableau_migration/migration_engine_manifest.py @@ -15,6 +15,57 @@ """Wrapper for classes in Tableau.Migration.Engine.Manifest namespace.""" +from tableau_migration import ( + cancellation_token +) + +from tableau_migration.migration import ( # noqa: E402, F401 + PyMigrationManifest, + get_service_provider, + get_service +) + +from Tableau.Migration.Engine.Manifest import ( # noqa: E402, F401 + MigrationManifestSerializer +) + +class PyMigrationManifestSerializer(): + """Provides functionality to serialize and deserialize migration manifests in JSON format.""" + + _dotnet_base = MigrationManifestSerializer + + def __init__(self) -> None: + """Creates a new PyMigrationManifestSerializer object. + + Args: + migration_manifest_serializer: A MigrationManifestSerializer object. + + Returns: None. + """ + self._services = get_service_provider() + self._dotnet = get_service(self._services, MigrationManifestSerializer) + + def save(self, manifest: PyMigrationManifest, path: str) -> None: + """Saves a manifest in JSON format. + + Args: + manifest: The manifest to save. + path: The file path to save the manifest to. + """ + self._dotnet.SaveAsync(manifest._migration_manifest, path).GetAwaiter().GetResult() + + def load(self, path: str) -> PyMigrationManifest: + """Loads a manifest from JSON format. + + Args: + path: The file path to load the manifest from. + cancel: A cancellation token to cancel the operation. + + Returns: The loaded MigrationManifest, or None if the manifest could not be loaded. + """ + result = self._dotnet.LoadAsync(path, cancellation_token).GetAwaiter().GetResult() + return None if result is None else PyMigrationManifest(result) + # region _generated from enum import IntEnum # noqa: E402, F401 diff --git a/src/Python/tests/test_classes.py b/src/Python/tests/test_classes.py index d84d8bb..728c172 100644 --- a/src/Python/tests/test_classes.py +++ b/src/Python/tests/test_classes.py @@ -67,6 +67,9 @@ PyMigrationPlanOptionsBuilder, PyMigrationPlanOptionsCollection) +from tableau_migration.migration_engine_manifest import ( + PyMigrationManifestSerializer) + from tableau_migration.migration_engine_migrators import ( PyMigrator) @@ -422,7 +425,8 @@ def test_overloaded_missing(self): (PyDestinationContentReferenceFinder, None), (PyDestinationContentReferenceFinderFactory, [ "ForContentType" ]), (PySourceContentReferenceFinder, None), - (PySourceContentReferenceFinderFactory, [ "ForContentType" ]) + (PySourceContentReferenceFinderFactory, [ "ForContentType" ]), + (PyMigrationManifestSerializer, None), ] _test_class_data.extend(_generated_class_data) diff --git a/src/Python/tests/test_migration_engine_manifest.py b/src/Python/tests/test_migration_engine_manifest.py index f42f913..65f6496 100644 --- a/src/Python/tests/test_migration_engine_manifest.py +++ b/src/Python/tests/test_migration_engine_manifest.py @@ -13,6 +13,41 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import tempfile +from tests.helpers.autofixture import AutoFixtureTestBase # noqa: E402, F401 +from Tableau.Migration import IMigrationManifest # noqa: E402, F401 +from tableau_migration import ( + IMigrationManifestEntry, + MigrationManifest, + MigrationManifestSerializer) + +class TestManifestSaveLoad(AutoFixtureTestBase): + + def test_saveload(self): + serializer = MigrationManifestSerializer() + manifest = MigrationManifest(self.create(IMigrationManifest)) + + with tempfile.TemporaryDirectory() as temp_dir: + temp_file_path = os.path.join(temp_dir, 'manifest.json') + serializer.save(manifest, temp_file_path) + loaded = serializer.load(temp_file_path) + + assert manifest.plan_id == loaded.plan_id + assert manifest.manifest_version == loaded.manifest_version + assert manifest.migration_id == loaded.migration_id + + manifest_entries = [IMigrationManifestEntry(x) for x in manifest.entries] + loaded_entries = [IMigrationManifestEntry(x) for x in loaded.entries] + assert len(manifest_entries) > 0 + assert len(loaded_entries) > 0 + assert len(manifest_entries) == len(loaded_entries) + + assert manifest.errors.Count > 0 + assert loaded.errors.Count > 0 + assert manifest.errors.Count == loaded.errors.Count + + # region _generated from enum import IntEnum # noqa: E402, F401 diff --git a/src/Python/tests/test_other.py b/src/Python/tests/test_other.py index 4632898..3c59a50 100644 --- a/src/Python/tests/test_other.py +++ b/src/Python/tests/test_other.py @@ -26,12 +26,13 @@ from Tableau.Migration.Config import IConfigReader from Tableau.Migration.Content import IUser +from Tableau.Migration.Content import IWorkbook class TestEndToEnd(): def test_main(self): '''This is mean to mimic a real application''' planBuilder = PyMigrationPlanBuilder() - + planBuilder = planBuilder \ .from_source_tableau_server("http://fakeSourceServer.com", "", "fakeTokenName", "fakeTokenValue") \ .to_destination_tableau_cloud("https://online.tableau.com", "", "fakeTokenName", "fakeTokenValue") \ @@ -45,13 +46,13 @@ def test_main(self): class TestLogging(): def test_logging(self): '''At this point the module __init()__ has already been called and the logging provider factory has been initialized''' - + # Create a migration sdk object so that an ILogger will be instaniated PyMigrationPlanBuilder() - + # tableau_migration module keeps track of instaniated loggers. Verify that we have at least one assert len(_logger_names) > 0 - + for name in _logger_names: # Given that we have a name, we should have a logger assert logging.getLogger(name) @@ -60,6 +61,7 @@ class TestConfig(): def test_config(self): ''' Verify that the MigrationSDK__BatchSize variable has been changed + Verify that the MigrationSDK__IncludeExtractEnabled variable has been changed Environment variables act different in different operating systems. For this reason we must set the MigrationSDK_BatchSize in pytest.ini, as that is set before the python @@ -70,5 +72,7 @@ def test_config(self): config_reader = get_service(services, IConfigReader) batch_size = config_reader.Get[IUser]().BatchSize + include_extract = config_reader.Get[IWorkbook]().IncludeExtractEnabled - assert batch_size==102 \ No newline at end of file + assert batch_size == 102 + assert include_extract is False diff --git a/src/Tableau.Migration/Api/DataSourcesApiClient.cs b/src/Tableau.Migration/Api/DataSourcesApiClient.cs index 1cf7c0a..a581d12 100644 --- a/src/Tableau.Migration/Api/DataSourcesApiClient.cs +++ b/src/Tableau.Migration/Api/DataSourcesApiClient.cs @@ -28,6 +28,7 @@ using Tableau.Migration.Api.Rest.Models.Requests; using Tableau.Migration.Api.Rest.Models.Responses; using Tableau.Migration.Api.Tags; +using Tableau.Migration.Config; using Tableau.Migration.Content; using Tableau.Migration.Content.Files; using Tableau.Migration.Content.Search; @@ -45,6 +46,7 @@ internal sealed class DataSourcesApiClient : private readonly IContentFileStore _fileStore; private readonly IDataSourcePublisher _dataSourcePublisher; private readonly IConnectionManager _connectionManager; + private readonly IConfigReader _configReader; public DataSourcesApiClient( IRestRequestBuilderFactory restRequestBuilderFactory, @@ -56,7 +58,8 @@ public DataSourcesApiClient( IDataSourcePublisher dataSourcePublisher, ITagsApiClientFactory tagsClientFactory, IConnectionManager connectionManager, - ILabelsApiClientFactory labelsCLientFactory) + ILabelsApiClientFactory labelsCLientFactory, + IConfigReader configReader) : base(restRequestBuilderFactory, finderFactory, loggerFactory, sharedResourcesLocalizer) { _fileStore = fileStore; @@ -65,6 +68,7 @@ public DataSourcesApiClient( Permissions = permissionsClientFactory.Create(this); Tags = tagsClientFactory.Create(this); _connectionManager = connectionManager; + _configReader = configReader; } #region - ILabelsContentApiClient Implementation - @@ -152,14 +156,12 @@ public async Task> GetDataSourceAsync(Guid dataSourc } /// - public async Task> DownloadDataSourceAsync( - Guid dataSourceId, - bool includeExtract, + public async Task> DownloadDataSourceAsync(Guid dataSourceId, CancellationToken cancel) { var downloadResult = await RestRequestBuilderFactory .CreateUri($"{UrlPrefix}/{dataSourceId.ToUrlSegment()}/{RestUrlPrefixes.Content}") - .WithQuery("includeExtract", includeExtract.ToString()) + .WithQuery("includeExtract", _configReader.Get().IncludeExtractEnabled.ToString()) .ForGetRequest() .DownloadAsync(cancel) .ConfigureAwait(false); @@ -245,7 +247,7 @@ public async Task> PullAsync( return connectionsResult.CastFailure(); } - var downloadResult = await DownloadDataSourceAsync(contentItem.Id, true, cancel).ConfigureAwait(false); + var downloadResult = await DownloadDataSourceAsync(contentItem.Id, cancel).ConfigureAwait(false); if (!downloadResult.Success) { return downloadResult.CastFailure(); diff --git a/src/Tableau.Migration/Api/IDataSourcesApiClient.cs b/src/Tableau.Migration/Api/IDataSourcesApiClient.cs index 059e9aa..2090a73 100644 --- a/src/Tableau.Migration/Api/IDataSourcesApiClient.cs +++ b/src/Tableau.Migration/Api/IDataSourcesApiClient.cs @@ -67,12 +67,10 @@ Task> GetDataSourceAsync( /// Downloads the data source file for the given ID. /// /// The ID to download the data source file for. - /// Whether or not to include extracts in the data source file. - /// The cancellation token to obey. + /// A cancellation token to obey. /// The file download result. Task> DownloadDataSourceAsync( Guid dataSourceId, - bool includeExtract, CancellationToken cancel); /// diff --git a/src/Tableau.Migration/Api/IWorkbooksApiClient.cs b/src/Tableau.Migration/Api/IWorkbooksApiClient.cs index c493d0b..8befac4 100644 --- a/src/Tableau.Migration/Api/IWorkbooksApiClient.cs +++ b/src/Tableau.Migration/Api/IWorkbooksApiClient.cs @@ -60,12 +60,10 @@ public interface IWorkbooksApiClient : /// Downloads the workbook file for the given ID. /// /// The ID to download the workbook file for. - /// Whether or not to include extracts in the workbook file. - /// The cancellation token to obey. + /// A cancellation token to obey. /// The file download result. Task> DownloadWorkbookAsync( Guid workbookId, - bool includeExtract, CancellationToken cancel); /// diff --git a/src/Tableau.Migration/Api/Models/FailedJobException.cs b/src/Tableau.Migration/Api/Models/FailedJobException.cs index 5ef6dd4..16f68c1 100644 --- a/src/Tableau.Migration/Api/Models/FailedJobException.cs +++ b/src/Tableau.Migration/Api/Models/FailedJobException.cs @@ -23,13 +23,25 @@ namespace Tableau.Migration.Api.Models /// /// Class representing a failed Tableau job. /// - public class FailedJobException : Exception + public class FailedJobException : Exception, IEquatable { /// /// Gets the failed job. /// public IJob FailedJob { get; } + /// + /// Creates a new object + /// + /// This should only be used for deserialization. + /// The failed job. + /// Message for base Exception. + internal FailedJobException(IJob job, string exceptionMessage) + : base(exceptionMessage) + { + FailedJob = job; + } + /// /// Creates a new object /// @@ -40,5 +52,35 @@ public FailedJobException(IJob job, ISharedResourcesLocalizer sharedResourcesLoc { FailedJob = job; } + + #region - IEquatable - + + /// + public bool Equals(FailedJobException? other) + { + if (other == null) return false; + + // Use IJob's IEquatable implementation for comparison + return FailedJob?.Equals(other.FailedJob) ?? other.FailedJob == null; + } + + /// + public override bool Equals(object? obj) + { + if (obj is FailedJobException other) + { + return Equals(other); + } + + return false; + } + + /// + public override int GetHashCode() + { + return FailedJob?.GetHashCode() ?? 0; + } + + #endregion } } diff --git a/src/Tableau.Migration/Api/Models/IJob.cs b/src/Tableau.Migration/Api/Models/IJob.cs index 65b8c2c..fe9f5d1 100644 --- a/src/Tableau.Migration/Api/Models/IJob.cs +++ b/src/Tableau.Migration/Api/Models/IJob.cs @@ -17,13 +17,14 @@ using System; using System.Collections.Immutable; +using Tableau.Migration.Api.Rest; namespace Tableau.Migration.Api.Models { /// /// Interface for an API client job model. /// - public interface IJob + public interface IJob : IRestIdentifiable, IEquatable { /// /// Gets the job's type. diff --git a/src/Tableau.Migration/Api/Models/Job.cs b/src/Tableau.Migration/Api/Models/Job.cs index 064f8d2..e283d52 100644 --- a/src/Tableau.Migration/Api/Models/Job.cs +++ b/src/Tableau.Migration/Api/Models/Job.cs @@ -68,7 +68,57 @@ public Job(JobResponse response) ProgressPercentage = response.Item.Progress; FinishCode = response.Item.FinishCode; - StatusNotes = response.Item.StatusNotes?.Select(s => new StatusNote(s)).ToImmutableArray() ?? ImmutableArray.Empty; + StatusNotes = response.Item.StatusNotes?.Select(s => (IStatusNote)new StatusNote(s)).ToImmutableArray() ?? ImmutableArray.Empty; } + + #region - IEquatable - + + /// + public override bool Equals(object? obj) + { + return Equals(obj as Job); + } + + /// + public bool Equals(IJob? other) + { + if (other == null) return false; + + bool statusNotesEqual = StatusNotes != null && other.StatusNotes != null ? + StatusNotes.SequenceEqual(other.StatusNotes) : + StatusNotes == null && other.StatusNotes == null; + + return Id == other.Id && + Type == other.Type && + CreatedAtUtc == other.CreatedAtUtc && + Nullable.Equals(UpdatedAtUtc, other.UpdatedAtUtc) && + Nullable.Equals(CompletedAtUtc, other.CompletedAtUtc) && + ProgressPercentage == other.ProgressPercentage && + FinishCode == other.FinishCode && + statusNotesEqual; + } + + /// + public override int GetHashCode() + { + HashCode hash = new HashCode(); + hash.Add(Id); + hash.Add(Type); + hash.Add(CreatedAtUtc); + hash.Add(UpdatedAtUtc); + hash.Add(CompletedAtUtc); + hash.Add(ProgressPercentage); + hash.Add(FinishCode); + if (StatusNotes != null) + { + foreach (var statusNote in StatusNotes) + { + hash.Add(statusNote); + } + } + return hash.ToHashCode(); + } + + #endregion } } diff --git a/src/Tableau.Migration/Api/Models/StatusNote.cs b/src/Tableau.Migration/Api/Models/StatusNote.cs index f92a0d1..b440d1a 100644 --- a/src/Tableau.Migration/Api/Models/StatusNote.cs +++ b/src/Tableau.Migration/Api/Models/StatusNote.cs @@ -15,11 +15,12 @@ // limitations under the License. // +using System; using Tableau.Migration.Api.Rest.Models.Responses; namespace Tableau.Migration.Api.Models { - internal class StatusNote : IStatusNote + internal record struct StatusNote : IStatusNote { /// public string? Type { get; } diff --git a/src/Tableau.Migration/Api/Models/TimeoutJobException.cs b/src/Tableau.Migration/Api/Models/TimeoutJobException.cs index e99058d..8686064 100644 --- a/src/Tableau.Migration/Api/Models/TimeoutJobException.cs +++ b/src/Tableau.Migration/Api/Models/TimeoutJobException.cs @@ -23,22 +23,58 @@ namespace Tableau.Migration.Api.Models /// /// Class representing a Tableau Job that timed out while waiting to finish /// - public class TimeoutJobException : Exception + public class TimeoutJobException : Exception, IEquatable { /// /// Gets the job that timed out. May be null if no job status was ever reported. /// public IJob? Job { get; } + /// + /// Creates a new object + /// + /// This should only be used for deserialization. + /// The last job status that timed out. May be null if no job status was ever reported. + /// Message of the base exception + internal TimeoutJobException(IJob? job, string exceptionMessage) + : base(exceptionMessage) + { + Job = job; + } + /// /// Creates a new object /// /// The last job status that timed out. May be null if no job status was ever reported. /// A string localizer. public TimeoutJobException(IJob? job, ISharedResourcesLocalizer sharedResourcesLocalizer) - : base(sharedResourcesLocalizer[SharedResourceKeys.TimeoutJobExceptionMessage]) + : this(job, sharedResourcesLocalizer[SharedResourceKeys.TimeoutJobExceptionMessage]) + { } + + #region - IEquatable - + + /// + public override bool Equals(object? obj) { - Job = job; + return Equals(obj as TimeoutJobException); + } + + /// + public bool Equals(TimeoutJobException? other) + { + if (other is null) return false; + if (ReferenceEquals(this, other)) return true; + + // Now directly using IJob's Equals method for comparison. + return Job?.Equals(other.Job) ?? other.Job == null; } + + /// + public override int GetHashCode() + { + return Job?.GetHashCode() ?? 0; + } + + #endregion } } diff --git a/src/Tableau.Migration/Api/Rest/RestException.cs b/src/Tableau.Migration/Api/Rest/RestException.cs index 06e3c38..13b3c01 100644 --- a/src/Tableau.Migration/Api/Rest/RestException.cs +++ b/src/Tableau.Migration/Api/Rest/RestException.cs @@ -25,7 +25,7 @@ namespace Tableau.Migration.Api.Rest /// /// Class representing an error from a Tableau REST API /// - public class RestException : Exception + public class RestException : Exception, IEquatable { /// /// Gets the request URI from Tableau API. @@ -55,16 +55,17 @@ public class RestException : Exception /// /// Creates a new instance. /// + /// This should only be used for deserialization. /// The http method that generated the current error. /// The request URI that generated the current error. /// The returned from the Tableau API. - /// A string localizer. - public RestException( + /// Message for base Exception. + internal RestException( HttpMethod? httpMethod, Uri? requestUri, Error error, - ISharedResourcesLocalizer sharedResourcesLocalizer) - : base(FormatError(httpMethod, requestUri, error, sharedResourcesLocalizer)) + string exceptionMessage) + : base(exceptionMessage) { HttpMethod = httpMethod; RequestUri = requestUri; @@ -73,6 +74,21 @@ public RestException( Summary = error.Summary; } + /// + /// Creates a new instance. + /// + /// The http method that generated the current error. + /// The request URI that generated the current error. + /// The returned from the Tableau API. + /// A string localizer. + public RestException( + HttpMethod? httpMethod, + Uri? requestUri, + Error error, + ISharedResourcesLocalizer sharedResourcesLocalizer) + : this(httpMethod, requestUri, error, FormatError(httpMethod, requestUri, error, sharedResourcesLocalizer)) + { } + private static string FormatError( HttpMethod? httpMethod, Uri? requestUri, @@ -89,5 +105,39 @@ private static string FormatError( error.Summary ?? nullValue, error.Detail ?? nullValue); } + + #region - IEquatable - + + /// + public bool Equals(RestException? other) + { + if (other == null) return false; + + return HttpMethod == other.HttpMethod && + RequestUri == other.RequestUri && + Code == other.Code && + Detail == other.Detail && + Summary == other.Summary; + } + + /// + public override bool Equals(object? obj) + { + if (obj is RestException other) + { + return Equals(other); + } + + return false; + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(HttpMethod, RequestUri, Code, Detail, Summary); + } + + #endregion + } } diff --git a/src/Tableau.Migration/Api/Simulation/Rest/Net/Responses/BuildResponseException.cs b/src/Tableau.Migration/Api/Simulation/Rest/Net/Responses/BuildResponseException.cs index 5180a23..ea2cdfd 100644 --- a/src/Tableau.Migration/Api/Simulation/Rest/Net/Responses/BuildResponseException.cs +++ b/src/Tableau.Migration/Api/Simulation/Rest/Net/Responses/BuildResponseException.cs @@ -20,7 +20,7 @@ namespace Tableau.Migration.Api.Simulation.Rest.Net.Responses { - internal class BuildResponseException : Exception + internal class BuildResponseException : Exception, IEquatable { public BuildResponseException(HttpStatusCode statusCode, int subCode, string summary, string detail) { @@ -35,5 +35,30 @@ public BuildResponseException(HttpStatusCode statusCode, int subCode, string sum public string Summary { get; set; } public string Detail { get; set; } + #region - IEquatable - + + /// + public override bool Equals(object? obj) + { + return Equals(obj as BuildResponseException); + } + + /// + public bool Equals(BuildResponseException? other) + { + return other != null && + StatusCode == other.StatusCode && + SubCode == other.SubCode && + Summary == other.Summary && + Detail == other.Detail; + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(StatusCode, SubCode, Summary, Detail); + } + + #endregion } } diff --git a/src/Tableau.Migration/Api/WorkbooksApiClient.cs b/src/Tableau.Migration/Api/WorkbooksApiClient.cs index 2eb3210..df7999f 100644 --- a/src/Tableau.Migration/Api/WorkbooksApiClient.cs +++ b/src/Tableau.Migration/Api/WorkbooksApiClient.cs @@ -27,6 +27,7 @@ using Tableau.Migration.Api.Rest.Models.Requests; using Tableau.Migration.Api.Rest.Models.Responses; using Tableau.Migration.Api.Tags; +using Tableau.Migration.Config; using Tableau.Migration.Content; using Tableau.Migration.Content.Files; using Tableau.Migration.Content.Search; @@ -43,6 +44,7 @@ internal sealed class WorkbooksApiClient : private readonly IContentFileStore _fileStore; private readonly IWorkbookPublisher _workbookPublisher; private readonly IConnectionManager _connectionManager; + private readonly IConfigReader _configReader; public WorkbooksApiClient( IRestRequestBuilderFactory restRequestBuilderFactory, @@ -54,7 +56,8 @@ public WorkbooksApiClient( IWorkbookPublisher workbookPublisher, ITagsApiClientFactory tagsClientFactory, IViewsApiClientFactory viewsClientFactory, - IConnectionManager connectionManager) + IConnectionManager connectionManager, + IConfigReader configReader) : base(restRequestBuilderFactory, finderFactory, loggerFactory, sharedResourcesLocalizer) { _fileStore = fileStore; @@ -63,6 +66,7 @@ public WorkbooksApiClient( Tags = tagsClientFactory.Create(this); Views = viewsClientFactory.Create(); _connectionManager = connectionManager; + _configReader = configReader; } #region - IPermissionsContentApiClientImplementation - @@ -150,12 +154,11 @@ public async Task> GetWorkbookAsync(Guid workbookId, C /// public async Task> DownloadWorkbookAsync( Guid workbookId, - bool includeExtract, CancellationToken cancel) { var downloadResult = await RestRequestBuilderFactory .CreateUri($"{UrlPrefix}/{workbookId.ToUrlSegment()}/{RestUrlPrefixes.Content}") - .WithQuery("includeExtract", includeExtract.ToString()) + .WithQuery("includeExtract", _configReader.Get().IncludeExtractEnabled.ToString()) .ForGetRequest() .DownloadAsync(cancel) .ConfigureAwait(false); @@ -245,7 +248,7 @@ public async Task> PullAsync( return connectionsResult.CastFailure(); } - var downloadResult = await DownloadWorkbookAsync(contentItem.Id, true, cancel).ConfigureAwait(false); + var downloadResult = await DownloadWorkbookAsync(contentItem.Id, cancel).ConfigureAwait(false); if (!downloadResult.Success) { return downloadResult.CastFailure(); diff --git a/src/Tableau.Migration/Config/ContentTypesOptions.cs b/src/Tableau.Migration/Config/ContentTypesOptions.cs index a987773..d7ec317 100644 --- a/src/Tableau.Migration/Config/ContentTypesOptions.cs +++ b/src/Tableau.Migration/Config/ContentTypesOptions.cs @@ -40,6 +40,11 @@ public static class Defaults /// The default migration batch publishing flag. /// public const bool BATCH_PUBLISHING_ENABLED = false; + + /// + /// The default migration include extract flag + /// + public const bool INCLUDE_EXTRACT_ENABLED = true; } /// @@ -68,6 +73,18 @@ public bool BatchPublishingEnabled set => _batchPublishingEnabled = value; } private bool? _batchPublishingEnabled; + + /// + /// Gets or sets the include extract flag for supported types. Default: enabled.
+ /// Important: This option is only available to and . + /// For more details, check the configuration article. + /// + public bool IncludeExtractEnabled + { + get => _includeExtractEnabled ?? Defaults.INCLUDE_EXTRACT_ENABLED; + set => _includeExtractEnabled = value; + } + private bool? _includeExtractEnabled; /// /// Checks if the content type in is valid. diff --git a/src/Tableau.Migration/Engine/IServiceCollectionExtensions.cs b/src/Tableau.Migration/Engine/IServiceCollectionExtensions.cs index 9681c63..80a0e43 100644 --- a/src/Tableau.Migration/Engine/IServiceCollectionExtensions.cs +++ b/src/Tableau.Migration/Engine/IServiceCollectionExtensions.cs @@ -95,6 +95,9 @@ internal static IServiceCollection AddMigrationEngine(this IServiceCollection se services.AddScoped(typeof(ItemPublishContentBatchMigrator<,,>)); services.AddScoped(typeof(ContentMigrator<>)); + //Serializer + services.AddSingleton(); + //Caches/Content Finders //Register concrete types so that the easy way to get interface types is through IMigrationPipeline. services.AddScoped(typeof(BulkSourceCache<>)); diff --git a/src/Tableau.Migration/Engine/Manifest/MigrationManifest.cs b/src/Tableau.Migration/Engine/Manifest/MigrationManifest.cs index 7351c1e..500fe80 100644 --- a/src/Tableau.Migration/Engine/Manifest/MigrationManifest.cs +++ b/src/Tableau.Migration/Engine/Manifest/MigrationManifest.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; +using System.Linq; using Microsoft.Extensions.Logging; using Tableau.Migration.Resources; @@ -41,7 +42,7 @@ public class MigrationManifest : IMigrationManifestEditor /// /// The latest manifest version number. /// - public const uint LatestManifestVersion = 2; + public const uint LatestManifestVersion = 3; /// /// Creates a new object. @@ -96,7 +97,12 @@ public bool Equals(IMigrationManifest? other) { if (other is null) return false; - var equal = PlanId.Equals(other.PlanId) && MigrationId.Equals(other.MigrationId) && ManifestVersion.Equals(other.ManifestVersion) && Entries.Equals(other.Entries); + var equal = + PlanId.Equals(other.PlanId) && + MigrationId.Equals(other.MigrationId) && + ManifestVersion.Equals(other.ManifestVersion) && + Entries.Equals(other.Entries) && + Errors.SequenceEqual(other.Errors, new ExceptionComparer()); return equal; } diff --git a/src/Tableau.Migration/Engine/Manifest/MigrationManifestEntry.cs b/src/Tableau.Migration/Engine/Manifest/MigrationManifestEntry.cs index b10fb1d..9b9f044 100644 --- a/src/Tableau.Migration/Engine/Manifest/MigrationManifestEntry.cs +++ b/src/Tableau.Migration/Engine/Manifest/MigrationManifestEntry.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.Linq; using Tableau.Migration.Content; namespace Tableau.Migration.Engine.Manifest @@ -55,16 +56,10 @@ public MigrationManifestEntry(IMigrationManifestEntryBuilder entryBuilder, _entryBuilder = entryBuilder; Source = previousMigrationEntry.Source; MappedLocation = previousMigrationEntry.MappedLocation; - - //Status is reset since this is a new run. - //The HasMigrated flag is copied since it tracks whether the - //item has migrated in any run. + Status = previousMigrationEntry.Status; + Destination = previousMigrationEntry.Destination; HasMigrated = previousMigrationEntry.HasMigrated; - - //Errors are reset since this is a new migration. - - //Destination is reset in case our IDs are out-of-date - //(e.g. a user manually deleted between runs) + _errors = previousMigrationEntry.Errors.ToImmutableArray(); } /// @@ -113,7 +108,6 @@ public virtual IContentReference? Destination /// /// Indicates if the current IMigrationManifestEntry is equal to another IMigrationManifestEntry. - /// *Note: This ignores if the errors are different* /// /// True if the current object is equal to the other parameter. Otherwise false. public static bool Equals(IMigrationManifestEntry entry, IMigrationManifestEntry? other) @@ -121,20 +115,19 @@ public static bool Equals(IMigrationManifestEntry entry, IMigrationManifestEntry if (other is null) return false; - if ((entry.Source.Equals(other.Source) && entry.MappedLocation.Equals(other.MappedLocation) && entry.Status.Equals(other.Status)) == false) + if (!entry.Source.Equals(other.Source) || + !entry.MappedLocation.Equals(other.MappedLocation) || + !entry.Status.Equals(other.Status) || + !entry.Errors.SequenceEqual(other.Errors, new ExceptionComparer())) return false; // Nullability of Destination must match - if (!Object.ReferenceEquals(entry.Destination, other.Destination)) + if (entry.Destination != other.Destination) { if (entry.Destination is null || other.Destination is null) - { return false; - } - else if (!entry.Destination.Equals(other.Destination)) - { - return false; - } + + return entry.Destination.Equals(other.Destination); } return true; diff --git a/tests/Tableau.Migration.TestComponents/Engine/Manifest/MigrationManifestSerializer.cs b/src/Tableau.Migration/Engine/Manifest/MigrationManifestSerializer.cs similarity index 66% rename from tests/Tableau.Migration.TestComponents/Engine/Manifest/MigrationManifestSerializer.cs rename to src/Tableau.Migration/Engine/Manifest/MigrationManifestSerializer.cs index 2d8c4f2..b902723 100644 --- a/tests/Tableau.Migration.TestComponents/Engine/Manifest/MigrationManifestSerializer.cs +++ b/src/Tableau.Migration/Engine/Manifest/MigrationManifestSerializer.cs @@ -15,17 +15,25 @@ // limitations under the License. // +using System; using System.Collections.Immutable; +using System.IO; using System.IO.Abstractions; using System.Text.Json; using System.Text.Json.Serialization; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using Tableau.Migration.Engine.Manifest; +using Tableau.Migration.JsonConverters; +using Tableau.Migration.JsonConverters.SerializableObjects; using Tableau.Migration.Resources; -using Tableau.Migration.TestComponents.JsonConverters; -namespace Tableau.Migration.TestComponents.Engine.Manifest + +namespace Tableau.Migration.Engine.Manifest { + /// + /// Provides functionality to serialize and deserialize migration manifests in JSON format. + /// public class MigrationManifestSerializer { private readonly IFileSystem _fileSystem; @@ -34,23 +42,44 @@ public class MigrationManifestSerializer private readonly ImmutableArray _converters; + /// + /// Initializes a new instance of the class. + /// public MigrationManifestSerializer(IFileSystem fileSystem, ISharedResourcesLocalizer localizer, ILoggerFactory loggerFactory) { _fileSystem = fileSystem; _localizer = localizer; _loggerFactory = loggerFactory; - _converters = new JsonConverter[] - { - new ExceptionJsonConverter(_loggerFactory.CreateLogger()), - new MigrationManifestEntryCollectionWriter(), - new MigrationManifestEntryCollectionReader(localizer, loggerFactory) - }.ToImmutableArray(); + _converters = CreateConverters(); } - // This is the current MigrationManifest.ManifestVersion that this serializer supports + /// + /// This is the current MigrationManifest.ManifestVersion that this serializer supports. + /// public const uint SupportedManifestVersion = MigrationManifest.LatestManifestVersion; + + /// + /// Creates the list of JSON converters used by the MigrationManifestSerializer. + /// + /// This is a static method so the tests can use the same list converters. + /// An immutable array of JSON converters. + internal static ImmutableArray CreateConverters() + { + return new JsonConverter[] + { + new PythonExceptionConverter(), + new SerializedExceptionJsonConverter(), + new BuildResponseExceptionJsonConverter(), + new JobJsonConverter(), + new TimeoutJobExceptionJsonConverter(), + new RestExceptionJsonConverter(), + new FailedJobExceptionJsonConverter(), + new ExceptionJsonConverterFactory(), // This needs to be at the end. This list is ordered. + }.ToImmutableArray(); + } + private JsonSerializerOptions MergeJsonOptions(JsonSerializerOptions? jsonOptions) { jsonOptions ??= new() { WriteIndented = true }; @@ -66,9 +95,10 @@ private JsonSerializerOptions MergeJsonOptions(JsonSerializerOptions? jsonOption /// /// Saves a manifest in JSON format. /// + /// This async function does not take a cancellation token. This is because the saving should happen, + /// no matter what the status of the cancellation token is. Otherwise the manifest is not saved if the migration is cancelled. /// The manifest to save. /// The file path to save the manifest to. - /// The cancellation token to obey. /// Optional JSON options to use. public async Task SaveAsync(IMigrationManifest manifest, string path, JsonSerializerOptions? jsonOptions = null) { @@ -80,13 +110,15 @@ public async Task SaveAsync(IMigrationManifest manifest, string path, JsonSerial _fileSystem.Directory.CreateDirectory(dir); } + var serializableManifest = new SerializableMigrationManifest(manifest); + var file = _fileSystem.File.Create(path); await using (file.ConfigureAwait(false)) { // If cancellation was requested, we still need to save the file, so use the default token. - await JsonSerializer.SerializeAsync(file, manifest, jsonOptions, default) + await JsonSerializer.SerializeAsync(file, serializableManifest, jsonOptions, default) .ConfigureAwait(false); - } + } } /// @@ -108,7 +140,7 @@ await JsonSerializer.SerializeAsync(file, manifest, jsonOptions, default) var file = _fileSystem.File.OpenRead(path); await using (file.ConfigureAwait(false)) { - var manifest = await JsonSerializer.DeserializeAsync(file, jsonOptions, cancel) + var manifest = await JsonSerializer.DeserializeAsync(file, jsonOptions, cancel) .ConfigureAwait(false); if (manifest is not null) @@ -116,9 +148,7 @@ await JsonSerializer.SerializeAsync(file, manifest, jsonOptions, default) if (manifest.ManifestVersion is not SupportedManifestVersion) throw new NotSupportedException($"This {nameof(MigrationManifestSerializer)} only supports Manifest version {SupportedManifestVersion}. The manifest being loaded is version {manifest.ManifestVersion}"); - var ret = new MigrationManifest(_localizer, _loggerFactory, manifest.PlanId, manifest.MigrationId); - manifest.Entries.CopyTo(ret.Entries); - return ret; + return manifest.ToMigrationManifest(_localizer, _loggerFactory) as MigrationManifest; } return null; diff --git a/src/Tableau.Migration/ExceptionComparer.cs b/src/Tableau.Migration/ExceptionComparer.cs new file mode 100644 index 0000000..9e025d7 --- /dev/null +++ b/src/Tableau.Migration/ExceptionComparer.cs @@ -0,0 +1,74 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections; +using System.Collections.Generic; +using System.Reflection; + +namespace Tableau.Migration +{ + /// + /// Provides methods for comparing exceptions + /// + public class ExceptionComparer : IEqualityComparer + { + /// + public bool Equals(Exception? x, Exception? y) + { + if (x is null && y is null) return true; + if (x is null || y is null) return false; + + // Check if x and y are of the same type + if (x.GetType() != y.GetType()) + { + return false; + } + + // Check if they implement IEquatable for their specific type + Type equatableType = typeof(IEquatable<>).MakeGenericType(x.GetType()); + if (equatableType.IsAssignableFrom(x.GetType())) + { + return (bool?)equatableType.GetMethod("Equals")?.Invoke(x, new object?[] { y }) ?? false; + } + else + { + return x.Message == y.Message; + } + } + + /// + public int GetHashCode(Exception obj) + { + // Check if the object's type overrides GetHashCode + MethodInfo? getHashCodeMethod = obj.GetType().GetMethod("GetHashCode", Type.EmptyTypes); + if (getHashCodeMethod != null && getHashCodeMethod.DeclaringType != typeof(object)) + { + return obj.GetHashCode(); + } + else if (obj is IEquatable) + { + return obj.GetHashCode(); + } + else + { + return obj.Message?.GetHashCode() ?? 0; + } + } + + } +} \ No newline at end of file diff --git a/src/Tableau.Migration/IEnumerableExtensions.cs b/src/Tableau.Migration/IEnumerableExtensions.cs index dbff628..9d1fe7a 100644 --- a/src/Tableau.Migration/IEnumerableExtensions.cs +++ b/src/Tableau.Migration/IEnumerableExtensions.cs @@ -97,6 +97,7 @@ public static IEnumerable ExceptNulls(this IEnumerabl where TResult : class => items.ExceptNulls().Select(selector).ExceptNulls(); + #endregion } } diff --git a/src/Tableau.Migration/IServiceCollectionExtensions.cs b/src/Tableau.Migration/IServiceCollectionExtensions.cs index 9d19c9f..1e14a75 100644 --- a/src/Tableau.Migration/IServiceCollectionExtensions.cs +++ b/src/Tableau.Migration/IServiceCollectionExtensions.cs @@ -21,6 +21,7 @@ using Tableau.Migration.Api; using Tableau.Migration.Config; using Tableau.Migration.Engine; +using Tableau.Migration.Engine.Manifest; using Tableau.Migration.Net; using Tableau.Migration.Resources; diff --git a/src/Tableau.Migration/JsonConverters/BuildResponseExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/BuildResponseExceptionJsonConverter.cs new file mode 100644 index 0000000..14736d1 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/BuildResponseExceptionJsonConverter.cs @@ -0,0 +1,97 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Net; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Simulation.Rest.Net.Responses; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes a . It does not support reading exceptions back in. + /// + internal class BuildResponseExceptionJsonConverter : JsonConverter + { + public override void Write(Utf8JsonWriter writer, BuildResponseException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + writer.WriteString(nameof(BuildResponseException.StatusCode), value.StatusCode.ToString()); + writer.WriteNumber(nameof(BuildResponseException.SubCode), value.SubCode); + writer.WriteString(nameof(BuildResponseException.Summary), value.Summary); + writer.WriteString(nameof(BuildResponseException.Detail), value.Detail); + JsonWriterUtils.WriteExceptionProperties(ref writer, value); + writer.WriteEndObject(); + } + + + public override BuildResponseException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + JsonReaderUtils.AssertStartObject(ref reader); + + HttpStatusCode? statusCode = null; + int? subCode = null; + string? summary = null; + string? detail = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? propertyName = reader.GetString(); + Guard.AgainstNullOrEmpty(propertyName, nameof(propertyName)); + + reader.Read(); // Move to the property value. + + switch(propertyName) + { + case nameof(BuildResponseException.StatusCode): + var statusCodeStr = reader.GetString(); + Guard.AgainstNull(statusCodeStr, nameof(statusCodeStr)); + statusCode = (HttpStatusCode)Enum.Parse(typeof(HttpStatusCode), statusCodeStr); + break; + + case nameof(BuildResponseException.SubCode): + subCode = reader.GetInt32(); + break; + + case nameof(BuildResponseException.Summary): + summary = reader.GetString(); + Guard.AgainstNull(summary, nameof(summary)); + break; + + case nameof(BuildResponseException.Detail): + detail = reader.GetString(); + Guard.AgainstNull(detail, nameof(detail)); + break; + + default: + break; + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + return new BuildResponseException(statusCode!.Value, subCode!.Value, summary!, detail!); + } + + } +} diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/Constants.cs b/src/Tableau.Migration/JsonConverters/Constants.cs similarity index 84% rename from tests/Tableau.Migration.TestComponents/JsonConverters/Constants.cs rename to src/Tableau.Migration/JsonConverters/Constants.cs index a49a38a..112c920 100644 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/Constants.cs +++ b/src/Tableau.Migration/JsonConverters/Constants.cs @@ -15,11 +15,13 @@ // limitations under the License. // -namespace Tableau.Migration.TestComponents.JsonConverters +namespace Tableau.Migration.JsonConverters { internal static class Constants { public const string PARTITION = "Partition"; public const string ENTRIES = "Entries"; + public const string CLASS_NAME = "ClassName"; + public const string EXCEPTION = "Exception"; } } diff --git a/src/Tableau.Migration/JsonConverters/ExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/ExceptionJsonConverter.cs new file mode 100644 index 0000000..e86464d --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/ExceptionJsonConverter.cs @@ -0,0 +1,87 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes and deserializes any type of . + /// + /// The type of the exception to convert. + public class ExceptionJsonConverter : JsonConverter where TException : Exception + { + /// + /// Writes the JSON representation of an Exception object. + /// + /// The to write to. + /// The object to serialize. + /// The to use for serialization. + public override void Write(Utf8JsonWriter writer, TException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + JsonWriterUtils.WriteExceptionProperties(ref writer, value); + writer.WriteEndObject(); + } + + /// + /// Reads the JSON representation of an Exception object. + /// + /// The to read from. + /// The type of the object to convert. + /// The to use for deserialization. + /// The deserialized object. + public override TException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + JsonReaderUtils.AssertStartObject(ref reader); + + string? message = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? propertyName = reader.GetString(); + Guard.AgainstNullOrEmpty(propertyName, nameof(propertyName)); + + reader.Read(); // Move to the property value. + + if (propertyName == "Message") + { + message = reader.GetString(); + Guard.AgainstNull(message, nameof(message)); // Message could be an empty string, so just check null + } + if (propertyName == "InnerException") + { + reader.Skip(); // Don't read in the inner exceptions + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + // Message must be deserialized by now + Guard.AgainstNull(message, nameof(message)); + + return (TException)Activator.CreateInstance(typeof(TException), message)!; + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/ExceptionJsonConverterFactory.cs b/src/Tableau.Migration/JsonConverters/ExceptionJsonConverterFactory.cs new file mode 100644 index 0000000..7282a0c --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/ExceptionJsonConverterFactory.cs @@ -0,0 +1,42 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverterFactory that creates converters for any type of . + /// + public class ExceptionJsonConverterFactory : JsonConverterFactory + { + /// + public override bool CanConvert(Type typeToConvert) + { + return typeof(Exception).IsAssignableFrom(typeToConvert); + } + + /// + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + { + var converterType = typeof(ExceptionJsonConverter<>).MakeGenericType(typeToConvert); + return (JsonConverter)Activator.CreateInstance(converterType)!; + } + } +} diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/Exceptions/MismatchException.cs b/src/Tableau.Migration/JsonConverters/Exceptions/MismatchException.cs similarity index 84% rename from tests/Tableau.Migration.TestComponents/JsonConverters/Exceptions/MismatchException.cs rename to src/Tableau.Migration/JsonConverters/Exceptions/MismatchException.cs index 47d0d0a..1b1fd68 100644 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/Exceptions/MismatchException.cs +++ b/src/Tableau.Migration/JsonConverters/Exceptions/MismatchException.cs @@ -15,8 +15,14 @@ // limitations under the License. // -namespace Tableau.Migration.TestComponents.JsonConverters.Exceptions +using System; + +namespace Tableau.Migration.JsonConverters.Exceptions { + /// + /// The exception that is thrown when a that was serialized and then deserialized but did not match the initial Manifest. + /// + /// This means that the either the serializer or deserializer has a bug. public class MismatchException : Exception { /// diff --git a/src/Tableau.Migration/JsonConverters/FailedJobExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/FailedJobExceptionJsonConverter.cs new file mode 100644 index 0000000..b3ba354 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/FailedJobExceptionJsonConverter.cs @@ -0,0 +1,73 @@ +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Models; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// Provides a custom JSON converter for objects, allowing for custom serialization and deserialization logic. + /// + public class FailedJobExceptionJsonConverter : JsonConverter + { + /// + /// Reads and converts the JSON to type . + /// + /// The reader to deserialize objects or value types. + /// The type of object to convert. + /// Options to control the behavior during reading. + /// A object deserialized from JSON. + public override FailedJobException Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + IJob? failedJob = null; + string? exceptionMessage = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + switch (propertyName) + { + case nameof(FailedJobException.FailedJob): + failedJob = JsonSerializer.Deserialize(ref reader, options); + break; + + case nameof(FailedJobException.Message): + exceptionMessage = reader.GetString(); + break; + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + Guard.AgainstNull(exceptionMessage, nameof(exceptionMessage)); + Guard.AgainstNull(failedJob, nameof(failedJob)); + + return new FailedJobException(failedJob, exceptionMessage); + } + + /// + /// Writes a specified object to JSON. + /// + /// The writer to serialize objects or value types. + /// The value to serialize. + /// Options to control the behavior during writing. + public override void Write(Utf8JsonWriter writer, FailedJobException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + writer.WritePropertyName(nameof(FailedJobException.FailedJob)); + JsonSerializer.Serialize(writer, value.FailedJob, options); + + writer.WriteString(nameof(FailedJobException.Message), value.Message); + + writer.WriteEndObject(); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/JobJsonConverter.cs b/src/Tableau.Migration/JsonConverters/JobJsonConverter.cs new file mode 100644 index 0000000..a4da43c --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/JobJsonConverter.cs @@ -0,0 +1,175 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Models; +using Tableau.Migration.Api.Rest.Models.Responses; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes a . + /// + internal class JobJsonConverter : JsonConverter + { + public JobJsonConverter() + { } + + public override void Write(Utf8JsonWriter writer, IJob value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + writer.WriteString(nameof(IJob.Id), value.Id); + writer.WriteString(nameof(IJob.Type), value.Type); + writer.WriteString(nameof(IJob.CreatedAtUtc), value.CreatedAtUtc); + + if (value.UpdatedAtUtc.HasValue) + { + writer.WriteString(nameof(IJob.UpdatedAtUtc), value.UpdatedAtUtc.Value); + } + + if (value.CompletedAtUtc.HasValue) + { + writer.WriteString(nameof(IJob.CompletedAtUtc), value.CompletedAtUtc.Value); + } + + writer.WriteNumber(nameof(IJob.ProgressPercentage), value.ProgressPercentage); + writer.WriteNumber(nameof(IJob.FinishCode), value.FinishCode); + + // Serializing StatusNotes + writer.WriteStartArray(nameof(IJob.StatusNotes)); + foreach (var statusNote in value.StatusNotes) + { + writer.WriteStartObject(); + writer.WriteString(nameof(IStatusNote.Type), statusNote.Type); + writer.WriteString(nameof(IStatusNote.Value), statusNote.Value); + writer.WriteString(nameof(IStatusNote.Text), statusNote.Text); + writer.WriteEndObject(); + } + writer.WriteEndArray(); + + writer.WriteEndObject(); + } + + + + public override IJob? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException("Expected StartObject token"); + } + + var jobResponse = new JobResponse.JobType(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the job object. + } + + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + switch (propertyName) + { + case nameof(IJob.Id): + jobResponse.Id = reader.GetGuid(); + break; + case nameof(IJob.Type): + jobResponse.Type = reader.GetString(); + break; + case nameof(IJob.CreatedAtUtc): + jobResponse.CreatedAt = reader.GetDateTime().ToString("o"); + break; + case nameof(IJob.UpdatedAtUtc): + jobResponse.UpdatedAt = reader.GetDateTime().ToString("o"); + break; + case nameof(IJob.CompletedAtUtc): + jobResponse.CompletedAt = reader.GetDateTime().ToString("o"); + break; + case nameof(IJob.ProgressPercentage): + jobResponse.Progress = reader.GetInt32(); + break; + case nameof(IJob.FinishCode): + jobResponse.FinishCode = reader.GetInt32(); + break; + case nameof(IJob.StatusNotes): + jobResponse.StatusNotes = ReadStatusNotes(ref reader); + break; + default: + reader.Skip(); // Skip unknown properties. + break; + } + } + } + + var jobResponseWrapper = new JobResponse { Item = jobResponse }; + return new Job(jobResponseWrapper); + } + + private JobResponse.JobType.StatusNoteType[] ReadStatusNotes(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.StartArray) + { + throw new JsonException("Expected StartArray token for StatusNotes"); + } + + var statusNotes = new List(); + + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) + { + if (reader.TokenType == JsonTokenType.StartObject) + { + var statusNote = new JobResponse.JobType.StatusNoteType(); + while (reader.Read() && reader.TokenType != JsonTokenType.EndObject) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + switch (propertyName) + { + case nameof(IStatusNote.Type): + statusNote.Type = reader.GetString(); + break; + case nameof(IStatusNote.Value): + statusNote.Value = reader.GetString(); + break; + case nameof(IStatusNote.Text): + statusNote.Text = reader.GetString(); + break; + default: + reader.Skip(); // Skip unknown properties. + break; + } + } + } + statusNotes.Add(statusNote); + } + } + + return statusNotes.ToArray(); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/JsonReaderUtils.cs b/src/Tableau.Migration/JsonConverters/JsonReaderUtils.cs new file mode 100644 index 0000000..3430d34 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/JsonReaderUtils.cs @@ -0,0 +1,104 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Tableau.Migration.JsonConverters +{ + internal static class JsonReaderUtils + { + /// + /// Verify that last json token was a + /// + internal static void AssertPropertyName(ref Utf8JsonReader reader, string? expected = null) + { + if (reader.TokenType != JsonTokenType.PropertyName) throw new JsonException("Expected property name"); + if (expected != null) + { + var actual = reader.GetString(); + if (!string.Equals(actual, expected, StringComparison.Ordinal)) + { + throw new JsonException($"Property value did not match expectation. Expected '{expected}' but got '{actual}'"); + } + } + } + + /// + /// Verify that last json token was a + /// + internal static void AssertStartArray(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.StartArray) throw new JsonException("Expected start of array"); + } + + /// + /// Verify that last json token was a + /// + internal static void AssertEndArray(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.EndArray) throw new JsonException("Expected end of array"); + } + + /// + /// Verify that last json token was a + /// + internal static void AssertStartObject(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.StartObject) throw new JsonException("Expected start of object"); + } + + /// + /// Verify that last json token was a + /// + internal static void AssertEndObject(ref Utf8JsonReader reader) + { + if (reader.TokenType != JsonTokenType.EndObject) throw new JsonException("Expected end of object"); + } + + /// + /// Read the next json token and verify that it's a + /// + internal static void ReadAndAssertPropertyName(ref Utf8JsonReader reader, string? expected = null) + { + reader.Read(); + AssertPropertyName(ref reader, expected); + } + + /// + /// Read the next json token and verify that it's a + /// + internal static void ReadAndAssertStartArray(ref Utf8JsonReader reader) + { + reader.Read(); + AssertStartArray(ref reader); + } + + /// + /// Read the next json token and verify that it's a + /// + internal static void ReadAndAssertStartObject(ref Utf8JsonReader reader) + { + reader.Read(); + AssertStartObject(ref reader); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/JsonWriterUtils.cs b/src/Tableau.Migration/JsonConverters/JsonWriterUtils.cs new file mode 100644 index 0000000..22df054 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/JsonWriterUtils.cs @@ -0,0 +1,48 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; + +namespace Tableau.Migration.JsonConverters +{ + internal static class JsonWriterUtils + { + internal static void WriteExceptionProperties(ref Utf8JsonWriter writer, Exception value) + { + writer.WriteString("Message", value.Message); + writer.WriteString("Type", value.GetType().FullName); + writer.WriteString("Source", value.Source); + writer.WriteString("StackTrace", value.StackTrace); + writer.WriteString("HelpLink", value.HelpLink); // Include HelpLink + writer.WriteNumber("HResult", value.HResult); // Include HResult + + // Leaving this at 1 level of depth. + if (value.InnerException != null) + { + writer.WriteStartObject("InnerException"); + writer.WriteString("Message", value.InnerException.Message); + writer.WriteString("Type", value.InnerException.GetType().FullName); + writer.WriteString("Source", value.InnerException.Source); + writer.WriteString("StackTrace", value.InnerException.StackTrace); + writer.WriteString("HelpLink", value.InnerException.HelpLink); // Include HelpLink for InnerException + writer.WriteNumber("HResult", value.InnerException.HResult); // Include HResult for InnerException + writer.WriteEndObject(); + } + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/PythonExceptionConverter.cs b/src/Tableau.Migration/JsonConverters/PythonExceptionConverter.cs new file mode 100644 index 0000000..7b78f77 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/PythonExceptionConverter.cs @@ -0,0 +1,106 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Python.Runtime; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes a . + /// + public class PythonExceptionConverter : JsonConverter + { + // **NOTE** + // I am not convinced this works. + // I've tried to create a PythonException manually to pass into the write and read method, but I can't create one. + // Every time I try I get an exception through in the Python.NET layer I get an exception thrown in the Python.NET layer. + // + // The other option is to actually run python code that will produce an PythonException, except that requires + // scaffolding as it requires the python library to be imported to PythonRuntime. + // See: https://github.com/pythonnet/pythonnet#embedding-python-in-net + // You must set Runtime.PythonDLL property or PYTHONNET_PYDLL environment variable starting with version 3.0, + // otherwise you will receive BadPythonDllException (internal, derived from MissingMethodException) upon calling Initialize. + // Typical values are python38.dll (Windows), libpython3.8.dylib (Mac), libpython3.8.so (most other Unix-like operating systems). + // This is difficult because I have no way to get a libpython. on mac. + // I could do this for just windows, but that would require us to add a python38.dll to the repo. + // + // I have tested that the "write" works manually. This is because I had a bug in the Python.TestApplication in a hook, that produced it. + // I have no idea how it actually made it into the manifest though, and at this point, I'm out of time. + // I have opened a user story to work on this some more at a later time. + + /// + /// Reads the JSON representation of the object. + /// + /// The reader to read from. + /// The type of the object. + /// The serializer options. + /// The deserialized object. + public override PythonException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + JsonReaderUtils.AssertStartObject(ref reader); + + string? message = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + string? propertyName = reader.GetString(); + Guard.AgainstNullOrEmpty(propertyName, nameof(propertyName)); + + reader.Read(); // Move to the property value. + + if (propertyName == "Message") + { + message = reader.GetString(); + Guard.AgainstNull(message, nameof(message)); // Message could be an empty string, so just check null + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + // Message must be deserialized by now + Guard.AgainstNull(message, nameof(message)); + var ret = new PythonException(PyType.Get(typeof(string)), PyType.None, null, message, null); + return ret; + } + + /// + /// Writes the JSON representation of the object. + /// + /// The writer to write to. + /// The object to write. + /// The serializer options. + public override void Write(Utf8JsonWriter writer, PythonException value, JsonSerializerOptions options) + { + if (value is PythonException pyException) + { + writer.WriteStartObject(); + writer.WriteString("ClassName", typeof(PythonException).FullName); + writer.WriteString("Message", pyException.Format()); + writer.WriteEndObject(); + return; + } + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/RestExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/RestExceptionJsonConverter.cs new file mode 100644 index 0000000..e22d638 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/RestExceptionJsonConverter.cs @@ -0,0 +1,145 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Rest; +using Tableau.Migration.Api.Rest.Models; +using Tableau.Migration.JsonConverters.SerializableObjects; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// Represents a collection of serializable entries, organized by a string key and a list of as the value. + /// This class extends to facilitate serialization and deserialization of migration manifest entries. + /// + public class RestExceptionJsonConverter : JsonConverter + { + /// + /// Reads and converts the JSON to type . + /// + /// The reader to deserialize objects or value types. + /// The type of object to convert. + /// Options to control the behavior during reading. + /// A object deserialized from JSON. + public override RestException Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + HttpMethod? httpMethod = null; + Uri? requestUri = null; + string? code = null; + string? detail = null; + string? summary = null; + string? exceptionMessage = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + switch (propertyName) + { + case nameof(RestException.HttpMethod): + var method = reader.GetString(); + if (method != null) + { + httpMethod = new HttpMethod(method); + } + break; + + case nameof(RestException.RequestUri): + var uriString = reader.GetString(); + if (uriString != null) + { + requestUri = new Uri(uriString); + } + break; + + case nameof(RestException.Code): + code = reader.GetString(); + break; + + case nameof(RestException.Detail): + detail = reader.GetString(); + break; + + case nameof(RestException.Summary): + summary = reader.GetString(); + break; + + case nameof(RestException.Message): + exceptionMessage = reader.GetString(); + break; + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + Guard.AgainstNull(exceptionMessage, nameof(exceptionMessage)); + + // Use the internal constructor for deserialization + return new RestException(httpMethod, requestUri, new Error { Code = code, Detail = detail, Summary = summary }, exceptionMessage); + } + + /// + /// Writes a specified object to JSON. + /// + /// The writer to serialize objects or value types. + /// The value to serialize. + /// Options to control the behavior during writing. + public override void Write(Utf8JsonWriter writer, RestException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + if (value.HttpMethod != null) + { + writer.WriteString(nameof(RestException.HttpMethod), value.HttpMethod.Method); + } + + if (value.RequestUri != null) + { + writer.WriteString(nameof(RestException.RequestUri), value.RequestUri.ToString()); + } + + if (value.Code != null) + { + writer.WriteString(nameof(RestException.Code), value.Code); + } + + if (value.Detail != null) + { + writer.WriteString(nameof(RestException.Detail), value.Detail); + } + + if (value.Summary != null) + { + writer.WriteString(nameof(RestException.Summary), value.Summary); + } + + JsonWriterUtils.WriteExceptionProperties(ref writer, value); + + writer.WriteEndObject(); + } + } +} diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonContentLocation.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentLocation.cs similarity index 52% rename from tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonContentLocation.cs rename to src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentLocation.cs index 3c294cf..b9fe5e5 100644 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonContentLocation.cs +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentLocation.cs @@ -15,29 +15,69 @@ // limitations under the License. // -using CommunityToolkit.Diagnostics; -using Tableau.Migration.TestComponents.JsonConverters.Exceptions; +using System; +using System.Linq; +using Tableau.Migration.JsonConverters.Exceptions; -namespace Tableau.Migration.TestComponents.JsonConverters.JsonObjects +namespace Tableau.Migration.JsonConverters.SerializableObjects { - - public class JsonContentLocation + /// + /// Represents a JSON serializable content location, providing details about the location of content within a migration context. + /// + public class SerializableContentLocation { + /// + /// Gets or sets the path segments that make up the content location. + /// public string[]? PathSegments { get; set; } + + /// + /// Gets or sets the path separator used between path segments. + /// public string? PathSeparator { get; set; } + + /// + /// Gets or sets the full path constructed from the path segments and separator. + /// public string? Path { get; set; } + + /// + /// Gets or sets the name of the content at this location. + /// public string? Name { get; set; } + + /// + /// Gets or sets a value indicating whether the content location is empty. + /// public bool IsEmpty { get; set; } + /// + /// Initializes a new instance of the class. + /// + public SerializableContentLocation() { } + + /// + /// Initializes a new instance of the class with details from an existing . + /// + /// The content location to serialize. + internal SerializableContentLocation(ContentLocation location) + { + PathSegments = location.PathSegments.ToArray(); + PathSeparator = location.PathSeparator; + Path = location.Path; + Name = location.Name; + IsEmpty = location.IsEmpty; + } + /// /// Throw exception if any values are still null /// public void VerifyDeseralization() { - Guard.IsNotNull(PathSegments, nameof(PathSegments)); - Guard.IsNotNull(PathSeparator, nameof(PathSeparator)); - Guard.IsNotNull(Path, nameof(Path)); - Guard.IsNotNull(Name, nameof(Name)); + Guard.AgainstNull(PathSegments, nameof(PathSegments)); + Guard.AgainstNull(PathSeparator, nameof(PathSeparator)); + Guard.AgainstNull(Path, nameof(Path)); + Guard.AgainstNull(Name, nameof(Name)); var expectedName = PathSegments.LastOrDefault() ?? string.Empty; if (!string.Equals(Name, expectedName, StringComparison.Ordinal)) diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentReference.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentReference.cs new file mode 100644 index 0000000..d2461cb --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableContentReference.cs @@ -0,0 +1,93 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using Tableau.Migration.Content; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a JSON serializable content reference. + /// + public class SerializableContentReference + { + /// + /// Gets or sets the unique identifier for the content. + /// + public string? Id { get; set; } + + /// + /// Gets or sets the URL associated with the content. + /// + public string? ContentUrl { get; set; } + + /// + /// Gets or sets the location information for the content. + /// + public SerializableContentLocation? Location { get; set; } + + /// + /// Gets or sets the name of the content. + /// + public string? Name { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public SerializableContentReference() { } + + /// + /// Initializes a new instance of the class with details from an . + /// + /// The content reference to serialize. + internal SerializableContentReference(IContentReference content) + { + Id = content.Id.ToString(); + ContentUrl = content.ContentUrl; + Location = new SerializableContentLocation(content.Location); + Name = content.Name; + } + + /// + /// Throw exception if any values are still null + /// + public void VerifyDeserialization() + { + Guard.AgainstNull(Id, nameof(Id)); + Guard.AgainstNull(ContentUrl, nameof(ContentUrl)); + Guard.AgainstNull(Location, nameof(Location)); + Guard.AgainstNull(Name, nameof(Name)); + } + + /// + /// Returns the current item as a + /// + /// + public ContentReferenceStub AsContentReferenceStub() + { + VerifyDeserialization(); + + var ret = new ContentReferenceStub( + Guid.Parse(Id!), + ContentUrl!, + Location!.AsContentLocation(), + Name!); + + return ret; + } + } +} \ No newline at end of file diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableEntryCollection.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableEntryCollection.cs new file mode 100644 index 0000000..32fcec6 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableEntryCollection.cs @@ -0,0 +1,50 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Tableau.Migration.Engine.Manifest; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a collection of serializable entries, organized by a string key and a list of as the value. + /// This class extends to facilitate serialization and deserialization of migration manifest entries. + /// + public class SerializableEntryCollection : Dictionary> + { + /// + /// Initializes a new instance of the class. + /// + public SerializableEntryCollection() { } + + /// + /// Initializes a new instance of the class with an existing dictionary of entries. + /// + /// The dictionary containing the initial entries for the collection. + public SerializableEntryCollection(Dictionary> entries) + { + foreach (var entry in entries) + { + Add(entry.Key, entry.Value); + } + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableException.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableException.cs new file mode 100644 index 0000000..72155a8 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableException.cs @@ -0,0 +1,47 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a serializable version of an exception, allowing exceptions to be serialized into JSON format. + /// + public class SerializableException + { + /// + /// Gets or sets the class name of the exception. + /// + public string? ClassName { get; set; } + + /// + /// Gets or sets the exception object. This property is not serialized and is used only for internal purposes. + /// + public Exception? Error { get; set; } = null; + + /// + /// Initializes a new instance of the class using the specified exception. + /// + /// The exception to serialize. + internal SerializableException(Exception ex) + { + ClassName = ex.GetType().FullName; + Error = ex; + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableManifestEntry.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableManifestEntry.cs new file mode 100644 index 0000000..bfa66b4 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableManifestEntry.cs @@ -0,0 +1,147 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Tableau.Migration.Engine.Manifest; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a JSON serializable entry in a migration manifest. This class implements + /// to allow for easy conversion between the manifest entry and its JSON representation. + /// + public class SerializableManifestEntry : IMigrationManifestEntry + { + /// + /// Gets or sets the source content reference. + /// + public SerializableContentReference? Source { get; set; } + + /// + /// Gets or sets the mapped location for the content. + /// + public SerializableContentLocation? MappedLocation { get; set; } + + /// + /// Gets or sets the destination content reference. + /// + public SerializableContentReference? Destination { get; set; } + + /// + /// Gets or sets the status of the migration for this entry. + /// + public int Status { get; set; } + + /// + /// Gets or sets a value indicating whether the content has been migrated. + /// + public bool HasMigrated { get; set; } + + /// + /// Gets or sets the list of errors encountered during the migration of this entry. + /// + public List? Errors { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public SerializableManifestEntry() { } + + /// + /// Initializes a new instance of the class with details from an . + /// + /// The migration manifest entry to serialize. + internal SerializableManifestEntry(IMigrationManifestEntry entry) + { + Source = new SerializableContentReference(entry.Source); + MappedLocation = new SerializableContentLocation(entry.MappedLocation); + Destination = entry.Destination == null ? null : new SerializableContentReference(entry.Destination); + Status = (int)entry.Status; + HasMigrated = entry.HasMigrated; + + Errors = entry.Errors.Select(e => new SerializableException(e)).ToList(); + } + + IContentReference IMigrationManifestEntry.Source => Source!.AsContentReferenceStub(); + + ContentLocation IMigrationManifestEntry.MappedLocation => MappedLocation!.AsContentLocation(); + + IContentReference? IMigrationManifestEntry.Destination => Destination?.AsContentReferenceStub(); + + MigrationManifestEntryStatus IMigrationManifestEntry.Status => (MigrationManifestEntryStatus)Status; + + bool IMigrationManifestEntry.HasMigrated => HasMigrated; + + IReadOnlyList IMigrationManifestEntry.Errors + { + get + { + if (Errors == null) + { + return Array.Empty(); + } + else + { + return Errors.Select(e => e.Error).ToImmutableArray(); + } + } + } + + /// + /// Sets the list of errors encountered during the migration of this entry. + /// + /// The list of errors to set. + public void SetErrors(List errors) + { + Errors = errors.Select(e => new SerializableException(e)).ToList(); + } + + /// + /// Throw exception if any values are still null + /// + public void VerifyDeseralization() + { + // Destination can be null, so we shouldn't do a nullability check on it + + Guard.AgainstNull(Source, nameof(Source)); + Guard.AgainstNull(MappedLocation, nameof(MappedLocation)); + + Source.VerifyDeserialization(); + MappedLocation.VerifyDeseralization(); + } + + /// + /// Returns current object as a + /// + /// + public IMigrationManifestEntry AsMigrationManifestEntry(IMigrationManifestEntryBuilder partition) + { + VerifyDeseralization(); + var ret = new MigrationManifestEntry(partition, this); + + return ret; + } + + /// + public bool Equals(IMigrationManifestEntry? other) + => MigrationManifestEntry.Equals(this, other); + } +} \ No newline at end of file diff --git a/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableMigrationManifest.cs b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableMigrationManifest.cs new file mode 100644 index 0000000..aa6cef4 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializableObjects/SerializableMigrationManifest.cs @@ -0,0 +1,129 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Linq; +using Microsoft.Extensions.Logging; +using Tableau.Migration.Engine.Manifest; +using Tableau.Migration.Resources; + +namespace Tableau.Migration.JsonConverters.SerializableObjects +{ + /// + /// Represents a serializable version of a migration manifest, which can be used for JSON serialization and deserialization. + /// + public class SerializableMigrationManifest + { + /// + /// Gets or sets the unique identifier for the migration plan. + /// + public Guid? PlanId { get; set; } + + /// + /// Gets or sets the unique identifier for the migration. + /// + public Guid? MigrationId { get; set; } + + /// + /// Gets or sets the list of errors encountered during the migration process. + /// + public List? Errors { get; set; } = new List(); + + /// + /// Gets or sets the collection of entries that are part of the migration manifest. + /// + public SerializableEntryCollection? Entries { get; set; } = new(); + + /// + /// Gets or sets the version of the migration manifest. + /// + public uint? ManifestVersion { get; set; } + + /// + /// Initializes a new instance of the class. + /// + public SerializableMigrationManifest() { } + + /// + /// Initializes a new instance of the class with details from an . + /// + /// The migration manifest to serialize. + public SerializableMigrationManifest(IMigrationManifest manifest) + { + PlanId = manifest.PlanId; + MigrationId = manifest.MigrationId; + ManifestVersion = manifest.ManifestVersion; + + Errors = manifest.Errors.Select(e => new SerializableException(e)).ToList(); + + foreach (var partitionType in manifest.Entries.GetPartitionTypes()) + { + Guard.AgainstNull(partitionType, nameof(partitionType)); + Guard.AgainstNullOrEmpty(partitionType.FullName, nameof(partitionType.FullName)); + + Entries.Add(partitionType!.FullName, manifest.Entries.ForContentType(partitionType).Select(entry => new SerializableManifestEntry(entry)).ToList()); + } + } + + /// + /// Converts the serializable migration manifest back into an instance. + /// + /// The shared resources localizer. + /// The logger factory. + /// An instance of . + public IMigrationManifest ToMigrationManifest(ISharedResourcesLocalizer localizer, ILoggerFactory loggerFactory) + { + VerifyDeserialization(); + + // Create the manifest to return + var manifest = new MigrationManifest(localizer, loggerFactory, PlanId!.Value, MigrationId!.Value); + + // Get the Tableau.Migration assembly to get the type from later + var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); + var tableauMigrationAssembly = loadedAssemblies.Where(a => a.ManifestModule.Name == "Tableau.Migration.dll").First(); + + // Copy the entries to the manifest + foreach (var partitionTypeStr in Entries!.Keys) + { + var partitionType = tableauMigrationAssembly.GetType(partitionTypeStr); + Guard.AgainstNull(partitionType, nameof(partitionType)); + + var partition = manifest.Entries.GetOrCreatePartition(partitionType); + + var partitionEntries = Entries.GetValueOrDefault(partitionTypeStr); + Guard.AgainstNull(partitionEntries, nameof(partitionEntries)); + + partition.CreateEntries(partitionEntries.ToImmutableArray()); + } + + manifest.AddErrors(Errors!.Where(e => e.Error is not null).Select(e => e.Error)!); + + return manifest; + } + + internal void VerifyDeserialization() + { + Guard.AgainstNull(PlanId, nameof(PlanId)); + Guard.AgainstNull(MigrationId, nameof(MigrationId)); + Guard.AgainstNull(Errors, nameof(Errors)); + Guard.AgainstNull(Entries, nameof(Entries)); + Guard.AgainstNull(ManifestVersion, nameof(ManifestVersion)); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/SerializedExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/SerializedExceptionJsonConverter.cs new file mode 100644 index 0000000..210add8 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/SerializedExceptionJsonConverter.cs @@ -0,0 +1,152 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Models; +using Tableau.Migration.JsonConverters.SerializableObjects; + + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that de/serializes a . + /// + public class SerializedExceptionJsonConverter : JsonConverter + { + internal static string GetNamespace(string fullTypeName) + { + if (string.IsNullOrEmpty(fullTypeName)) + { + throw new ArgumentException("The type name cannot be null or empty.", nameof(fullTypeName)); + } + + int lastDotIndex = fullTypeName.LastIndexOf('.'); + if (lastDotIndex == -1) + { + throw new ArgumentException("The type name does not contain a namespace.", nameof(fullTypeName)); + } + + return fullTypeName.Substring(0, lastDotIndex); + } + + /// + /// Reads a from JSON. + /// + /// The to read from. + /// The type of the object to convert. + /// The to use for deserialization. + /// The deserialized . + public override SerializableException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + SerializableException? ret = null; + + // Check if the reader has never been read from + if (reader.TokenStartIndex == 0 && reader.CurrentDepth == 0) + { + reader.Read(); + } + + JsonReaderUtils.AssertStartObject(ref reader); + + while (reader.Read()) + { + // Make sure it starts with "ClassName" + JsonReaderUtils.AssertPropertyName(ref reader, Constants.CLASS_NAME); + + // Read the type of exception class this is + reader.Read(); + + string exceptionTypeStr = reader.GetString() ?? ""; + + string exceptionNamespace = GetNamespace(exceptionTypeStr); + + Type? exceptionType = null; + if (exceptionNamespace == "System") + { + exceptionType = Type.GetType($"{exceptionTypeStr}"); + } + else + { + exceptionType = Type.GetType($"{exceptionTypeStr}, {exceptionNamespace}"); + } + + // Check if this is a built-in exception type + if (exceptionType is null) + { + // exception type is not a built in type, looking through Tableau.Migration.dll + exceptionType = typeof(FailedJobException).Assembly.GetType(exceptionTypeStr); + if (exceptionType is null) + { + if (exceptionTypeStr == "Python.Runtime.PythonException") + { + exceptionType = typeof(Python.Runtime.PythonException); + } + else if (exceptionType != typeof(Python.Runtime.PythonException)) + { + throw new InvalidOperationException($"Unable to get type of '{exceptionTypeStr}'"); + } + + } + } + + // Make sure the next property is the Exception + JsonReaderUtils.ReadAndAssertPropertyName(ref reader, Constants.EXCEPTION); + + // Deserialize the exception + reader.Read(); + var ex = JsonSerializer.Deserialize(ref reader, exceptionType, options) as Exception; + + Guard.AgainstNull(ex, nameof(ex)); + + ret = new SerializableException(ex); + + JsonReaderUtils.AssertEndObject(ref reader); + reader.Read(); + break; + } + + return ret; + } + + /// + /// Writes a to JSON. + /// + /// The to write to. + /// The to write. + /// The to use for serialization. + public override void Write(Utf8JsonWriter writer, SerializableException value, JsonSerializerOptions options) + { + Guard.AgainstNullOrEmpty(value.ClassName, nameof(value.ClassName)); + Guard.AgainstNull(value.Error, nameof(value.Error)); + + // Start our serialized Exception object + writer.WriteStartObject(); + + // Save the type of exception it is + writer.WriteString(Constants.CLASS_NAME, value.ClassName); + + // Save the exception itself + writer.WritePropertyName(Constants.EXCEPTION); + JsonSerializer.Serialize(writer, value.Error, value.Error.GetType(), options); + + // End of serialized exception object + writer.WriteEndObject(); + } + } +} diff --git a/src/Tableau.Migration/JsonConverters/TimeoutJobExceptionJsonConverter.cs b/src/Tableau.Migration/JsonConverters/TimeoutJobExceptionJsonConverter.cs new file mode 100644 index 0000000..6c42490 --- /dev/null +++ b/src/Tableau.Migration/JsonConverters/TimeoutJobExceptionJsonConverter.cs @@ -0,0 +1,83 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; +using Tableau.Migration.Api.Models; + +namespace Tableau.Migration.JsonConverters +{ + /// + /// JsonConverter that serializes a . It does not support reading exceptions back in. + /// + internal class TimeoutJobExceptionJsonConverter : JsonConverter + { + public TimeoutJobExceptionJsonConverter() + { } + + public override void Write(Utf8JsonWriter writer, TimeoutJobException value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + JsonWriterUtils.WriteExceptionProperties(ref writer, value); + + // Serialize the Job property if it's not null + if (value.Job != null) + { + writer.WritePropertyName("Job"); + JsonSerializer.Serialize(writer, value.Job, options); + } + + writer.WriteEndObject(); + } + + public override TimeoutJobException? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + IJob? job = null; + string? message = null; + + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.PropertyName) + { + var propertyName = reader.GetString(); + reader.Read(); // Move to the property value. + + if (propertyName == "Job") + { + job = JsonSerializer.Deserialize(ref reader, options); + } + else if(propertyName == "Message") + { + message = reader.GetString(); + } + } + else if (reader.TokenType == JsonTokenType.EndObject) + { + break; // End of the object. + } + } + + Guard.AgainstNull(message, nameof(message)); // Message could be an empty string, so just check null + + return new TimeoutJobException(job, message); + } + + + } +} diff --git a/src/Tableau.Migration/Tableau.Migration.csproj b/src/Tableau.Migration/Tableau.Migration.csproj index fe480a8..63ff1bb 100644 --- a/src/Tableau.Migration/Tableau.Migration.csproj +++ b/src/Tableau.Migration/Tableau.Migration.csproj @@ -45,16 +45,17 @@ Note: This SDK is specific for migrating from Tableau Server to Tableau Cloud. I - - + + + all runtime; build; native; contentfiles; analyzers; buildtransitive - + diff --git a/tests/Python.ExampleApplication.Tests/Python.ExampleApplication.Tests.pyproj b/tests/Python.ExampleApplication.Tests/Python.ExampleApplication.Tests.pyproj new file mode 100644 index 0000000..61382e0 --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/Python.ExampleApplication.Tests.pyproj @@ -0,0 +1,60 @@ + + + Debug + 2.0 + {b1884017-8e25-4a26-8c89-d9d880cfa392} + + + + + ..\..\src\Python\dist + . + . + Python.ExampleApplication.Tests + Python.ExampleApplication.Tests + MSBuild|env|$(MSBuildProjectFullPath) + False + Pytest + + + true + false + + + true + false + + + + + + + + + + + + + + + + + env + 3.11 + env (Python 3.11 (64-bit)) + Scripts\python.exe + Scripts\pythonw.exe + PYTHONPATH + X64 + + + + + + + + + + \ No newline at end of file diff --git a/tests/Python.ExampleApplication.Tests/README.md b/tests/Python.ExampleApplication.Tests/README.md new file mode 100644 index 0000000..d51ccfc --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/README.md @@ -0,0 +1,11 @@ +# Intro + +This is the test project for the Python Example Application we reference in code samples for documentation. + +## Instructions to run tests + +Refer to [contributing.md](../../src/Python/CONTRIBUTING.md#first-steps) for instructions to install Python and Hatch(optional). Then, + +- Switch to this directory +- Run `python -m pip install -r .\requirements.txt` +- If using `pytest`, simply run `pytest`. If using hatch, run `hatch run test:test` diff --git a/tests/Python.ExampleApplication.Tests/pyproject.toml b/tests/Python.ExampleApplication.Tests/pyproject.toml new file mode 100644 index 0000000..994186a --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/pyproject.toml @@ -0,0 +1,35 @@ +[project] +name = "tableau_migration_example_app_tests" +dynamic = ["version"] + +authors = [ + { name="Salesforce, Inc." }, +] +description = "Tableau Migration SDK - Example Application Tests" + +classifiers = [ + "Programming Language :: Python :: 3", + "Operating System :: OS Independent", +] + +dependencies = [ + "tableau_migration", + "pytest>=8.2.2" +] + +[tool.ruff] +# Enable pycodestyle (`E`) and Pyflakes (`F`) codes by default. +select = ["E", "F"] +ignore = ["E501"] + +[tool.hatch.envs.test] +dev-mode = false +dependencies = [ + "pytest>=8.2.2" +] + +[tool.hatch.envs.test.scripts] +test = "pytest" + +[[tool.hatch.envs.test.matrix]] +python = ["3.8", "3.9", "3.10", "3.11", "3.12"] \ No newline at end of file diff --git a/tests/Python.ExampleApplication.Tests/pytest.ini b/tests/Python.ExampleApplication.Tests/pytest.ini new file mode 100644 index 0000000..b5220b5 --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +testpaths = + tests + +pythonpath = + ..\..\examples\Python.ExampleApplication \ No newline at end of file diff --git a/tests/Python.ExampleApplication.Tests/requirements.txt b/tests/Python.ExampleApplication.Tests/requirements.txt new file mode 100644 index 0000000..652bc81 --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/requirements.txt @@ -0,0 +1,2 @@ +tableau_migration +pytest>=8.2.2 diff --git a/tests/Python.ExampleApplication.Tests/tests/__init__.py b/tests/Python.ExampleApplication.Tests/tests/__init__.py new file mode 100644 index 0000000..b7e800d --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/tests/__init__.py @@ -0,0 +1,61 @@ +# Copyright (c) 2024, Salesforce, Inc. +# SPDX-License-Identifier: Apache-2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Testing module for the Python.TestApplication.""" + +import sys +import os +from os.path import abspath +from pathlib import Path +import tableau_migration + +print("Adding example application paths to sys.path") +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/batch_migration_completed")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/filters")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/mappings")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/migration_action_completed")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/post_publish")) +sys.path.append(abspath(Path(__file__).parent.resolve().__str__() + "/../../../examples/Python.ExampleApplication/hooks/transformers")) + + +if os.environ.get('MIG_SDK_PYTHON_BUILD', 'false').lower() == 'true': + print("MIG_SDK_PYTHON_BUILD set to true. Building dotnet binaries for python tests.") + # Not required for GitHub Actions + import subprocess + import shutil + _bin_path = abspath(Path(__file__).parent.resolve().__str__() + "/../../../src/Python/src/tableau_migration/bin") + sys.path.append(_bin_path) + + shutil.rmtree(_bin_path, True) + print("Building required binaries") + _build_script = abspath(Path(__file__).parent.resolve().__str__() + "/../../../src/Python/scripts/build-package.ps1") + subprocess.run(["pwsh", "-c", _build_script, "-Fast", "-IncludeTests"]) +else: + print("MIG_SDK_PYTHON_BUILD set to false. Skipping dotnet build for python tests.") + +print("Adding test helpers to sys.path") +_autofixture_helper_path = abspath(Path(__file__).parent.resolve().__str__() + "/../../../src/Python/tests/helpers") +sys.path.append(_autofixture_helper_path) + +from tableau_migration import clr +clr.AddReference("AutoFixture") +clr.AddReference("AutoFixture.AutoMoq") +clr.AddReference("Moq") +clr.AddReference("Tableau.Migration.Tests") +clr.AddReference("Tableau.Migration") + + + diff --git a/tests/Python.ExampleApplication.Tests/tests/test_default_project_filter.py b/tests/Python.ExampleApplication.Tests/tests/test_default_project_filter.py new file mode 100644 index 0000000..77dd8b6 --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/tests/test_default_project_filter.py @@ -0,0 +1,41 @@ +# Copyright (c) 2024, Salesforce, Inc. +# SPDX-License-Identifier: Apache-2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from autofixture import AutoFixtureTestBase + +from default_project_filter import DefaultProjectFilter + +from tableau_migration import ContentMigrationItem +from tableau_migration import IProject + +from Tableau.Migration.Content import IProject as DotnetIProject +from Tableau.Migration.Engine import ContentMigrationItem as DotnetContentMigrationItem + +class TestDefaultProjectFilter(AutoFixtureTestBase): + def test_init(self): + DefaultProjectFilter() + + def test_should_migrate(self): + + dotnet_item = self.create(DotnetContentMigrationItem[DotnetIProject]) + item = ContentMigrationItem[IProject](dotnet_item) + + filter = DefaultProjectFilter() + result = filter.should_migrate(item) + + assert item.source_item.name !='Default' + assert result == True + + \ No newline at end of file diff --git a/tests/Python.ExampleApplication.Tests/tests/test_log_migration_batches_hook.py b/tests/Python.ExampleApplication.Tests/tests/test_log_migration_batches_hook.py new file mode 100644 index 0000000..12da4aa --- /dev/null +++ b/tests/Python.ExampleApplication.Tests/tests/test_log_migration_batches_hook.py @@ -0,0 +1,24 @@ +# Copyright (c) 2024, Salesforce, Inc. +# SPDX-License-Identifier: Apache-2 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from log_migration_batches_hook import LogMigrationBatchesHookForUsers + +class TestLogMigrationBatchesHookForUsers(): + def test_init(self): + LogMigrationBatchesHookForUsers() + + def test_execute(self): + hook = LogMigrationBatchesHookForUsers(); + assert hook._content_type == "User" diff --git a/tests/Python.TestApplication/Python.TestApplication.pyproj b/tests/Python.TestApplication/Python.TestApplication.pyproj index 443cfbb..6405905 100644 --- a/tests/Python.TestApplication/Python.TestApplication.pyproj +++ b/tests/Python.TestApplication/Python.TestApplication.pyproj @@ -27,13 +27,10 @@ - - - @@ -54,9 +51,6 @@ X64 - - - - CA2007 - - - - - - - - - - - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - runtime; build; native; contentfiles; analyzers; buildtransitive - all - - - - - - - - diff --git a/tests/Tableau.Migration.TestComponents.Tests/Usings.cs b/tests/Tableau.Migration.TestComponents.Tests/Usings.cs deleted file mode 100644 index d64812f..0000000 --- a/tests/Tableau.Migration.TestComponents.Tests/Usings.cs +++ /dev/null @@ -1,18 +0,0 @@ -// -// Copyright (c) 2024, Salesforce, Inc. -// SPDX-License-Identifier: Apache-2 -// -// Licensed under the Apache License, Version 2.0 (the "License") -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -global using Xunit; \ No newline at end of file diff --git a/tests/Tableau.Migration.TestComponents/IServiceCollectionExtensions.cs b/tests/Tableau.Migration.TestComponents/IServiceCollectionExtensions.cs deleted file mode 100644 index e4a1361..0000000 --- a/tests/Tableau.Migration.TestComponents/IServiceCollectionExtensions.cs +++ /dev/null @@ -1,39 +0,0 @@ -// -// Copyright (c) 2024, Salesforce, Inc. -// SPDX-License-Identifier: Apache-2 -// -// Licensed under the Apache License, Version 2.0 (the "License") -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -using Microsoft.Extensions.DependencyInjection; -using Tableau.Migration.TestComponents.Engine.Manifest; - -namespace Tableau.Migration.TestComponents -{ - /// - /// Static class containing extension methods for objects. - /// - public static class IServiceCollectionExtensions - { - /// - /// Registers services required for using the Tableau Migration SDK Test Components. - /// - /// The service collection to register services with. - /// The same service collection as the parameter. - public static IServiceCollection AddTestComponents(this IServiceCollection services) - { - services.AddSingleton(); - return services; - } - } -} diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/ExceptionJsonConverter.cs b/tests/Tableau.Migration.TestComponents/JsonConverters/ExceptionJsonConverter.cs deleted file mode 100644 index 380c78a..0000000 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/ExceptionJsonConverter.cs +++ /dev/null @@ -1,99 +0,0 @@ -// -// Copyright (c) 2024, Salesforce, Inc. -// SPDX-License-Identifier: Apache-2 -// -// Licensed under the Apache License, Version 2.0 (the "License") -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -using System.Collections.Immutable; -using System.Reflection; -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.Logging; -using Python.Runtime; - -namespace Tableau.Migration.TestComponents.JsonConverters -{ - // Source: https://code-maze.com/dotnet-serialize-exceptions-as-json/ - /// - /// JsonConverter that serializes a . It does not support reading exceptions back in. - /// - public class ExceptionJsonConverter : JsonConverter - { - private readonly ILogger _logger; - private static readonly ImmutableHashSet IGNORED_PROPERTY_TYPES = new[] - { - typeof(Type), - typeof(CancellationToken) - }.ToImmutableHashSet(); - - public ExceptionJsonConverter(ILogger logger) - { - _logger = logger; - } - - public override Exception? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - // Reading exception back in is not supported. - reader.TrySkip(); - return null; - } - - public override void Write(Utf8JsonWriter writer, Exception value, JsonSerializerOptions options) - { - var exceptionType = value.GetType(); - - if (value is PythonException pyException) - { - writer.WriteStartObject(); - writer.WriteString("ClassName", exceptionType.FullName); - writer.WriteString("Message", pyException.Format()); - writer.WriteEndObject(); - return; - } - - var properties = exceptionType.GetProperties() - .Where(e => !IGNORED_PROPERTY_TYPES.Contains(e.PropertyType)) - .Where(e => e.PropertyType.Namespace != typeof(MemberInfo).Namespace) - .ToImmutableArray(); - - writer.WriteStartObject(); - writer.WriteString("ClassName", exceptionType.FullName); - foreach (var property in properties) - { - // We can't write cancellation tokens because they are disposed by the time we get here. - if(property.PropertyType == typeof(CancellationToken)) - continue; - - try - { - var propertyValue = property.GetValue(value, null); - - if (propertyValue is null) - continue; - - writer.WritePropertyName(property.Name); - - JsonSerializer.Serialize(writer, propertyValue, property.PropertyType, options); - } - catch (Exception ex) - { - _logger.LogError(ex, "Unable to write property"); - throw new Exception($"Error serializing {exceptionType.FullName}.{property.Name} (type {property.PropertyType}).", ex); - } - } - - writer.WriteEndObject(); - } - } -} diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonContentReference.cs b/tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonContentReference.cs deleted file mode 100644 index d291c57..0000000 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonContentReference.cs +++ /dev/null @@ -1,59 +0,0 @@ -// -// Copyright (c) 2024, Salesforce, Inc. -// SPDX-License-Identifier: Apache-2 -// -// Licensed under the Apache License, Version 2.0 (the "License") -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -using CommunityToolkit.Diagnostics; -using Tableau.Migration.Content; - -namespace Tableau.Migration.TestComponents.JsonConverters.JsonObjects -{ - - public class JsonContentReference - { - public string? Id { get; set; } - public string? ContentUrl { get; set; } - public JsonContentLocation? Location { get; set; } - public string? Name { get; set; } - - /// - /// Throw exception if any values are still null - /// - public void VerifyDeseralization() - { - Guard.IsNotNull(Id, nameof(Id)); - Guard.IsNotNull(ContentUrl, nameof(ContentUrl)); - Guard.IsNotNull(Location, nameof(Location)); - Guard.IsNotNull(Name, nameof(Name)); - } - - /// - /// Returns the current item as a - /// - /// - public ContentReferenceStub AsContentReferenceStub() - { - VerifyDeseralization(); - - var ret = new ContentReferenceStub( - Guid.Parse(Id!), - ContentUrl!, - Location!.AsContentLocation(), - Name!); - - return ret; - } - } -} \ No newline at end of file diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonManifestEntry.cs b/tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonManifestEntry.cs deleted file mode 100644 index f1e12c1..0000000 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/JsonObjects/JsonManifestEntry.cs +++ /dev/null @@ -1,72 +0,0 @@ -// -// Copyright (c) 2024, Salesforce, Inc. -// SPDX-License-Identifier: Apache-2 -// -// Licensed under the Apache License, Version 2.0 (the "License") -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -using CommunityToolkit.Diagnostics; -using Tableau.Migration.Engine.Manifest; - -namespace Tableau.Migration.TestComponents.JsonConverters.JsonObjects -{ - public class JsonManifestEntry : IMigrationManifestEntry - { - public JsonContentReference? Source { get; set; } - public JsonContentLocation? MappedLocation { get; set; } - public JsonContentReference? Destination { get; set; } - public int Status { get; set; } - public bool HasMigrated { get; set; } - - IContentReference IMigrationManifestEntry.Source => Source!.AsContentReferenceStub(); - - ContentLocation IMigrationManifestEntry.MappedLocation => MappedLocation!.AsContentLocation(); - - IContentReference? IMigrationManifestEntry.Destination => Destination?.AsContentReferenceStub(); - - MigrationManifestEntryStatus IMigrationManifestEntry.Status => (MigrationManifestEntryStatus)Status; - - bool IMigrationManifestEntry.HasMigrated => HasMigrated; - - IReadOnlyList IMigrationManifestEntry.Errors => Array.Empty(); - - /// - /// Throw exception if any values are still null - /// - public void VerifyDeseralization() - { - // Destination can be null, so we shouldn't do a nullability check on it - - Guard.IsNotNull(Source, nameof(Source)); - Guard.IsNotNull(MappedLocation, nameof(MappedLocation)); - - Source.VerifyDeseralization(); - MappedLocation.VerifyDeseralization(); - } - - /// - /// Returns current object as a - /// - /// - public IMigrationManifestEntry AsMigrationManifestEntry(IMigrationManifestEntryBuilder partition) - { - VerifyDeseralization(); - var ret = new MigrationManifestEntry(partition, this); - - return ret; - } - - public bool Equals(IMigrationManifestEntry? other) - => MigrationManifestEntry.Equals(this, other); - } -} \ No newline at end of file diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/MigrationManifestEntryCollectionReader.cs b/tests/Tableau.Migration.TestComponents/JsonConverters/MigrationManifestEntryCollectionReader.cs deleted file mode 100644 index 9c4049e..0000000 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/MigrationManifestEntryCollectionReader.cs +++ /dev/null @@ -1,177 +0,0 @@ -// -// Copyright (c) 2024, Salesforce, Inc. -// SPDX-License-Identifier: Apache-2 -// -// Licensed under the Apache License, Version 2.0 (the "License") -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -using System.Text.Json; -using System.Text.Json.Serialization; -using Microsoft.Extensions.Logging; -using Tableau.Migration.Engine.Manifest; -using Tableau.Migration.Resources; -using Tableau.Migration.TestComponents.JsonConverters.JsonObjects; - -namespace Tableau.Migration.TestComponents.JsonConverters -{ - public class MigrationManifestEntryCollectionReader : JsonConverter - { - private readonly ISharedResourcesLocalizer _localizer; - private readonly ILoggerFactory _loggerFactory; - - public MigrationManifestEntryCollectionReader(ISharedResourcesLocalizer localizer, ILoggerFactory loggerFactory) - { - _localizer = localizer; - _loggerFactory = loggerFactory; - } - - public override MigrationManifestEntryCollection? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - // Get the assembly name of a well known Tableau.Migration Type. This is required later to get the type from a string - var assemblyName = typeof(MigrationManifestEntryCollection).Assembly.FullName; - MigrationManifestEntryCollection collection = new(_localizer, _loggerFactory); - - while (reader.Read()) - { - switch (reader.TokenType) - { - // Once we reach a property name, we can dive in and deserialize based on the property type - case JsonTokenType.PropertyName: - var propName = reader.GetString(); - if (propName == Constants.PARTITION) - { - // We're in a known property, read the next token which should be string representing the property type - reader.Read(); - - // Translate the partion type string into a type object - string propTypeStr = reader.GetString() ?? throw new JsonException("Property Type should not be null"); - Type contentType = Type.GetType($"{propTypeStr}, {assemblyName}") ?? throw new TypeLoadException($"Unable to convert {propTypeStr} to type"); ; - - MigrationManifestEntryCollectionReader.ReadAndAssertPropertyName(ref reader, Constants.ENTRIES); - - // Get all the entries and deserialize them - var deserializeEntries = JsonSerializer.Deserialize>(ref reader) ?? throw new JsonException("Unable to deserialize entries"); - - // Get the partition for this Content Type - var partition = collection.GetOrCreatePartition(contentType); - var partitionAsBuilder = partition as IMigrationManifestEntryBuilder ?? throw new Exception($"Unable to convert partition to {nameof(IMigrationManifestEntryBuilder)}"); - - // Convert all the entries - var migrationEntries = deserializeEntries.Select(x => x.AsMigrationManifestEntry(partitionAsBuilder)).ToList(); - - // Add converted entries to partition - partition.CreateEntries(migrationEntries); - } - else - { - throw new JsonException($"Unknown property {propName}"); - } - break; - - // We're done with the array, so we're returning out collection - case JsonTokenType.EndArray: - return collection; - - // No ops - case JsonTokenType.StartArray: - case JsonTokenType.StartObject: - case JsonTokenType.EndObject: - case JsonTokenType.True: - case JsonTokenType.False: - case JsonTokenType.Null: - break; - - // We should never be hitting a json string or number as the property reader should be doing that - // That means we should skip the property - case JsonTokenType.String: - case JsonTokenType.Number: - default: - reader.Skip(); - break; - } - - } - - return collection; - } - - #region - Asserters - - - /// - /// Verify that last json token was a - /// - internal static void AssertPropertyName(ref Utf8JsonReader reader) - { - if (reader.TokenType != JsonTokenType.PropertyName) throw new JsonException("Expected property name"); - } - - /// - /// Verify that last json token was a - /// - internal static void AssertStartArray(ref Utf8JsonReader reader) - { - if (reader.TokenType != JsonTokenType.StartArray) throw new JsonException("Expected start of array"); - } - - /// - /// Verify that last json token was a - /// - internal static void AssertStartObject(ref Utf8JsonReader reader) - { - if (reader.TokenType != JsonTokenType.StartObject) throw new JsonException("Expected start of object"); - } - - /// - /// Read the next json token and verify that it's a - /// - /// The expected string value. If null, then it's not checked - internal static void ReadAndAssertPropertyName(ref Utf8JsonReader reader, string? expected = null) - { - reader.Read(); - MigrationManifestEntryCollectionReader.AssertPropertyName(ref reader); - if (expected != null) - { - var actual = reader.GetString(); - string.Equals(actual, expected, StringComparison.Ordinal); - } - } - - /// - /// Read the next json token and verify that it's a - /// - internal static void ReadAndAssertStartArray(ref Utf8JsonReader reader) - { - reader.Read(); - MigrationManifestEntryCollectionReader.AssertStartArray(ref reader); - } - - /// - /// Read the next json token and verify that it's a - /// - internal static void ReadAndAssertStartObject(ref Utf8JsonReader reader) - { - reader.Read(); - MigrationManifestEntryCollectionReader.AssertStartObject(ref reader); - } - - #endregion - - #region - Not Implemented - - public override void Write(Utf8JsonWriter writer, IMigrationManifestEntryCollectionEditor value, JsonSerializerOptions options) - { - throw new NotImplementedException($"{nameof(MigrationManifestEntryCollectionReader)} only supports deserialization of {nameof(MigrationManifest)}s. Use {nameof(MigrationManifestEntryCollectionWriter)} to serialize"); - } - #endregion - - } -} diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/MigrationManifestEntryCollectionWriter.cs b/tests/Tableau.Migration.TestComponents/JsonConverters/MigrationManifestEntryCollectionWriter.cs deleted file mode 100644 index aeed85e..0000000 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/MigrationManifestEntryCollectionWriter.cs +++ /dev/null @@ -1,61 +0,0 @@ -// -// Copyright (c) 2024, Salesforce, Inc. -// SPDX-License-Identifier: Apache-2 -// -// Licensed under the Apache License, Version 2.0 (the "License") -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -using System.Text.Json; -using System.Text.Json.Serialization; -using Tableau.Migration.Engine.Manifest; - -namespace Tableau.Migration.TestComponents.JsonConverters -{ - public class MigrationManifestEntryCollectionWriter : JsonConverter - { - public override void Write(Utf8JsonWriter writer, IMigrationManifestEntryCollection value, JsonSerializerOptions options) - { - writer.WriteStartArray(); // Start an array of all entries - - foreach (var partitionType in value.GetPartitionTypes()) - { - writer.WriteStartObject(); // Each partion is an object in the array of entries - writer.WriteString(Constants.PARTITION, partitionType.FullName); // Write the type of the partion - - writer.WritePropertyName(Constants.ENTRIES); // Entries per partion (yes, entries is overloaded here but naming is hard) - writer.WriteStartArray(); // Start the array of entries per partition - - var entries = value.ForContentType(partitionType); - foreach (var entry in entries) - { - JsonSerializer.Serialize(writer, entry, options); // Each individual entry can be reflected, so we can let the JsonSerializer handle it - } - - writer.WriteEndArray(); // End of the array of entries per partion - writer.WriteEndObject(); // End of partion object - } - - writer.WriteEndArray(); // End of array of all entries - } - - #region - Not Implemented - - - public override MigrationManifestEntryCollection? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - throw new NotImplementedException($"{nameof(MigrationManifestEntryCollectionWriter)} only supports serialization of {nameof(MigrationManifest)}s. Use {nameof(MigrationManifestEntryCollectionReader)} to deserialize"); - } - - #endregion - - } -} diff --git a/tests/Tableau.Migration.TestComponents/JsonConverters/SerializeableMigrationManifest.cs b/tests/Tableau.Migration.TestComponents/JsonConverters/SerializeableMigrationManifest.cs deleted file mode 100644 index fce3145..0000000 --- a/tests/Tableau.Migration.TestComponents/JsonConverters/SerializeableMigrationManifest.cs +++ /dev/null @@ -1,35 +0,0 @@ -// -// Copyright (c) 2024, Salesforce, Inc. -// SPDX-License-Identifier: Apache-2 -// -// Licensed under the Apache License, Version 2.0 (the "License") -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -using System.Collections.Immutable; -using Tableau.Migration.Engine.Manifest; - -namespace Tableau.Migration.TestComponents.JsonConverters -{ - public class SerializeableMigrationManifest - { - public Guid PlanId { get; set; } - - public Guid MigrationId { get; set; } - - public IReadOnlyList Errors { get; set; } = new ImmutableArray(); - - public IMigrationManifestEntryCollectionEditor Entries { get; set; } = null!; - - public uint ManifestVersion { get; set; } - } -} diff --git a/tests/Tableau.Migration.TestComponents/Tableau.Migration.TestComponents.csproj b/tests/Tableau.Migration.TestComponents/Tableau.Migration.TestComponents.csproj deleted file mode 100644 index 51729f7..0000000 --- a/tests/Tableau.Migration.TestComponents/Tableau.Migration.TestComponents.csproj +++ /dev/null @@ -1,18 +0,0 @@ - - - - net6.0;net8.0 - enable - enable - - - - - - - - - - - - diff --git a/tests/Tableau.Migration.Tests/AutoFixtureTestBase.cs b/tests/Tableau.Migration.Tests/AutoFixtureTestBase.cs index 4555367..0be21a5 100644 --- a/tests/Tableau.Migration.Tests/AutoFixtureTestBase.cs +++ b/tests/Tableau.Migration.Tests/AutoFixtureTestBase.cs @@ -21,6 +21,8 @@ using System.Threading; using AutoFixture; using AutoFixture.Kernel; +using Moq; +using Tableau.Migration.Engine.Manifest; namespace Tableau.Migration.Tests { @@ -31,6 +33,8 @@ public abstract class AutoFixtureTestBase /// protected readonly IFixture AutoFixture = CreateFixture(); + protected readonly Mock MockEntryBuilder; + protected readonly CancellationTokenSource CancelSource = new(); protected CancellationToken Cancel => CancelSource.Token; @@ -44,6 +48,8 @@ public AutoFixtureTestBase() { TestCancellationTimeout = TimeSpan.FromSeconds(15); } + + MockEntryBuilder = Create>(); } /// diff --git a/tests/Tableau.Migration.Tests/AutoFixtureTestBaseTests.cs b/tests/Tableau.Migration.Tests/AutoFixtureTestBaseTests.cs new file mode 100644 index 0000000..d738770 --- /dev/null +++ b/tests/Tableau.Migration.Tests/AutoFixtureTestBaseTests.cs @@ -0,0 +1,72 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Linq; +using Tableau.Migration.JsonConverters.Exceptions; +using Xunit; + +namespace Tableau.Migration.Tests +{ + public class AutoFixtureTestBaseTests : AutoFixtureTestBase + { + [Fact] + public void Verify_CreateErrors_create_all_exceptions() + { + // Create all the exceptions + var errors = FixtureFactory.CreateErrors(AutoFixture); + + var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); + + var tableauMigrationAssembly = loadedAssemblies.Where(a => a.ManifestModule.Name == "Tableau.Migration.dll").First(); + + // Find all the exception types in the Tableau.Migration assembly + var exceptionTypes = tableauMigrationAssembly.GetTypes() + .Where(t => t.BaseType == typeof(Exception)) + .Where(t => t != typeof(MismatchException)) + .ToList(); + + // Assert that all the types in exceptionTypes exist in the errors list + Assert.True(exceptionTypes.All(t => errors.Any(e => e.GetType() == t))); + } + + [Fact] + public void Verify_CreateErrors_nullable_properties_not_null() + { + string[] ignoredPropertyNames = new string[] { "InnerException" }; + + // Call CreateErrors + var errors = FixtureFactory.CreateErrors(AutoFixture); + + // Verify that every property in all the objects is not null + foreach (var error in errors) + { + var properties = error.GetType().GetProperties() + .Where(prop => !ignoredPropertyNames.Contains(prop.Name)) + .ToArray(); + + Assert.All(properties, (prop) => Assert.NotNull(prop)); + + foreach (var property in properties) + { + var value = property.GetValue(error); + Assert.NotNull(value); + } + } + } + } +} diff --git a/tests/Tableau.Migration.Tests/ExceptionComparerTests.cs b/tests/Tableau.Migration.Tests/ExceptionComparerTests.cs new file mode 100644 index 0000000..fb46e5a --- /dev/null +++ b/tests/Tableau.Migration.Tests/ExceptionComparerTests.cs @@ -0,0 +1,123 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using Xunit; +using Tableau.Migration; + +namespace Tableau.Migration.Tests.Unit +{ + public class ExceptionComparerTests + { + // Custom exception class for testing IEquatable + private class EquatableException : Exception, IEquatable + { + public EquatableException(string message) : base(message) + { } + + public bool Equals(EquatableException? other) + { + return other != null && Message == other.Message; + } + + public override int GetHashCode() + { + return base.GetHashCode(); + } + } + + private readonly ExceptionComparer _comparer = new ExceptionComparer(); + + [Fact] + public void TestEquals_BothNull_ReturnsTrue() + { + Assert.True(_comparer.Equals(null, null)); + } + + [Fact] + public void TestEquals_OneNull_ReturnsFalse() + { + var ex = new Exception(); + Assert.False(_comparer.Equals(null, ex)); + Assert.False(_comparer.Equals(ex, null)); + } + + [Fact] + public void TestEquals_DifferentTypes_ReturnsFalse() + { + var ex1 = new Exception(); + var ex2 = new InvalidOperationException(); + Assert.False(_comparer.Equals(ex1, ex2)); + } + + [Fact] + public void TestEquals_SameTypeDifferentMessages_ReturnsFalse() + { + var ex1 = new Exception("Message 1"); + var ex2 = new Exception("Message 2"); + Assert.False(_comparer.Equals(ex1, ex2)); + } + + [Fact] + public void TestEquals_SameTypeSameMessage_ReturnsTrue() + { + var ex1 = new Exception("Message"); + var ex2 = new Exception("Message"); + Assert.True(_comparer.Equals(ex1, ex2)); + } + + [Fact] + public void TestEquals_ImplementsIEquatable_ReturnsTrue() + { + var ex1 = new EquatableException("Message"); + var ex2 = new EquatableException("Message"); + Assert.True(_comparer.Equals(ex1, ex2)); + } + + [Fact] + public void TestEquals_ExceptionVsEquatableException_ReturnsFalse() + { + var standardException = new Exception("Message"); + var equatableException = new EquatableException("Message"); + Assert.False(_comparer.Equals(standardException, equatableException)); + } + + + [Fact] + public void TestGetHashCode_DifferentMessages_DifferentHashCodes() + { + var ex1 = new Exception("Message 1"); + var ex2 = new Exception("Message 2"); + Assert.NotEqual(_comparer.GetHashCode(ex1), _comparer.GetHashCode(ex2)); + } + + [Fact] + public void TestGetHashCode_SameMessage_SameHashCode() + { + var ex1 = new Exception("Message"); + var ex2 = new Exception("Message"); + Assert.Equal(_comparer.GetHashCode(ex1), _comparer.GetHashCode(ex2)); + } + + [Fact] + public void TestGetHashCode_ImplementsIEquatable_ConsistentHashCode() + { + var ex = new EquatableException("Message"); + Assert.Equal(ex.GetHashCode(), _comparer.GetHashCode(ex)); + } + } +} diff --git a/tests/Tableau.Migration.Tests/FixtureFactory.cs b/tests/Tableau.Migration.Tests/FixtureFactory.cs index 3cc2fbc..c26b883 100644 --- a/tests/Tableau.Migration.Tests/FixtureFactory.cs +++ b/tests/Tableau.Migration.Tests/FixtureFactory.cs @@ -18,13 +18,25 @@ using System; using System.Collections.Generic; using System.IO; +using System.IO.Abstractions; +using System.IO.Abstractions.TestingHelpers; using System.Linq; +using System.Net.Http; using AutoFixture; using AutoFixture.AutoMoq; +using AutoFixture.Kernel; +using Microsoft.Extensions.Logging; using Moq; +using Tableau.Migration.Api.Models; using Tableau.Migration.Api.Rest.Models.Requests; using Tableau.Migration.Api.Rest.Models.Responses; +using Tableau.Migration.Content; using Tableau.Migration.Content.Schedules; +using Tableau.Migration.Engine.Manifest; +using Tableau.Migration.Engine.Pipelines; +using Tableau.Migration.JsonConverters.Exceptions; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Tableau.Migration.Resources; using CloudResponses = Tableau.Migration.Api.Rest.Models.Responses.Cloud; using ServerResponses = Tableau.Migration.Api.Rest.Models.Responses.Server; @@ -40,6 +52,8 @@ private static IFixture Customize(IFixture fixture) fixture.Customizations.Add(new ImmutableCollectionSpecimenBuilder()); + fixture.Register(() => new MockFileSystem()); + fixture.Register(() => fixture.Create().Object); fixture.Register(() => fixture.Create>().Object); @@ -265,7 +279,161 @@ string GetRandomExtractType() ); #endregion + #region - TimeoutException - + + fixture.Register(() => + { + return new TimeoutJobException(new Job(fixture.Create()), fixture.Create()); + }); + + #endregion + + #region - Serializable Objects - + + fixture.Register(() => CreateSerializableContentLocation(fixture)); + + fixture.Customize(composer => composer + .With(c => c.Id, Guid.NewGuid().ToString())); + + fixture.Register(() => CreateSerializableManifestEntry(fixture)); + + #endregion + + #region - MigrationManifestEntry - + fixture.Register(() => + { + var mockEntryBuilder = fixture.Create>(); + return new MigrationManifestEntry(mockEntryBuilder.Object, fixture.Create()); + }); + + #endregion + + #region - MigrationManifest - + + fixture.Register(() => CreateMigrationManifest(fixture)); + fixture.Register(() => CreateMigrationManifest(fixture)); + + #endregion + return fixture; } + + /// + /// This creates a that follows the requirements of a + /// + internal static SerializableContentLocation CreateSerializableContentLocation(IFixture fixture) + { + var ret = new SerializableContentLocation(); + + string[] pathSegments = fixture.CreateMany().ToArray(); + + ret.PathSeparator = Constants.PathSeparator; + ret.PathSegments = pathSegments; + ret.Path = string.Join(Constants.PathSeparator, pathSegments); + ret.Name = pathSegments.LastOrDefault() ?? string.Empty; + ret.IsEmpty = (pathSegments.Length == 0); + + return ret; + } + + /// + /// Creates a SerializableManifestEntry that follows the requirements of + /// + /// + internal static SerializableManifestEntry CreateSerializableManifestEntry(IFixture fixture) + { + var ret = new SerializableManifestEntry(); + + ret.Source = fixture.Create(); + ret.Destination = fixture.Create(); + ret.MappedLocation = ret.Destination.Location; + ret.Status = (int)fixture.Create(); + ret.SetErrors(CreateErrors(fixture)); + + return ret; + } + + internal static MigrationManifest CreateMigrationManifest(IFixture fixture) + { + var ret = new MigrationManifest(fixture.Create(), fixture.Create(), Guid.NewGuid(), Guid.NewGuid()); + + foreach (var type in ServerToCloudMigrationPipeline.ContentTypes) + { + var p = ret.Entries.GetOrCreatePartition(type.ContentType); + p.CreateEntries(fixture.CreateMany().ToList()); + } + + ret.AddErrors(CreateErrors(fixture)); + + return ret; + } + + /// + /// Creates a list of exceptions of all the classes in Tableau.Migration.dll that inhert from Exception + /// The exception types are registered with AutoFixture to produce fully populated objects. + /// + /// List of exceptions of different types. + public static List CreateErrors(IFixture fixture, int countOfEach = 1) + { + // Note: + // This is bit of a hack, but we need a complete exception object. + // The easiest way I can produce this is to throw the exception, catch it, + // and then add it to the list. + + var ret = new List(); + + // First, a default exception + try + { + throw fixture.Create(); + } + catch (Exception ex) + { + ret.Add(ex); + } + + // Now a specialized exception + try + { + var httpException = new HttpRequestException(fixture.Create(), fixture.Create(), System.Net.HttpStatusCode.BadRequest); + httpException.HelpLink = fixture.Create(); + + throw httpException; + } + catch (HttpRequestException ex) + { + ret.Add(ex); + } + + var loadedAssemblies = AppDomain.CurrentDomain.GetAssemblies(); + var tableauMigrationAssembly = loadedAssemblies.Where(a => a.ManifestModule.Name == "Tableau.Migration.dll").First(); + + var exceptionTypes = tableauMigrationAssembly.GetTypes() + .Where(t => t.BaseType == typeof(Exception)) + .Where(t => t != typeof(MismatchException)) // MismatchException will never be in a manifest + .ToList(); + + foreach (var exceptionType in exceptionTypes) + { + for (int i = 0; i < countOfEach; i++) + { + try + { + throw (Exception)fixture.Create(exceptionType, new SpecimenContext(fixture)); + } + catch (Exception ex) + { + if (ex.HelpLink is null) + { + ex.HelpLink = fixture.Create(); + } + + ret.Add(ex); + } + } + } + + return ret; + } } } diff --git a/tests/Tableau.Migration.Tests/ObjectExtensions.cs b/tests/Tableau.Migration.Tests/ObjectExtensions.cs index 9748d3b..9f188fa 100644 --- a/tests/Tableau.Migration.Tests/ObjectExtensions.cs +++ b/tests/Tableau.Migration.Tests/ObjectExtensions.cs @@ -16,11 +16,21 @@ // using System; +using System.Linq; namespace Tableau.Migration.Tests { public static class ObjectExtensions { public static bool IsDisposable(this object obj) => obj is IDisposable || obj is IAsyncDisposable; + + public static bool ImplementsEquatable(this object obj) + { + if (obj == null) return false; + + var type = obj.GetType(); + // Check if the type implements IEquatable<> interface + return type.GetInterfaces().Any(i => i.IsGenericType && i.GetGenericTypeDefinition() == typeof(IEquatable<>)); + } } } diff --git a/tests/Tableau.Migration.Tests/Simulation/Tests/Api/WorkbooksApiClientTests.cs b/tests/Tableau.Migration.Tests/Simulation/Tests/Api/WorkbooksApiClientTests.cs index 6255ddb..8a3afe6 100644 --- a/tests/Tableau.Migration.Tests/Simulation/Tests/Api/WorkbooksApiClientTests.cs +++ b/tests/Tableau.Migration.Tests/Simulation/Tests/Api/WorkbooksApiClientTests.cs @@ -126,7 +126,7 @@ public async Task Returns_success_on_success() Api.Data.AddWorkbook(workbook, fileData: Constants.DefaultEncoding.GetBytes(workbook.ToXml())); - var result = await sitesClient.Workbooks.DownloadWorkbookAsync(workbook.Id, true, Cancel); + var result = await sitesClient.Workbooks.DownloadWorkbookAsync(workbook.Id, Cancel); Assert.Empty(result.Errors); Assert.True(result.Success); diff --git a/tests/Tableau.Migration.Tests/Simulation/Tests/IncrementalMigrationTests.cs b/tests/Tableau.Migration.Tests/Simulation/Tests/IncrementalMigrationTests.cs index c8d1f4f..ab1cdf0 100644 --- a/tests/Tableau.Migration.Tests/Simulation/Tests/IncrementalMigrationTests.cs +++ b/tests/Tableau.Migration.Tests/Simulation/Tests/IncrementalMigrationTests.cs @@ -54,7 +54,7 @@ public async Task IncrementalFilterAsync() { //Scenario: We are migrating workbooks but have mistakenly //included a filter that prevents some of the workbooks from migrating. - //We want to re-run the migration an only migrate the workbooks we missed the first run. + //We want to re-run the migration and only migrate the workbooks we missed the first run. //Arrange - create source content to migrate. var sourceProjects = PrepareSourceProjectsData(); diff --git a/tests/Tableau.Migration.Tests/Tableau.Migration.Tests.csproj b/tests/Tableau.Migration.Tests/Tableau.Migration.Tests.csproj index 91ab1fb..d88a72e 100644 --- a/tests/Tableau.Migration.Tests/Tableau.Migration.Tests.csproj +++ b/tests/Tableau.Migration.Tests/Tableau.Migration.Tests.csproj @@ -20,11 +20,11 @@ - + - + all runtime; build; native; contentfiles; analyzers; buildtransitive @@ -32,8 +32,8 @@ - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive diff --git a/tests/Tableau.Migration.Tests/Unit/Api/DataSourcesApiClientTests.cs b/tests/Tableau.Migration.Tests/Unit/Api/DataSourcesApiClientTests.cs index 39b9af7..87230fa 100644 --- a/tests/Tableau.Migration.Tests/Unit/Api/DataSourcesApiClientTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Api/DataSourcesApiClientTests.cs @@ -24,6 +24,7 @@ using Moq; using Tableau.Migration.Api; using Tableau.Migration.Api.Rest.Models.Responses; +using Tableau.Migration.Config; using Tableau.Migration.Content; using Tableau.Migration.Tests.Unit.Api.Permissions; using Xunit; @@ -34,6 +35,12 @@ public class DataSourcesApiClientTests { public abstract class DataSourcesApiClientTest : PermissionsApiClientTestBase { + public DataSourcesApiClientTest() + { + MockConfigReader + .Setup(x => x.Get()) + .Returns(new ContentTypesOptions()); + } internal DataSourcesApiClient DataSourcesApiClient => GetApiClient(); } @@ -145,7 +152,7 @@ public async Task ErrorAsync() var dataSourceId = Guid.NewGuid(); - var result = await ApiClient.DownloadDataSourceAsync(dataSourceId, true, Cancel); + var result = await ApiClient.DownloadDataSourceAsync(dataSourceId, Cancel); result.AssertFailure(); @@ -164,7 +171,7 @@ public async Task FailureResponseAsync() var dataSourceId = Guid.NewGuid(); - var result = await ApiClient.DownloadDataSourceAsync(dataSourceId, true, Cancel); + var result = await ApiClient.DownloadDataSourceAsync(dataSourceId, Cancel); result.AssertFailure(); @@ -185,13 +192,13 @@ public async Task SuccessAsync() var dataSourceId = Guid.NewGuid(); - var result = await ApiClient.DownloadDataSourceAsync(dataSourceId, false, Cancel); + var result = await ApiClient.DownloadDataSourceAsync(dataSourceId, Cancel); result.AssertSuccess(); Assert.NotNull(result.Value); var request = MockHttpClient.AssertSingleRequest(); - request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/sites/{SiteId}/datasources/{dataSourceId}/content?includeExtract=False"); + request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/sites/{SiteId}/datasources/{dataSourceId}/content?includeExtract=True"); } } diff --git a/tests/Tableau.Migration.Tests/Unit/Api/WorkbooksApiClientTests.cs b/tests/Tableau.Migration.Tests/Unit/Api/WorkbooksApiClientTests.cs index 89ad442..5a5506a 100644 --- a/tests/Tableau.Migration.Tests/Unit/Api/WorkbooksApiClientTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Api/WorkbooksApiClientTests.cs @@ -27,6 +27,7 @@ using Tableau.Migration.Api; using Tableau.Migration.Api.Models; using Tableau.Migration.Api.Rest.Models.Responses; +using Tableau.Migration.Config; using Tableau.Migration.Content; using Tableau.Migration.Content.Files; using Tableau.Migration.Content.Permissions; @@ -39,6 +40,12 @@ public class WorkbooksApiClientTests { public abstract class WorkbooksApiClientTest : PermissionsApiClientTestBase { + public WorkbooksApiClientTest() + { + MockConfigReader + .Setup(x => x.Get()) + .Returns(new ContentTypesOptions()); + } internal WorkbooksApiClient WorkbooksApiClient => GetApiClient(); } @@ -213,7 +220,7 @@ public async Task ErrorAsync() var workbookId = Guid.NewGuid(); - var result = await ApiClient.DownloadWorkbookAsync(workbookId, true, Cancel); + var result = await ApiClient.DownloadWorkbookAsync(workbookId, Cancel); result.AssertFailure(); @@ -232,7 +239,7 @@ public async Task FailureResponseAsync() var workbookId = Guid.NewGuid(); - var result = await ApiClient.DownloadWorkbookAsync(workbookId, true, Cancel); + var result = await ApiClient.DownloadWorkbookAsync(workbookId, Cancel); result.AssertFailure(); @@ -254,13 +261,13 @@ public async Task SuccessAsync() var workbookId = Guid.NewGuid(); - var result = await ApiClient.DownloadWorkbookAsync(workbookId, false, Cancel); + var result = await ApiClient.DownloadWorkbookAsync(workbookId, Cancel); result.AssertSuccess(); Assert.NotNull(result.Value); var request = MockHttpClient.AssertSingleRequest(); - request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/sites/{SiteId}/workbooks/{workbookId}/content?includeExtract=False"); + request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/sites/{SiteId}/workbooks/{workbookId}/content?includeExtract=True"); } } diff --git a/tests/Tableau.Migration.Tests/Unit/Config/ConfigReaderTests.cs b/tests/Tableau.Migration.Tests/Unit/Config/ConfigReaderTests.cs index b8b20f4..68eb5bb 100644 --- a/tests/Tableau.Migration.Tests/Unit/Config/ConfigReaderTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Config/ConfigReaderTests.cs @@ -67,24 +67,33 @@ protected static List GetContentTypesOptionsTestData() new() { Type = "Workbook", - BatchSize = 226 + BatchSize = 226, + IncludeExtractEnabled = false }, new() { Type = "DataSource", - BatchSize = 227 + BatchSize = 227, + IncludeExtractEnabled = false } ]; - public void AssertCustomResult(ContentTypesOptions expected, ContentTypesOptions actual) + public void AssertCustomBatchSizeResult(ContentTypesOptions expected, ContentTypesOptions actual) { Assert.Equal(expected.BatchSize, actual.BatchSize); } + + public void AssertCustomIncludeExtractResult(ContentTypesOptions expected, ContentTypesOptions actual) + { + Assert.Equal(expected.IncludeExtractEnabled, actual.IncludeExtractEnabled); + } public void AssertDefaultResult(ContentTypesOptions actual) { Assert.Equal(ContentTypesOptions.Defaults.BATCH_SIZE, actual.BatchSize); + Assert.Equal(ContentTypesOptions.Defaults.BATCH_PUBLISHING_ENABLED, actual.BatchPublishingEnabled); + Assert.Equal(ContentTypesOptions.Defaults.INCLUDE_EXTRACT_ENABLED, actual.IncludeExtractEnabled); } public class GetContentTypeSpecific : ConfigReaderTests @@ -96,11 +105,14 @@ public void GetsCustomValues() var testData = GetContentTypesOptionsTestData(); - AssertCustomResult(testData.First(i => i.Type == "User"), Reader.Get()); - AssertCustomResult(testData.First(i => i.Type == "Group"), Reader.Get()); - AssertCustomResult(testData.First(i => i.Type == "Project"), Reader.Get()); - AssertCustomResult(testData.First(i => i.Type == "Workbook"), Reader.Get()); - AssertCustomResult(testData.First(i => i.Type == "DataSource"), Reader.Get()); + AssertCustomBatchSizeResult(testData.First(i => i.Type == "User"), Reader.Get()); + AssertCustomBatchSizeResult(testData.First(i => i.Type == "Group"), Reader.Get()); + AssertCustomBatchSizeResult(testData.First(i => i.Type == "Project"), Reader.Get()); + AssertCustomBatchSizeResult(testData.First(i => i.Type == "Workbook"), Reader.Get()); + AssertCustomBatchSizeResult(testData.First(i => i.Type == "DataSource"), Reader.Get()); + + AssertCustomIncludeExtractResult(testData.First(i => i.Type == "Workbook"), Reader.Get()); + AssertCustomIncludeExtractResult(testData.First(i => i.Type == "DataSource"), Reader.Get()); } [Fact] diff --git a/tests/Tableau.Migration.Tests/Unit/Content/Schedules/VolatileCacheTests.cs b/tests/Tableau.Migration.Tests/Unit/Content/Schedules/VolatileCacheTests.cs index 6d66ba3..5e9028f 100644 --- a/tests/Tableau.Migration.Tests/Unit/Content/Schedules/VolatileCacheTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Content/Schedules/VolatileCacheTests.cs @@ -20,7 +20,6 @@ using System.Collections.Immutable; using System.Linq; using System.Threading.Tasks; -using Tableau.Migration.Api.Rest.Models.Types; using Tableau.Migration.Content.Schedules; using Tableau.Migration.Content.Schedules.Cloud; using Xunit; @@ -50,9 +49,9 @@ public async Task EmptyList_LoadsOnce() new Dictionary<(ExtractRefreshContentType, Guid), ImmutableList>()); }); - var result1 = await Cache.GetAndRelease((ExtractRefreshContentType.DataSource,Guid.NewGuid()), Cancel); + var result1 = await Cache.GetAndRelease((ExtractRefreshContentType.DataSource, Guid.NewGuid()), Cancel); - var result2 = await Cache.GetAndRelease((ExtractRefreshContentType.Workbook,Guid.NewGuid()), Cancel); + var result2 = await Cache.GetAndRelease((ExtractRefreshContentType.Workbook, Guid.NewGuid()), Cancel); Assert.Null(result1); Assert.Null(result2); @@ -70,7 +69,7 @@ public async Task SingleItem_LoadsAndReturnsOnce() { loaded++; return Task.FromResult( - new Dictionary<(ExtractRefreshContentType, Guid), ImmutableList> + new Dictionary<(ExtractRefreshContentType, Guid), ImmutableList> { [(ExtractRefreshContentType.DataSource, id)] = list }); @@ -102,7 +101,7 @@ public async Task MultipleItems_LoadsAndReturnsOnce() { loaded++; return Task.FromResult( - new Dictionary<(ExtractRefreshContentType, Guid), ImmutableList> + new Dictionary<(ExtractRefreshContentType, Guid), ImmutableList> { [(ExtractRefreshContentType.DataSource, id1)] = list1, [(ExtractRefreshContentType.Workbook, id2)] = list2, @@ -168,7 +167,7 @@ public async Task SingleItem_LoadsAndReturnsOnce() { loaded++; return Task.FromResult( - new Dictionary + new Dictionary { [id] = item }); @@ -209,7 +208,7 @@ public async Task MultipleItems_LoadsAndReturnsOnce() { loaded++; return Task.FromResult( - new Dictionary + new Dictionary { [id1] = item1, [id2] = item2, @@ -221,7 +220,7 @@ public async Task MultipleItems_LoadsAndReturnsOnce() .Range(1, totalThreads) .Select(x => Cache .GetAndRelease( - x % 3 != 0 + x % 3 != 0 ? x % 3 != 1 ? Guid.NewGuid() : id2 diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Hooks/Transformers/Default/CloudScheduleCompatibilityTransformerTests.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Hooks/Transformers/Default/CloudScheduleCompatibilityTransformerTests.cs index 630db6e..f0fa912 100644 --- a/tests/Tableau.Migration.Tests/Unit/Engine/Hooks/Transformers/Default/CloudScheduleCompatibilityTransformerTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Hooks/Transformers/Default/CloudScheduleCompatibilityTransformerTests.cs @@ -55,7 +55,7 @@ public async Task Skips_intervals_longer_than_1_hour(string frequency) // Assert Assert.NotNull(result); Assert.Equal(input.Schedule.FrequencyDetails.Intervals.Count, result.Schedule.FrequencyDetails.Intervals.Count); - Assert.Empty(Logger.Messages.Where(m => m.LogLevel == LogLevel.Warning)); + Assert.Empty(Logger.Messages.Where(m => m.LogLevel == LogLevel.Warning).ToList()); } [Fact] diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestEntryCollectionTests.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestEntryCollectionTests.cs index a1e369b..3bef19a 100644 --- a/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestEntryCollectionTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestEntryCollectionTests.cs @@ -184,12 +184,8 @@ public void LazyCreatesAndReusesPartitions() #region - Equality - public class Equality : MigrationManifestEntryCollectionTest { - protected readonly Mock MockEntryBuilder; - public Equality() - { - MockEntryBuilder = Create>(); - } + { } private List CreateMigrationManifestEntries(int? count = 10) { diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestEntryTests.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestEntryTests.cs index 8942832..d4330a1 100644 --- a/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestEntryTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestEntryTests.cs @@ -16,7 +16,9 @@ // using System; +using System.Collections.Generic; using System.Linq; +using AutoFixture; using Moq; using Tableau.Migration.Content; using Tableau.Migration.Engine.Manifest; @@ -28,11 +30,19 @@ public class MigrationManifestEntryTests { public class MigrationManifestEntryTest : AutoFixtureTestBase { - protected readonly Mock MockEntryBuilder; - public MigrationManifestEntryTest() { - MockEntryBuilder = Create>(); + AutoFixture.Register(() => + { + return new Exception(Create()); + }); + + AutoFixture.Register(() => + { + var ret = new MigrationManifestEntry(MockEntryBuilder.Object, Create()); + ret.SetFailed(FixtureFactory.CreateErrors(AutoFixture)); + return ret; + }); } } @@ -40,6 +50,21 @@ public MigrationManifestEntryTest() public class Ctor : MigrationManifestEntryTest { + IMigrationManifestEntry CreateManifestEntry() + { + var errors = new List(); + errors.Add(new Exception("Test Error")); + + var ret = new Mock(); + ret.Setup(x => x.Source).Returns(Create()); + ret.Setup(x => x.Destination).Returns(Create()); + ret.Setup(x => x.MappedLocation).Returns(Create()); + ret.Setup(x => x.Status).Returns(MigrationManifestEntryStatus.Migrated); + ret.Setup(x => x.Errors).Returns(new List(errors)); + + return ret.Object; + } + [Fact] public void FromSourceReference() { @@ -57,7 +82,7 @@ public void FromSourceReference() [Fact] public void FromPreviousMigration() { - var previousEntry = Create(); + var previousEntry = CreateManifestEntry(); var e = new MigrationManifestEntry(MockEntryBuilder.Object, previousEntry); @@ -65,20 +90,15 @@ public void FromPreviousMigration() Assert.Equal(previousEntry.MappedLocation, e.MappedLocation); Assert.Equal(previousEntry.Status, e.Status); Assert.Equal(previousEntry.HasMigrated, e.HasMigrated); - - //Errors are reset. - Assert.NotEmpty(previousEntry.Errors); - Assert.Empty(e.Errors); - - //Destination is reset. - Assert.NotNull(previousEntry.Destination); - Assert.Null(e.Destination); + Assert.Equal(previousEntry.Destination, e.Destination); + Assert.Equal(previousEntry.Errors, e.Errors); } [Fact] public void FromUpdatedPreviousMigration() { - var previousEntry = Create(); + var previousEntry = CreateManifestEntry(); + var sourceRef = Create(); var e = new MigrationManifestEntry(MockEntryBuilder.Object, previousEntry, sourceRef); @@ -87,14 +107,8 @@ public void FromUpdatedPreviousMigration() Assert.Equal(previousEntry.MappedLocation, e.MappedLocation); Assert.Equal(previousEntry.Status, e.Status); Assert.Equal(previousEntry.HasMigrated, e.HasMigrated); - - //Errors are reset. - Assert.NotEmpty(previousEntry.Errors); - Assert.Empty(e.Errors); - - //Destination is reset. - Assert.NotNull(previousEntry.Destination); - Assert.Null(e.Destination); + Assert.Equal(previousEntry.Destination, e.Destination); + Assert.Equal(previousEntry.Errors, e.Errors); } } @@ -416,21 +430,34 @@ public void DestinationNull() } [Fact] - public void ErrorsDoNotCauseInequality() + public void ErrorsDifferent() { var e1 = new MigrationManifestEntry(MockEntryBuilder.Object, BaseSource); var e2 = new MigrationManifestEntry(MockEntryBuilder.Object, BaseSource); - // Set failed, but with different errors e1.SetFailed(CreateMany(5).ToList()); - e2.SetFailed(); + e2.SetFailed(CreateMany(5).ToList()); - // Both ManifestEntries are the same, except one has errors set. They should be equal. - Assert.True(e1.Equals(e2)); - Assert.True(e2.Equals(e1)); + Assert.False(e1.Equals(e2)); + Assert.False(e2.Equals(e1)); - Assert.True(e1 == e2); - Assert.False(e1 != e2); + Assert.False(e1 == e2); + Assert.True(e1 != e2); + } + + [Fact] + public void ErrorsNull() + { + var e1 = new MigrationManifestEntry(MockEntryBuilder.Object, BaseSource); + var e2 = new MigrationManifestEntry(MockEntryBuilder.Object, BaseSource); + + e1.SetFailed(CreateMany(5).ToList()); + + Assert.False(e1.Equals(e2)); + Assert.False(e2.Equals(e1)); + + Assert.False(e1 == e2); + Assert.True(e1 != e2); } } diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestFactoryTests.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestFactoryTests.cs index e80d157..16eadf5 100644 --- a/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestFactoryTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/MigrationManifestFactoryTests.cs @@ -18,6 +18,7 @@ using System; using System.Collections.Immutable; using System.Linq; +using AutoFixture; using Microsoft.Extensions.Logging; using Moq; using Tableau.Migration.Engine; @@ -61,6 +62,10 @@ public Create() [Fact] public void InitializesEmptyManifestWithInput() { + // Customize the AutoFixture instance to remove the customization for IMigrationManifest that was created from AutoFixtureTestBase/FixtureFactory + AutoFixture.Customize(c => c.FromFactory(() => + new MigrationManifest(AutoFixture.Create(), AutoFixture.Create(), Guid.NewGuid(), Guid.NewGuid()))); + var manifest = _factory.Create(_mockInput.Object, _migrationId); Assert.Equal(_mockInput.Object.Plan.PlanId, manifest.PlanId); diff --git a/tests/Tableau.Migration.TestComponents.Tests/Engine/Manifest/TestMigrationManifestSerializer.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/TestMigrationManifestSerializer.cs similarity index 73% rename from tests/Tableau.Migration.TestComponents.Tests/Engine/Manifest/TestMigrationManifestSerializer.cs rename to tests/Tableau.Migration.Tests/Unit/Engine/Manifest/TestMigrationManifestSerializer.cs index 69d03c4..e5cd319 100644 --- a/tests/Tableau.Migration.TestComponents.Tests/Engine/Manifest/TestMigrationManifestSerializer.cs +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Manifest/TestMigrationManifestSerializer.cs @@ -15,19 +15,35 @@ // limitations under the License. // +using System; +using System.IO; using System.IO.Abstractions; using System.IO.Abstractions.TestingHelpers; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; using AutoFixture; using Microsoft.Extensions.Logging; using Moq; using Tableau.Migration.Engine.Manifest; using Tableau.Migration.Resources; -using Tableau.Migration.TestComponents.Engine.Manifest; +using Xunit; -namespace Tableau.Migration.TestComponents.Tests.Engine.Manifest +namespace Tableau.Migration.Tests.Unit.Engine.Manifest { public class TestMigrationManifestSerializer : AutoFixtureTestBase { + // If you need to debug these tests and you need access to the file that is saved and loaded, + // you need to make some temporary changes to this file. + // + // The TestMigrationManifestSerializer ctor creates a MockFileSystem, so files are not actually + // saved to disk. If you want to see the manifest that is created, change MockFileSystem line to + // AutoFixture.Register(() => new FileSystem()); + // That will mean the test will use the real file system. + // + // The actual tests also a the temp file to save the manifest to. You can change that to a real filepath + // so it's easier to find during the actual debugging. + public TestMigrationManifestSerializer() { AutoFixture.Register(() => new MockFileSystem()); @@ -54,7 +70,7 @@ public async Task ManifestSaveLoadAsync() // Assert Assert.NotNull(loadedManifest); - Assert.Equal(manifest, loadedManifest); + Assert.Equal(manifest as MigrationManifest, loadedManifest); } [Fact] diff --git a/tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonContentLocation.cs b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentLocation.cs similarity index 77% rename from tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonContentLocation.cs rename to tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentLocation.cs index f65194d..d55b4d1 100644 --- a/tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonContentLocation.cs +++ b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentLocation.cs @@ -15,17 +15,19 @@ // limitations under the License. // -using Tableau.Migration.TestComponents.JsonConverters.Exceptions; -using Tableau.Migration.TestComponents.JsonConverters.JsonObjects; +using System; +using Tableau.Migration.JsonConverters.Exceptions; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Xunit; -namespace Tableau.Migration.TestComponents.Tests.JsonConverter.JsonObjects +namespace Tableau.Migration.Tests.Unit.JsonConverter.SerializableObjects { - public class TestJsonContentLocation : AutoFixtureTestBase + public class TestSerializableContentLocation : AutoFixtureTestBase { [Fact] public void AsContentLocation() { - var input = Create(); + var input = Create(); var contentLocation = input.AsContentLocation(); @@ -39,7 +41,7 @@ public void AsContentLocation() [Fact] public void BadDeserialization_NullPath() { - var input = Create(); + var input = Create(); input.Path = null; Assert.Throws(() => input.AsContentLocation()); @@ -48,7 +50,7 @@ public void BadDeserialization_NullPath() [Fact] public void BadDeserialization_NullPathSegments() { - var input = Create(); + var input = Create(); input.PathSegments = null; Assert.Throws(() => input.AsContentLocation()); @@ -57,7 +59,7 @@ public void BadDeserialization_NullPathSegments() [Fact] public void BadDeserialization_NullPathSeperator() { - var input = Create(); + var input = Create(); input.PathSeparator = null; Assert.Throws(() => input.AsContentLocation()); @@ -66,7 +68,7 @@ public void BadDeserialization_NullPathSeperator() [Fact] public void BadDeserialization_NullName() { - var input = Create(); + var input = Create(); input.Name = null; Assert.Throws(() => input.AsContentLocation()); @@ -75,7 +77,7 @@ public void BadDeserialization_NullName() [Fact] public void BadDeserialization_PathDoesNotMatchSegments() { - var input = Create(); + var input = Create(); input.Path = "Path"; Assert.Throws(() => input.AsContentLocation()); @@ -84,7 +86,7 @@ public void BadDeserialization_PathDoesNotMatchSegments() [Fact] public void BadDeserialization_NameDoesNotMatchSegments() { - var input = Create(); + var input = Create(); input.Name = "Name"; Assert.Throws(() => input.AsContentLocation()); @@ -93,7 +95,7 @@ public void BadDeserialization_NameDoesNotMatchSegments() [Fact] public void BadDeserialization_IsEmptyDoesNotMatchSegments() { - var input = Create(); + var input = Create(); input.IsEmpty = !input.IsEmpty; Assert.Throws(() => input.AsContentLocation()); diff --git a/tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonContentReference.cs b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentReference.cs similarity index 77% rename from tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonContentReference.cs rename to tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentReference.cs index 6263c46..1ebb214 100644 --- a/tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonContentReference.cs +++ b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableContentReference.cs @@ -15,16 +15,19 @@ // limitations under the License. // -using Tableau.Migration.TestComponents.JsonConverters.JsonObjects; -namespace Tableau.Migration.TestComponents.Tests.JsonConverter.JsonObjects +using System; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.JsonConverter.SerializableObjects { - public class TestJsonContentReference : AutoFixtureTestBase + public class TestSerializableContentReference : AutoFixtureTestBase { [Fact] public void AsContentReferenceStub() { - var input = Create(); + var input = Create(); var output = input.AsContentReferenceStub(); @@ -39,7 +42,7 @@ public void AsContentReferenceStub() [Fact] public void BadDeserialization_NullId() { - var input = Create(); + var input = Create(); input.Id = null; Assert.Throws(() => input.AsContentReferenceStub()); @@ -48,7 +51,7 @@ public void BadDeserialization_NullId() [Fact] public void BadDeserialization_NullContentUrl() { - var input = Create(); + var input = Create(); input.ContentUrl = null; Assert.Throws(() => input.AsContentReferenceStub()); @@ -57,7 +60,7 @@ public void BadDeserialization_NullContentUrl() [Fact] public void BadDeserialization_NullLocation() { - var input = Create(); + var input = Create(); input.Location = null; Assert.Throws(() => input.AsContentReferenceStub()); @@ -66,7 +69,7 @@ public void BadDeserialization_NullLocation() [Fact] public void BadDeserialization_NullName() { - var input = Create(); + var input = Create(); input.Name = null; Assert.Throws(() => input.AsContentReferenceStub()); diff --git a/tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonManifestEntry.cs b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableManifestEntry.cs similarity index 74% rename from tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonManifestEntry.cs rename to tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableManifestEntry.cs index f03b4a2..83600d8 100644 --- a/tests/Tableau.Migration.TestComponents.Tests/JsonConverter/JsonObjects/TestJsonManifestEntry.cs +++ b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializableObjects/TestSerializableManifestEntry.cs @@ -15,20 +15,23 @@ // limitations under the License. // +using System; +using System.Linq; using Moq; using Tableau.Migration.Engine.Manifest; -using Tableau.Migration.TestComponents.JsonConverters.JsonObjects; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Xunit; -namespace Tableau.Migration.TestComponents.Tests.JsonConverter.JsonObjects +namespace Tableau.Migration.Tests.Unit.JsonConverter.SerializableObjects { - public class TestJsonManifestEntry : AutoFixtureTestBase + public class TestSerializableManifestEntry : AutoFixtureTestBase { - Mock mockPartition = new(); + private readonly Mock mockPartition = new(); [Fact] public void AsMigrationManifestEntryWithDestination() { - var input = Create(); + var input = Create(); var output = input.AsMigrationManifestEntry(mockPartition.Object); @@ -38,15 +41,14 @@ public void AsMigrationManifestEntryWithDestination() Assert.Equal(input.MappedLocation!.AsContentLocation(), output.MappedLocation); Assert.Equal((int)input.Status, (int)output.Status); Assert.Equal(input.HasMigrated, output.HasMigrated); - - //Destination location is reset on copy. - Assert.Null(output.Destination); + Assert.Equal(input.Destination?.AsContentReferenceStub(), output.Destination); + Assert.Equal(input?.Errors?.Select(e => e.Error), output.Errors); } [Fact] public void AsMigrationManifestEntryWithoutDestination() { - var input = Create(); + var input = Create(); input.Destination = null; input.MappedLocation = input.Source!.Location; @@ -59,12 +61,13 @@ public void AsMigrationManifestEntryWithoutDestination() Assert.Equal(input.MappedLocation!.AsContentLocation(), output.MappedLocation); Assert.Equal((int)input.Status, (int)output.Status); Assert.Equal(input.HasMigrated, output.HasMigrated); + Assert.Equal(input?.Errors?.Select(e => e.Error), output.Errors); } [Fact] public void BadDeserialization_NullSource() { - var input = Create(); + var input = Create(); input.Source = null; Assert.Throws(() => input.AsMigrationManifestEntry(mockPartition.Object)); @@ -73,7 +76,7 @@ public void BadDeserialization_NullSource() [Fact] public void BadDeserialization_NullMappedLocation() { - var input = Create(); + var input = Create(); input.MappedLocation = null; Assert.Throws(() => input.AsMigrationManifestEntry(mockPartition.Object)); diff --git a/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializedExceptionJsonConverterTests.cs b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializedExceptionJsonConverterTests.cs new file mode 100644 index 0000000..e37701b --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/JsonConverter/SerializedExceptionJsonConverterTests.cs @@ -0,0 +1,122 @@ +// +// Copyright (c) 2024, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the "License") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Text.Json; +using Tableau.Migration.Engine.Manifest; +using Tableau.Migration.JsonConverters; +using Tableau.Migration.JsonConverters.SerializableObjects; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.JsonConverter +{ + public class SerializedExceptionJsonConverterTests : AutoFixtureTestBase + { + private readonly JsonSerializerOptions _serializerOptions; + + public SerializedExceptionJsonConverterTests() + { + _serializerOptions = new JsonSerializerOptions() + { + WriteIndented = true, + }; + + foreach (var converter in MigrationManifestSerializer.CreateConverters()) + { + _serializerOptions.Converters.Add(converter); + } + } + + [Theory] + [ClassData(typeof(SerializableExceptionTypeData))] + public void WriteAndReadBack_ExceptionObject_SerializesAndDeserializesToJson(SerializableException ex) + { + // Arrange + var converter = new SerializedExceptionJsonConverter(); + + Assert.NotNull(ex.Error); + var exceptionNamespace = ex.Error.GetType().Namespace; + Assert.NotNull(exceptionNamespace); + + if (!exceptionNamespace.StartsWith("System")) // Built in Exception is not equatable + { + // We require all custom exception to be equatable so we can test serializability. + Assert.True(ex.Error.ImplementsEquatable()); + } + + // Serialize + using (var memoryStream = new MemoryStream()) + { + var writer = new Utf8JsonWriter(memoryStream); + converter.Write(writer, ex, _serializerOptions); + writer.Flush(); + var json = Encoding.UTF8.GetString(memoryStream.ToArray()); + + var expectedJsonMessage = ex.Error?.Message.Replace("'", "\\u0027") ?? ""; + + // Assert Writer + Assert.NotNull(json); + Assert.NotEmpty(json); + Assert.Contains(expectedJsonMessage, json); + + // Deserialize + var reader = new Utf8JsonReader(Encoding.UTF8.GetBytes(json)); + var result = converter.Read(ref reader, typeof(Exception), _serializerOptions); + + // Assert + Assert.NotNull(result); + Assert.NotNull(result.Error); + Assert.Equal(ex.Error!.Message, result.Error.Message); + + if (!exceptionNamespace.StartsWith("System")) // Built in Exception is not equatable + { + Assert.Equal(ex.Error, result.Error); + } + } + } + + + /// + /// Provides a collection of serializable exception objects. + /// + public class SerializableExceptionTypeData : AutoFixtureTestBase, IEnumerable + { + /// + /// Returns an enumerator that iterates through the collection. + /// + /// An enumerator that can be used to iterate through the collection. + public IEnumerator GetEnumerator() + { + foreach (var e in FixtureFactory.CreateErrors(AutoFixture)) + { + var ex = new SerializableException(e); + yield return new object[] { ex }; + } + } + + /// + /// Returns an enumerator that iterates through the collection. + /// + /// An enumerator that can be used to iterate through the collection. + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/LoggingServiceCollectionExtensionsTests.cs b/tests/Tableau.Migration.Tests/Unit/LoggingServiceCollectionExtensionsTests.cs index e5f9c1f..d660253 100644 --- a/tests/Tableau.Migration.Tests/Unit/LoggingServiceCollectionExtensionsTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/LoggingServiceCollectionExtensionsTests.cs @@ -37,7 +37,7 @@ public void ServiceCollectionWithoutLoggerFactoryAndProvider() // Assert Assert.NotEmpty(serviceCollection.Where(descriptor => descriptor.ServiceType == typeof(ILoggerFactory))); - Assert.Empty(serviceCollection.Where(descriptor => descriptor.ServiceType == typeof(ILoggerProvider))); + Assert.Empty(serviceCollection.Where(descriptor => descriptor.ServiceType == typeof(ILoggerProvider)).ToList()); } ///