diff --git a/.editorconfig b/.editorconfig index a758cc6..c68c8b4 100644 --- a/.editorconfig +++ b/.editorconfig @@ -103,4 +103,5 @@ dotnet_diagnostic.SA1208.severity = warning csharp_style_unused_value_assignment_preference = discard_variable:suggestion dotnet_diagnostic.IDE0051.severity = error dotnet_diagnostic.IDE0060.severity = error +dotnet_diagnostic.IDE0073.severity = error csharp_style_prefer_primary_constructors = true:suggestion \ No newline at end of file diff --git a/.github/workflows/sdk-workflow.yml b/.github/workflows/sdk-workflow.yml index 57e5828..982dd5a 100644 --- a/.github/workflows/sdk-workflow.yml +++ b/.github/workflows/sdk-workflow.yml @@ -13,7 +13,6 @@ on: - .gitignore - '**/.gitignore' - 'CODEOWNERS' - - '.git2gus/config.json' pull_request: branches: - main @@ -30,7 +29,6 @@ on: - .gitignore - '**/.gitignore' - 'CODEOWNERS' - - '.git2gus/config.json' workflow_dispatch: inputs: publish-release: @@ -129,15 +127,15 @@ jobs: publish-docs-dry-run: needs: [ dotnet-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/publishdocs-dryrun.yml - if: ${{ (inputs.publish-release != 'Prod' && inputs.publish-release != 'Prod-Internal' && inputs.publish-docs == false) || (github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/')) }} + if: ${{ (inputs.publish-release != 'Prod' && inputs.publish-release != 'Prod-Internal' && inputs.publish-docs == false) || (github.ref != 'refs/heads/main' && !startsWith(github.ref, 'refs/heads/release/') && !startsWith(github.ref, 'refs/tags/release/')) }} with: runs-on-config: ${{ vars.PUBLISH_OS }} build-config: ${{ vars.PUBLISH_CONFIGURATION }} python-version: ${{ vars.PYTHON_PUBLISH_DOCS_VERSION }} publish-docs: - needs: [ dotnet-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/publishdocs.yml - if: ${{ (inputs.publish-release == 'Prod' || inputs.publish-release == 'Prod-Internal' || inputs.publish-docs == true) && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/')) }} + if: ${{ (inputs.publish-release == 'Prod' || inputs.publish-release == 'Prod-Internal' || inputs.publish-docs == true) && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/tags/release/')) }} with: runs-on-config: ${{ vars.PUBLISH_OS }} build-config: ${{ vars.PUBLISH_CONFIGURATION }} @@ -159,7 +157,7 @@ jobs: release-version: ${{ needs.define-version.outputs.beta-version != '' && needs.define-version.outputs.beta-version || needs.define-version.outputs.code-version }} is-pre-release: ${{ inputs.publish-release != 'Prod' && inputs.publish-release != 'Prod-Internal' }} dotnet-publish-package-internal-beta: - needs: [ dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/dotnet-package.yml secrets: inherit if: ${{ inputs.publish-release == 'Beta' || inputs.publish-release == 'Beta-Internal' }} @@ -169,7 +167,7 @@ jobs: build-config: ${{ vars.PUBLISH_CONFIGURATION }} publish-environment: internal-beta dotnet-publish-package-public-beta: - needs: [ dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/dotnet-package.yml secrets: inherit if: ${{ inputs.publish-release == 'Beta' }} @@ -179,27 +177,27 @@ jobs: build-config: ${{ vars.PUBLISH_CONFIGURATION }} publish-environment: public-beta dotnet-publish-package-internal-prod: - needs: [ dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/dotnet-package.yml secrets: inherit - if: ${{ (inputs.publish-release == 'Prod' || inputs.publish-release == 'Prod-Internal') && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/')) }} + if: ${{ (inputs.publish-release == 'Prod' || inputs.publish-release == 'Prod-Internal') && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/tags/release/')) }} with: published-os: ${{ vars.PUBLISH_OS }} runs-on-config: ${{ vars.PUBLISH_PACKAGE_OS }} build-config: ${{ vars.PUBLISH_CONFIGURATION }} publish-environment: internal-prod dotnet-publish-package-public-prod: - needs: [ dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/dotnet-package.yml secrets: inherit - if: ${{ inputs.publish-release == 'Prod' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/')) }} + if: ${{ inputs.publish-release == 'Prod' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/tags/release/')) }} with: published-os: ${{ vars.PUBLISH_OS }} runs-on-config: ${{ vars.PUBLISH_OS }} build-config: ${{ vars.PUBLISH_CONFIGURATION }} publish-environment: public-prod python-publish-package-internal-beta: - needs: [ define-version, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ define-version, dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/python-package.yml secrets: inherit if: ${{ inputs.publish-release == 'Beta' || inputs.publish-release == 'Beta-Internal' }} @@ -209,7 +207,7 @@ jobs: publish-environment: internal-beta beta-version: ${{ needs.define-version.outputs.beta-version }} python-publish-package-public-beta: - needs: [ define-version, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ define-version, dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/python-package.yml secrets: inherit if: ${{ inputs.publish-release == 'Beta' }} @@ -219,20 +217,20 @@ jobs: publish-environment: public-beta beta-version: ${{ needs.define-version.outputs.beta-version }} python-publish-package-internal-prod: - needs: [ define-version, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ define-version, dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/python-package.yml secrets: inherit - if: ${{ (inputs.publish-release == 'Prod' || inputs.publish-release == 'Prod-Internal') && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/')) }} + if: ${{ (inputs.publish-release == 'Prod' || inputs.publish-release == 'Prod-Internal') && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/tags/release/')) }} with: runs-on-config: ${{ vars.PUBLISH_PACKAGE_OS }} build-config: ${{ vars.PUBLISH_CONFIGURATION }} publish-environment: internal-prod beta-version: ${{ needs.define-version.outputs.beta-version }} python-publish-package-public-prod: - needs: [ define-version, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] + needs: [ define-version, dotnet-test, python-test, dotnet-publish-package-internal-dryrun, dotnet-publish-package-public-dryrun, python-publish-package-internal-dryrun, python-publish-package-public-dryrun ] uses: ./.github/workflows/python-package.yml secrets: inherit - if: ${{ inputs.publish-release == 'Prod' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/')) }} + if: ${{ inputs.publish-release == 'Prod' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/heads/release/') || startsWith(github.ref, 'refs/tags/release/')) }} with: runs-on-config: ${{ vars.PUBLISH_OS }} build-config: ${{ vars.PUBLISH_CONFIGURATION }} diff --git a/Directory.Build.props b/Directory.Build.props index 2a2a93a..c2203b1 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -4,7 +4,7 @@ enable true true - 1.0.0 + 1.1.0 Tableau Software, LLC Tableau Software, LLC Copyright (c) 2024, Tableau Software, LLC and its licensors diff --git a/Migration SDK.sln b/Migration SDK.sln index 85705b5..19e5b58 100644 --- a/Migration SDK.sln +++ b/Migration SDK.sln @@ -55,6 +55,33 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{9204 EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DependencyInjection.ExampleApplication", "examples\DependencyInjection.ExampleApplication\DependencyInjection.ExampleApplication.csproj", "{99DA12FB-BB16-4EE1-9C9C-047755210255}" EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".github", ".github", "{90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4}" + ProjectSection(SolutionItems) = preProject + .github\pull_request_template.md = .github\pull_request_template.md + .github\release.yaml = .github\release.yaml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "actions", "actions", "{6B735E6E-1FFB-4C37-8CF6-BD979B4F8D9B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "workflows", "workflows", "{EDD52CC0-7289-4167-A01E-E88B015FC67F}" + ProjectSection(SolutionItems) = preProject + .github\workflows\create-release.yml = .github\workflows\create-release.yml + .github\workflows\dotnet-build.yml = .github\workflows\dotnet-build.yml + .github\workflows\dotnet-package.yml = .github\workflows\dotnet-package.yml + .github\workflows\dotnet-test.yml = .github\workflows\dotnet-test.yml + .github\workflows\publishdocs-dryrun.yml = .github\workflows\publishdocs-dryrun.yml + .github\workflows\publishdocs.yml = .github\workflows\publishdocs.yml + .github\workflows\python-package.yml = .github\workflows\python-package.yml + .github\workflows\python-test.yml = .github\workflows\python-test.yml + .github\workflows\README.md = .github\workflows\README.md + .github\workflows\sdk-workflow.yml = .github\workflows\sdk-workflow.yml + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "setup-dotnet", "setup-dotnet", "{454EF272-D967-4668-A20D-AD6B3EE96C1A}" + ProjectSection(SolutionItems) = preProject + .github\actions\setup-dotnet\action.yml = .github\actions\setup-dotnet\action.yml + EndProjectSection +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -112,6 +139,10 @@ Global {B0D16449-BC58-4873-8B95-F62E805EE39C} = {00B78B56-BF89-4F26-AF75-435A4B88D43F} {92044843-B4D7-4062-B1D5-DE7596072E33} = {C5BD8316-60C9-4C96-9B15-820E8BA4DF7F} {99DA12FB-BB16-4EE1-9C9C-047755210255} = {00B78B56-BF89-4F26-AF75-435A4B88D43F} + {90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4} = {C5BD8316-60C9-4C96-9B15-820E8BA4DF7F} + {6B735E6E-1FFB-4C37-8CF6-BD979B4F8D9B} = {90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4} + {EDD52CC0-7289-4167-A01E-E88B015FC67F} = {90102C4B-EC3B-4279-A6C6-A6CFDFCD4DB4} + {454EF272-D967-4668-A20D-AD6B3EE96C1A} = {6B735E6E-1FFB-4C37-8CF6-BD979B4F8D9B} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {2C9E9FF4-E825-47A4-90BE-5499D5EDF3CC} diff --git a/README.md b/README.md index aa2fce5..ea6f303 100644 --- a/README.md +++ b/README.md @@ -9,28 +9,29 @@ The Tableau Migration SDK is a client library to build an application to facilit ## Get started Quickstart: -- For Python install using PIP `pip install tableau-migration` -- For C# install using NuGet +- Install a [.NET Runtime](https://dotnet.microsoft.com/en-us/download). +- For Python install using PIP: + - [PIP CLI](https://pip.pypa.io/en/stable/cli/pip_install): `pip install tableau-migration` +- For C# install using NuGet: - [dotnet CLI](https://learn.microsoft.com/en-us/nuget/quickstart/install-and-use-a-package-using-the-dotnet-cli): `dotnet add package Tableau.Migration --version 1.0.0` - - [Nuget Package Manager](https://learn.microsoft.com/en-us/nuget/quickstart/install-and-use-a-package-in-visual-studio): Search for Tableau.Migration -Then check out our [code samples](https://tableau.github.io/migration-sdk/samples/intro.html) + - [Nuget Package Manager](https://learn.microsoft.com/en-us/nuget/quickstart/install-and-use-a-package-in-visual-studio): Search for `Tableau.Migration`. +- Then check out our [code samples](https://tableau.github.io/migration-sdk/samples/intro.html). -To look at source code and delve deeper: - First, clone the repo: +To look at source code and delve deeper, first clone the repo: - 1. Open a command line interface - 2. `cd` to the parent directory where the repo directory will live - 3. `git clone https://github.com/tableau/tableau-migration-sdk.git` +1. Open a command line interface. +1. `cd` to the parent directory where the repo directory will live. +1. `git clone https://github.com/tableau/tableau-migration-sdk.git` - After cloning the repo: +After cloning the repo: - - Open `Migration SDK.sln` using Visual Studio or Visual Studio Code +- Open `Migration SDK.sln` using Visual Studio or Visual Studio Code. ## Introduction [Migration SDK Overview](https://help.tableau.com/current/api/migration_sdk/en-us/index.html) - Understanding the Migration SDK -- Preparing your migration +- Preparing Your Migration - Migrating to Tableau Cloud [Migration SDK API References](https://tableau.github.io/migration-sdk/) @@ -39,4 +40,4 @@ To look at source code and delve deeper: [Contributing Guide for Migration SDK Developers](https://github.com/tableau/tableau-migration-sdk/blob/main/CONTRIBUTING.md) -[Migration SDK Security Considerations](https://github.com/tableau/tableau-migration-sdk/blob/main/SECURITY.md) +[Migration SDK Security Considerations](https://github.com/tableau/tableau-migration-sdk/blob/main/SECURITY.md) \ No newline at end of file diff --git a/examples/Csharp.ExampleApplication/Csharp.ExampleApplication.csproj b/examples/Csharp.ExampleApplication/Csharp.ExampleApplication.csproj index 46f44a0..13715dd 100644 --- a/examples/Csharp.ExampleApplication/Csharp.ExampleApplication.csproj +++ b/examples/Csharp.ExampleApplication/Csharp.ExampleApplication.csproj @@ -2,8 +2,8 @@ Exe net6.0;net7.0;net8.0 - - CA2007 + + CA2007,IDE0073 7d7631f1-dc4a-49de-89d5-a194544705c1 diff --git a/examples/DependencyInjection.ExampleApplication/DependencyInjection.ExampleApplication.csproj b/examples/DependencyInjection.ExampleApplication/DependencyInjection.ExampleApplication.csproj index a80c905..1ecd9a5 100644 --- a/examples/DependencyInjection.ExampleApplication/DependencyInjection.ExampleApplication.csproj +++ b/examples/DependencyInjection.ExampleApplication/DependencyInjection.ExampleApplication.csproj @@ -3,8 +3,8 @@ Exe net6.0;net7.0;net8.0 - - CA2007 + + CA2007,IDE0073 diff --git a/src/Documentation/articles/configuration.md b/src/Documentation/articles/configuration.md index 1169921..8e27541 100644 --- a/src/Documentation/articles/configuration.md +++ b/src/Documentation/articles/configuration.md @@ -54,12 +54,17 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines [`MigrationSdkOptions`](xref:Tableau.Migration.Config.MigrationSdkOptions) is the configuration class the Migration SDK uses internally to process a migration. It contains adjustable properties that change some engine behaviors. These properties are useful tools to troubleshoot and tune a migration process. Start with this class and others in the [Config](xref:Tableau.Migration.Config) section for more details. +When writting a C# application, it is recommended that a [.NET Generic Host](https://learn.microsoft.com/en-us/dotnet/core/extensions/generic-host?tabs=appbuilder) is used to initialize the application. This will enable setting configuration values via `appsettings.json` which can be passed into `userOptions` in [`.AddTableauMigrationSdk`](xref:Tableau.Migration.IServiceCollectionExtensions.html#Tableau_Migration_IServiceCollectionExtensions_AddTableauMigrationSdk_Microsoft_Extensions_DependencyInjection_IServiceCollection_Microsoft_Extensions_Configuration_IConfiguration_). See [.NET getting started examples](..\samples\csharp.md) for more info. + +When writting a python application, configuration values are set via environment variables. The `:` delimiter doesn't work with environment variable hierarchical keys on all platforms. For example, the `:` delimiter is not supported by Bash. The double underscore (`__`), which is supported on all platforms, automatically replaces any `:` delimiters in environment variables. All configuration environment variables start with `MigrationSDK__`. ### BatchSize *Reference:* [`MigrationSdkOptions.BatchSize`](xref:Tableau.Migration.Config.MigrationSdkOptions#Tableau_Migration_Config_MigrationSdkOptions_BatchSize). *Default:* [`MigrationSdkOptions.Defaults.BATCH_SIZE`](xref:Tableau.Migration.Config.MigrationSdkOptions.Defaults#Tableau_Migration_Config_MigrationSdkOptions_Defaults_BATCH_SIZE). +*Python Environment Variable:* `MigrationSDK__BatchSize` + *Reload on Edit?:* **Yes**. The update will apply next time the Migration SDK requests a list of objects. *Description:* The Migration SDK uses the **BatchSize** property to define the page size of each List Request. For more details, check the [Tableau REST API Paginating Results documentation](https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_concepts_paging.htm). @@ -70,6 +75,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`MigrationSdkOptions.Defaults.MIGRATION_PARALLELISM`](xref:Tableau.Migration.Config.MigrationSdkOptions.Defaults#Tableau_Migration_Config_MigrationSdkOptions_Defaults_MIGRATION_PARALLELISM). +*Python Environment Variable:* `MigrationSDK__MigrationParallelism` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK publishes a new batch. *Description:* The Migration SDK uses [two methods](advanced_config/hooks/index.md#hook-execution-flow) to publish the content to a destination server: the **bulk process**, where a single call to the API will push multiple items to the server, and the **individual process**, where it publishes a single item with a single call to the API. This configuration only applies to the **individual process**. The SDK uses the **MigrationParallelism** property to define the number of parallel tasks migrating the same type of content simultaneosly. It is possible to tune the Migration SDK processing time with this configuration. @@ -82,6 +89,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`FileOptions.Defaults.DISABLE_FILE_ENCRYPTION`](xref:Tableau.Migration.Config.FileOptions.Defaults#Tableau_Migration_Config_FileOptions_Defaults_DISABLE_FILE_ENCRYPTION). +*Python Environment Variable:* `MigrationSDK__Files__DisableFileEncryption` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK executes a migration plan. *Description:* As part of the migration process, the Migration SDK has to adjust existing references for file-based content types like Workbooks and Data Sources. The SDK has to download and temporarily store the content in the migration machine to be able to read and edit these files. The Migration SDK uses the **DisableFileEncryption** property to define whether it will encrypt the temporary file. @@ -94,6 +103,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`FileOptions.Defaults.ROOT_PATH`](xref:Tableau.Migration.Config.FileOptions.Defaults#Tableau_Migration_Config_FileOptions_Defaults_ROOT_PATH). +*Python Environment Variable:* `MigrationSDK__Files__RootPath` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK executes a migration plan. *Description:* As part of the migration process, the Migration SDK has to adjust existing references for file-based content types like Workbooks and Data Sources. The SDK has to download and temporarily store the content in the migration machine to be able to read and edit these files. The Migration SDK uses the **RootPath** property to define the location where it will store the temporary files. @@ -104,6 +115,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`NetworkOptions.Defaults.FILE_CHUNK_SIZE_KB`](xref:Tableau.Migration.Config.NetworkOptions.Defaults#Tableau_Migration_Config_NetworkOptions_Defaults_FILE_CHUNK_SIZE_KB). +*Python Environment Variable:* `MigrationSDK__Network__FileChunkSizeKB` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK publishes a new file. *Description:* As part of the migration process, the Migration SDK has to publish file-based content types like Workbooks and Data Sources. Some of these files are very large. The Migration SDK uses the **FileChunkSizeKB** property to split these files into smaller pieces, making the publishing process more reliable. For more details, check the [Tableau REST API Publishing Resources documentation](https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_concepts_publish.htm). @@ -114,6 +127,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`NetworkOptions.Defaults.LOG_HEADERS_ENABLED`](xref:Tableau.Migration.Config.NetworkOptions.Defaults#Tableau_Migration_Config_NetworkOptions_Defaults_LOG_HEADERS_ENABLED). +*Python Environment Variable:* `MigrationSDK__Network__HeadersLoggingEnabled` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK logs a new HTTP request. *Description:* Check the [logging article](logging.md) for more details. @@ -124,6 +139,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`NetworkOptions.Defaults.LOG_CONTENT_ENABLED`](xref:Tableau.Migration.Config.NetworkOptions.Defaults#Tableau_Migration_Config_NetworkOptions_Defaults_LOG_CONTENT_ENABLED). +*Python Environment Variable:* `MigrationSDK__Network__ContentLoggingEnabled` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK logs a new HTTP request. *Description:* Check the [logging article](logging.md) for more details. @@ -134,6 +151,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`NetworkOptions.Defaults.LOG_BINARY_CONTENT_ENABLED`](xref:Tableau.Migration.Config.NetworkOptions.Defaults#Tableau_Migration_Config_NetworkOptions_Defaults_LOG_BINARY_CONTENT_ENABLED). +*Python Environment Variable:* `MigrationSDK__Network__BinaryContentLoggingEnabled` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK logs a new HTTP request. *Description:* Check the [logging article](logging.md) for more details. @@ -144,6 +163,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`NetworkOptions.Defaults.LOG_EXCEPTIONS_ENABLED`](xref:Tableau.Migration.Config.NetworkOptions.Defaults#Tableau_Migration_Config_NetworkOptions_Defaults_LOG_EXCEPTIONS_ENABLED). +*Python Environment Variable:* `MigrationSDK__Network__ExceptionsLoggingEnabled` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK logs a new HTTP request. *Description:* Check the [logging article](logging.md) for more details. @@ -154,6 +175,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.RETRY_ENABLED`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_RETRY_ENABLED). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__RetryEnabled` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **RetryEnabled** property to define whether it will retry failed requests. @@ -164,6 +187,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.RETRY_INTERVALS`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_RETRY_INTERVALS). +*Python Environment Variable:* **Not Supported** + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **RetryIntervals** property to define the number of retries and the interval between each retry. @@ -174,6 +199,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.RETRY_OVERRIDE_RESPONSE_CODES`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_RETRY_OVERRIDE_RESPONSE_CODES). +*Python Environment Variable:* **Not Supported** + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **RetryOverrideResponseCodes** property to override the default list of error status codes for retries with a specific list of status codes. @@ -184,6 +211,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.CONCURRENT_REQUESTS_LIMIT_ENABLED`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_CONCURRENT_REQUESTS_LIMIT_ENABLED). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__ConcurrentRequestsLimitEnabled` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **ConcurrentRequestsLimitEnabled** property to define whether it will limit concurrent requests. @@ -194,6 +223,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.MAX_CONCURRENT_REQUESTS`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_MAX_CONCURRENT_REQUESTS). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__MaxConcurrentRequests` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **MaxConcurrentRequests** property to define the maximum quantity of concurrent API requests. @@ -204,6 +235,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.CONCURRENT_WAITING_REQUESTS_QUEUE`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_CONCURRENT_WAITING_REQUESTS_QUEUE). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__ConcurrentWaitingRequestsOnQueue` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **ConcurrentWaitingRequestsOnQueue** property to define the quantity of concurrent API requests waiting on queue. @@ -214,6 +247,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.CLIENT_THROTTLE_ENABLED`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_CLIENT_THROTTLE_ENABLED). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__ClientThrottleEnabled` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **ClientThrottleEnabled** property to define whether it will limit requests to a given endpoint on the client side. @@ -224,6 +259,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.MAX_READ_REQUESTS`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_MAX_READ_REQUESTS). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__MaxReadRequests` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **MaxReadRequests** property to define the maximum quantity of GET requests on the client side. @@ -234,6 +271,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.MAX_READ_REQUESTS_INTERVAL`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_MAX_READ_REQUESTS_INTERVAL). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__MaxReadRequestsInterval` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **MaxReadRequestsInterval** property to define the interval for the limit of GET requests on the client side. @@ -244,6 +283,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.MAX_BURST_READ_REQUESTS`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_MAX_BURST_READ_REQUESTS). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__MaxBurstReadRequests` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. With only the previous configuration values (`Network.Resilience.MaxReadRequests` and `Network.Resilience.MaxReadRequestsInterval`), the SDK will calculate the minimum interval to complete a single request. Any other request at the same period will be blocked. The SDK uses the **MaxBurstReadRequests** property to define the maximum quantity of GET requests on the calculated period. @@ -254,6 +295,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.MAX_PUBLISH_REQUESTS`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_MAX_PUBLISH_REQUESTS). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__MaxPublishRequests` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **MaxPublishRequests** property to define the maximum quantity of non-GET requests on the client side. @@ -264,6 +307,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.MAX_PUBLISH_REQUESTS_INTERVAL`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_MAX_PUBLISH_REQUESTS_INTERVAL). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__MaxPublishRequestsInterval` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **MaxPublishRequestsInterval** property to define the interval for the limit of non-GET requests on the client side. @@ -274,6 +319,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.MAX_BURST_PUBLISH_REQUESTS`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_MAX_BURST_PUBLISH_REQUESTS). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__MaxBurstPublishRequests` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. With only the previous configuration values (`Network.Resilience.MaxPublishRequests` and `Network.Resilience.MaxPublishRequestsInterval`), the SDK will calculate the minimum interval to complete a single request. Any other request at the same period will be blocked. The SDK uses the **MaxBurstPublishRequests** property to define the maximum quantity of non-GET requests on the calculated period. @@ -284,6 +331,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.SERVER_THROTTLE_ENABLED`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_SERVER_THROTTLE_ENABLED). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__ServerThrottleEnabled` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **ServerThrottleEnabled** property to define whether it will retry requests throttled on the server. @@ -294,6 +343,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.SERVER_THROTTLE_LIMIT_RETRIES`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_SERVER_THROTTLE_LIMIT_RETRIES). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__ServerThrottleLimitRetries` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **ServerThrottleLimitRetries** property to define whether it will have a limit of retries to a throttled request. @@ -304,6 +355,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.SERVER_THROTTLE_RETRY_INTERVALS`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_SERVER_THROTTLE_RETRY_INTERVALS). +*Python Environment Variable:* **Not Supported** + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **ServerThrottleRetryIntervals** property to define the interval between each retry for throttled requests without the 'Retry-After' header. If `ServerThrottleLimitRetries` is enabled, this configuration defines the maximum number of retries. Otherwise, the subsequent retries use the last interval value. @@ -314,6 +367,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.REQUEST_TIMEOUT`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_REQUEST_TIMEOUT). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__PerRequestTimeout` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **PerRequestTimeout** property to define the maximum duration of non-FileTransfer requests. @@ -324,6 +379,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`ResilienceOptions.Defaults.FILE_TRANSFER_REQUEST_TIMEOUT`](xref:Tableau.Migration.Config.ResilienceOptions.Defaults#Tableau_Migration_Config_ResilienceOptions_Defaults_FILE_TRANSFER_REQUEST_TIMEOUT). +*Python Environment Variable:* `MigrationSDK__Network__Resilience__PerFileTransferRequestTimeout` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK makes a new HTTP request. *Description:* The Migration SDK uses [Polly](https://github.com/App-vNext/Polly) as a resilience and transient-fault layer. The SDK uses the **PerFileTransferRequestTimeout** property to define the maximum duration of FileTransfer requests. @@ -334,6 +391,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`DefaultPermissionsContentTypeUrlSegments`](xref:Tableau.Migration.Content.Permissions.DefaultPermissionsContentTypeUrlSegments). +*Python Environment Variable:* **Not Supported** + *Reload on Edit?:* **No**. Any changes to this configuration will reflect on the next time the application starts. *Description:* The SDK uses the **UrlSegments** property as a list of types of default permissions of given project. For more details, check the [Query Default Permissions documentation](https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_permissions.htm#query_default_permissions). @@ -344,6 +403,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`JobOptions.Defaults.JOB_POLL_RATE`](xref:Tableau.Migration.Config.JobOptions.Defaults#Tableau_Migration_Config_JobOptions_Defaults_JOB_POLL_RATE). +*Python Environment Variable:* `MigrationSDK__Jobs__JobPollRate` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK delays the processing status recheck. *Description:* The Migration SDK uses [two methods](advanced_config/hooks/index.md#hook-execution-flow) to publish the content to a destination server: the **bulk process**, where a single call to the API will push multiple items to the server, and the **individual process**, where it publishes a single item with a single call to the API. This configuration only applies to the **bulk process**. After publishing a batch, the API will return a Job ID. With it, the SDK can call another API to check the job processing status. The SDK uses the **JobPollRate** property to define the interval it will wait to recheck processing status. For more details, check the [Tableau REST API Query Job documentation](https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_jobs_tasks_and_schedules.htm#query_job). @@ -356,6 +417,8 @@ The [`IMigrationPlan`](xref:Tableau.Migration.IMigrationPlan) interface defines *Default:* [`JobOptions.Defaults.JOB_TIMEOUT`](xref:Tableau.Migration.Config.JobOptions.Defaults#Tableau_Migration_Config_JobOptions_Defaults_JOB_TIMEOUT). +*Python Environment Variable:* `MigrationSDK__Jobs__JobTimeout` + *Reload on Edit?:* **Yes**. The update will apply the next time the Migration SDK validates the total time it has waited for the job to complete. *Description:* The Migration SDK uses [two methods](advanced_config/hooks/index.md#hook-execution-flow) to publish the content to a destination server: the **bulk process**, where a single call to the API will push multiple items to the server, and the **individual process**, where it publishes a single item with a single call to the API. This configuration only applies to the **bulk process**. The SDK uses the **JobTimeout** property to define the maximum interval it will wait for a job to complete. For more details, check the [Tableau REST API Query Job documentation](https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_jobs_tasks_and_schedules.htm#query_job). diff --git a/src/Documentation/articles/python_wrapper.md b/src/Documentation/articles/python_wrapper.md index 53f8dab..316270a 100644 --- a/src/Documentation/articles/python_wrapper.md +++ b/src/Documentation/articles/python_wrapper.md @@ -4,16 +4,16 @@ The Migration SDK is written in .NET. It has a Python wrapper package that provi ## Capabilities -With the Python Wrapper, you can +With the Python Wrapper, you can: - Provide basic [configuration](configuration.md) values to the Migration SDK via the PlanBuilder. +- Set configuration options as described in [`MigrationSdkOptions`](xref:Tableau.Migration.Config.MigrationSdkOptions) with environment variables. - Configure Python [logging](logging.md#python-support). - Run a migration using the wrapper. - Write [Python hooks](advanced_config/hooks/python_hooks.md) (See [Hooks](advanced_config/hooks/index.md) for an overview). ## Current limitations -There are advanced features of the Migration SDK that the Python Wrapper cannot currently access +There are advanced features of the Migration SDK that the Python Wrapper cannot currently access: -- Configuration options as described in [`MigrationSdkOptions`](xref:Tableau.Migration.Config.MigrationSdkOptions). - Override `C#` classes and methods to change how the SDK works. diff --git a/src/Documentation/articles/troubleshooting/errors.md b/src/Documentation/articles/troubleshooting/errors.md index a620619..6f573cb 100644 --- a/src/Documentation/articles/troubleshooting/errors.md +++ b/src/Documentation/articles/troubleshooting/errors.md @@ -2,7 +2,7 @@ This section provides a list of potential error and warning log messages that you may encounter in the logs. Each entry includes a description to assist you in debugging. -### Warning: `Group [group name] cannot map [user name]` +### Warning: `Could not add a user to the destination Group [group name]. Reason: Could not find the destination user for [user name].` This warning message indicates that the `GroupUsersTransformer` was unable to add the user, denoted as `[user name]`, to the group, denoted as `[group name]`. diff --git a/src/Python/pyproject.toml b/src/Python/pyproject.toml index 39ade41..780f1bc 100644 --- a/src/Python/pyproject.toml +++ b/src/Python/pyproject.toml @@ -54,8 +54,9 @@ lint = "ruff ." [tool.hatch.envs.test] dev-mode = false dependencies = [ - "pytest==7.4.4", - "pytest-cov==4.1.0" + "pytest>=7.4.4", + "pytest-cov>=4.1.0", + "pytest-env>=1.1.3" ] [tool.hatch.envs.test.scripts] diff --git a/src/Python/pytest.ini b/src/Python/pytest.ini index cd52859..605ca33 100644 --- a/src/Python/pytest.ini +++ b/src/Python/pytest.ini @@ -3,4 +3,7 @@ testpaths = tests pythonpath = - src \ No newline at end of file + src + +env = + MigrationSDK__BatchSize = 102 \ No newline at end of file diff --git a/src/Python/tests/test_migrations_engine_hooks_transformers.py b/src/Python/tests/test_migrations_engine_hooks_transformers.py index 318b590..8d462fe 100644 --- a/src/Python/tests/test_migrations_engine_hooks_transformers.py +++ b/src/Python/tests/test_migrations_engine_hooks_transformers.py @@ -25,7 +25,7 @@ from Tableau.Migration.Engine.Hooks import IMigrationHook from Tableau.Migration.Engine.Hooks.Transformers import ContentTransformerBuilder, IContentTransformer, ContentTransformerBase from Tableau.Migration.Interop.Hooks.Transformers import ISyncContentTransformer, ISyncXmlContentTransformer -from Tableau.Migration.Tests import TestFileContentType +from Tableau.Migration.Tests import TestFileContentType as PyTestFileContentType # Needed as this class name starts with Test, which means pytest wants to pick it up class ClassImplementation(ISyncContentTransformer[IUser]): __namespace__ = "Tableau.Migration.Custom.Hooks.Transformers" @@ -51,16 +51,17 @@ class WithImplementation(ISyncContentTransformer[str]): def Execute(self, ctx: str) -> str: return ctx -class TestXmlTransformer(ISyncXmlContentTransformer[TestFileContentType]): +class TestXmlTransformer(ISyncXmlContentTransformer[PyTestFileContentType]): __namespace__ = "Tableau.Migration.Custom.Hooks.Transformers" - + __test__ = False # Needed as this class name starts with Test, which means pytest wants to pick it up + def __init__(self): self.called = False - def NeedsXmlTransforming(self, ctx: TestFileContentType) -> bool: + def NeedsXmlTransforming(self, ctx: PyTestFileContentType) -> bool: return True - def Execute(self, ctx: TestFileContentType, xml) -> None: + def Execute(self, ctx: PyTestFileContentType, xml) -> None: self.called = True class TestContentTransformerBuilderTests(): @@ -240,13 +241,13 @@ def classcallback(context: IProject): def test_xml_execute(self): - content = TestFileContentType() + content = PyTestFileContentType() provider = ServiceCollectionContainerBuilderExtensions.BuildServiceProvider(ServiceCollection()) pyTransformer = TestXmlTransformer() - hookFactory = PyContentTransformerBuilder(ContentTransformerBuilder()).add(TestFileContentType, pyTransformer).build().get_hooks(IContentTransformer[TestFileContentType]) + hookFactory = PyContentTransformerBuilder(ContentTransformerBuilder()).add(PyTestFileContentType, pyTransformer).build().get_hooks(IContentTransformer[PyTestFileContentType]) assert len(hookFactory) == 1 - hook = hookFactory[0].Create[IMigrationHook[TestFileContentType]](provider) + hook = hookFactory[0].Create[IMigrationHook[PyTestFileContentType]](provider) try: result = hook.ExecuteAsync(content, CancellationToken(False)).GetAwaiter().GetResult() except Exception: diff --git a/src/Python/tests/test_migrations_engine_options.py b/src/Python/tests/test_migrations_engine_options.py index 90ae44d..4edac77 100644 --- a/src/Python/tests/test_migrations_engine_options.py +++ b/src/Python/tests/test_migrations_engine_options.py @@ -29,7 +29,7 @@ import System -from Tableau.Migration.Tests import TestPlanOptions +from Tableau.Migration.Tests import TestPlanOptions as PyTestPlanOptions # Needed as this class name starts with Test, which means pytest wants to pick it up from Tableau.Migration.Engine.Hooks import IMigrationHook @@ -43,13 +43,14 @@ class TestMigrationPlanOptions(): + def test_init_collections(self): migration_plan_options_colllection_mock = Moq.Mock[IMigrationPlanOptionsCollection]() PyMigrationPlanOptionsCollection(migration_plan_options_colllection_mock.Object) def test_get(self): """Verify that the MigrationPlanOptionsBuilder can return the options that were configured""" - input_option = TestPlanOptions() + input_option = PyTestPlanOptions() services = get_service_provider() dotnet_plan_options_builder = get_service(services, IMigrationPlanOptionsBuilder) @@ -57,6 +58,6 @@ def test_get(self): builder = PyMigrationPlanOptionsBuilder(dotnet_plan_options_builder) builder.configure(input_option) - options = builder.build().get(TestPlanOptions) + options = builder.build().get(PyTestPlanOptions) assert input_option == options diff --git a/src/Python/tests/test_other.py b/src/Python/tests/test_other.py index ba1c82c..cb20115 100644 --- a/src/Python/tests/test_other.py +++ b/src/Python/tests/test_other.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import os import logging import tableau_migration from tableau_migration.migration_engine import ( PyMigrationPlanBuilder) +from Tableau.Migration.Config import IConfigReader + class TestEndToEnd(): def test_main(self): '''This is mean to mimic a real application''' @@ -48,3 +50,20 @@ def test_logging(self): for name in tableau_migration._logger_names: # Given that we have a name, we should have a logger assert logging.getLogger(name) + +class TestConfig(): + def test_config(self): + ''' + Verify that the MigrationSDK__BatchSize variable has been changed + + Environment variables act different in different operating systems. For this reason we + must set the MigrationSDK_BatchSize in pytest.ini, as that is set before the python + process starts and hence the env variables take and that way an int can be set to a env var. + ''' + + services = tableau_migration.migration.get_service_provider() + config_reader = tableau_migration.migration.get_service(services, IConfigReader) + + batch_size = config_reader.Get().BatchSize + + assert batch_size==102 \ No newline at end of file diff --git a/src/Tableau.Migration/Api/ApiClient.cs b/src/Tableau.Migration/Api/ApiClient.cs index 20ad17e..20fc1c6 100644 --- a/src/Tableau.Migration/Api/ApiClient.cs +++ b/src/Tableau.Migration/Api/ApiClient.cs @@ -20,6 +20,7 @@ using Tableau.Migration.Api.Models; using Tableau.Migration.Api.Rest.Models.Requests; using Tableau.Migration.Api.Rest.Models.Responses; +using Tableau.Migration.Content; using Tableau.Migration.Net.Rest; using Tableau.Migration.Resources; @@ -57,14 +58,17 @@ public ApiClient( tokenProvider.RefreshRequestedAsync += async (cancel) => { var signInResult = await GetSignInResultAsync(cancel).ConfigureAwait(false); - - if (signInResult.Success) + + if(!signInResult.Success) { - tokenProvider.Set(signInResult.Value.Token); + return signInResult.CastFailure(); } + + return Result.Succeeded(signInResult.Value.Token); }; } + /// public async Task> SignInAsync(CancellationToken cancel) { // Set the default API version if it hasn't been set already so we know which sign-in API version to call. @@ -83,7 +87,7 @@ public async Task> SignInAsync(Cancellat return AsyncDisposableResult.Failed(signInResult.Errors); } - _sessionProvider.SetCurrentUserAndSite(signInResult.Value); + await _sessionProvider.SetCurrentUserAndSiteAsync(signInResult.Value, cancel).ConfigureAwait(false); return AsyncDisposableResult.Succeeded(_sitesApiClient); } @@ -102,6 +106,7 @@ private async Task> GetSignInResultAsync(CancellationToke return signInResult; } + /// public async Task> GetServerInfoAsync(CancellationToken cancel) { // The first version this endpoint is available. @@ -122,5 +127,19 @@ public async Task> GetServerInfoAsync(CancellationToken can return serverInfoResult; } + + /// + public async Task> GetCurrentServerSessionAsync(CancellationToken cancel) + { + var serverSessionResult = await RestRequestBuilderFactory + .CreateUri("/sessions/current") + .WithSiteId(null) + .ForGetRequest() + .SendAsync(cancel) + .ToResultAsync(r => new ServerSession(r), SharedResourcesLocalizer) + .ConfigureAwait(false); + + return serverSessionResult; + } } } diff --git a/src/Tableau.Migration/Api/AuthenticationTokenProvider.cs b/src/Tableau.Migration/Api/AuthenticationTokenProvider.cs index eef244d..5d65c96 100644 --- a/src/Tableau.Migration/Api/AuthenticationTokenProvider.cs +++ b/src/Tableau.Migration/Api/AuthenticationTokenProvider.cs @@ -14,6 +14,7 @@ // limitations under the License. // +using System; using System.Threading; using System.Threading.Tasks; @@ -21,18 +22,83 @@ namespace Tableau.Migration.Api { internal sealed class AuthenticationTokenProvider : IAuthenticationTokenProvider { - public event AsyncEventHandler? RefreshRequestedAsync; + private readonly SemaphoreSlim _tokenSemaphore = new(1, 1); - public string? Token { get; private set; } + private string? _token; - public void Set(string token) => Token = token; + /// + public event RefreshAuthenticationTokenDelegate? RefreshRequestedAsync; - public void Clear() => Token = null; + /// + public async Task GetAsync(CancellationToken cancel) + { + await _tokenSemaphore.WaitAsync(cancel).ConfigureAwait(false); + try + { + return _token; + } + finally + { + _tokenSemaphore.Release(); + } + } + + /// + public async Task SetAsync(string token, CancellationToken cancel) + { + await _tokenSemaphore.WaitAsync(cancel).ConfigureAwait(false); + try + { + _token = token; + } + finally + { + _tokenSemaphore.Release(); + } + } - public async Task RequestRefreshAsync(CancellationToken cancel) + /// + public async Task ClearAsync(CancellationToken cancel) { - if (RefreshRequestedAsync is not null) - await RefreshRequestedAsync.Invoke(cancel).ConfigureAwait(false); + await _tokenSemaphore.WaitAsync(cancel).ConfigureAwait(false); + try + { + _token = null; + } + finally + { + _tokenSemaphore.Release(); + } + } + + /// + public async Task RequestRefreshAsync(string? previousToken, CancellationToken cancel) + { + if (RefreshRequestedAsync is null) + { + return; + } + + await _tokenSemaphore.WaitAsync(cancel).ConfigureAwait(false); + try + { + // Another thread refreshed the token while we waited for the refresh lock. + if(!string.Equals(previousToken, _token, StringComparison.Ordinal)) + { + return; + } + + var newTokenResult = await RefreshRequestedAsync.Invoke(cancel).ConfigureAwait(false); + + if(newTokenResult.Success) + { + _token = newTokenResult.Value; + } + } + finally + { + _tokenSemaphore.Release(); + } } } } diff --git a/src/Tableau.Migration/Api/IApiClient.cs b/src/Tableau.Migration/Api/IApiClient.cs index 0472103..ca3a878 100644 --- a/src/Tableau.Migration/Api/IApiClient.cs +++ b/src/Tableau.Migration/Api/IApiClient.cs @@ -17,6 +17,7 @@ using System.Threading; using System.Threading.Tasks; using Tableau.Migration.Api.Models; +using Tableau.Migration.Content; namespace Tableau.Migration.Api { @@ -26,17 +27,24 @@ namespace Tableau.Migration.Api public interface IApiClient { /// - /// Signs into Tableau Server + /// Signs into Tableau Server. /// - /// The cancellation token - /// An authenticated + /// The cancellation token. + /// An authenticated . Task> SignInAsync(CancellationToken cancel); /// - /// Gets the version information for the Tableau Server + /// Gets the version information for the Tableau Server. /// - /// The cancellation token - /// The information for the current Tableau Server + /// The cancellation token. + /// The information for the current Tableau Server. Task> GetServerInfoAsync(CancellationToken cancel); + + /// + /// Gets the current session information. + /// + /// The cancellation token. + /// The session information. + Task> GetCurrentServerSessionAsync(CancellationToken cancel); } } diff --git a/src/Tableau.Migration/Api/IAuthenticationTokenProvider.cs b/src/Tableau.Migration/Api/IAuthenticationTokenProvider.cs index e553a34..99cdd42 100644 --- a/src/Tableau.Migration/Api/IAuthenticationTokenProvider.cs +++ b/src/Tableau.Migration/Api/IAuthenticationTokenProvider.cs @@ -20,35 +20,46 @@ namespace Tableau.Migration.Api { /// - /// Interface for a class representing the current authentication token. + /// Interface for a thread safe class representing the current authentication token. /// public interface IAuthenticationTokenProvider { /// /// Event that fires when an authentication token refresh is requested. /// - event AsyncEventHandler? RefreshRequestedAsync; + event RefreshAuthenticationTokenDelegate? RefreshRequestedAsync; /// /// Gets the authentication token. /// - string? Token { get; } + /// A cancellation token to obey. + /// A task to await for the current authentication token. + Task GetAsync(CancellationToken cancel); /// /// Sets the authentication token. /// /// The authentication token received from the server. - void Set(string token); + /// A cancellation token to obey. + /// The task to await. + Task SetAsync(string token, CancellationToken cancel); /// /// Clears the authentication token. /// - void Clear(); + /// A cancellation token to obey. + /// The task to await. + Task ClearAsync(CancellationToken cancel); /// /// Requests an authentication token refresh. /// - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - Task RequestRefreshAsync(CancellationToken cancel); + /// + /// The token that was previously used before the refresh was requested. + /// Used to de-duplicate refresh requests. + /// + /// A cancellation token to obey. + /// The task to await. + Task RequestRefreshAsync(string? previousToken, CancellationToken cancel); } } \ No newline at end of file diff --git a/src/Tableau.Migration/Api/IServerSessionProvider.cs b/src/Tableau.Migration/Api/IServerSessionProvider.cs index e8bca11..5621fbe 100644 --- a/src/Tableau.Migration/Api/IServerSessionProvider.cs +++ b/src/Tableau.Migration/Api/IServerSessionProvider.cs @@ -15,6 +15,8 @@ // using System; +using System.Threading; +using System.Threading.Tasks; using Tableau.Migration.Api.Models; namespace Tableau.Migration.Api @@ -48,7 +50,9 @@ public interface IServerSessionProvider /// Sets the current user and site information. /// /// The sign-in result containing the current user and site information. - void SetCurrentUserAndSite(ISignInResult signInResult); + /// The cancellation token to obey. + /// The task to await. + Task SetCurrentUserAndSiteAsync(ISignInResult signInResult, CancellationToken cancel); /// /// Sets the current user and site information. @@ -57,12 +61,16 @@ public interface IServerSessionProvider /// The current site's ID. /// The current site's content URL. /// The current user's authentication token. - void SetCurrentUserAndSite(Guid userId, Guid siteId, string siteContentUrl, string authenticationToken); + /// The cancellation token to obey. + /// The task to await. + Task SetCurrentUserAndSiteAsync(Guid userId, Guid siteId, string siteContentUrl, string authenticationToken, CancellationToken cancel); /// /// Clears the current user and site information. /// - void ClearCurrentUserAndSite(); + /// The cancellation token to obey. + /// The task to await. + Task ClearCurrentUserAndSiteAsync(CancellationToken cancel); /// /// Sets the current version information. diff --git a/src/Tableau.Migration/Api/ISitesApiClient.cs b/src/Tableau.Migration/Api/ISitesApiClient.cs index 8b999a9..d6601ac 100644 --- a/src/Tableau.Migration/Api/ISitesApiClient.cs +++ b/src/Tableau.Migration/Api/ISitesApiClient.cs @@ -29,37 +29,37 @@ namespace Tableau.Migration.Api public interface ISitesApiClient : IAsyncDisposable, IContentApiClient { /// - /// The API client for group operations. + /// Gets the API client for group operations. /// IGroupsApiClient Groups { get; } /// - /// The API client for job operations. + /// Gets the API client for job operations. /// IJobsApiClient Jobs { get; } /// - /// The API client for project operations. + /// Gets the API client for project operations. /// IProjectsApiClient Projects { get; } /// - /// The API client for user operations. + /// Gets the API client for user operations. /// IUsersApiClient Users { get; } /// - /// The API client for data source operations. + /// Gets the API client for data source operations. /// IDataSourcesApiClient DataSources { get; } /// - /// The API client for workbook operations. + /// Gets the API client for workbook operations. /// IWorkbooksApiClient Workbooks { get; } /// - /// The API client for views operations. + /// Gets the API client for views operations. /// IViewsApiClient Views { get; } @@ -79,6 +79,14 @@ public interface ISitesApiClient : IAsyncDisposable, IContentApiClient /// The site with the specified content URL. Task> GetSiteAsync(string contentUrl, CancellationToken cancel); + /// + /// Updates the site. + /// + /// The settings to update on the site.. + /// The cancellation token. + /// The site information returned after the update. + Task> UpdateSiteAsync(ISiteSettingsUpdate update, CancellationToken cancel); + /// /// Gets the for the given content type. /// diff --git a/src/Tableau.Migration/AsyncEventHandler.cs b/src/Tableau.Migration/Api/RefreshAuthenticationTokenDelegate.cs similarity index 69% rename from src/Tableau.Migration/AsyncEventHandler.cs rename to src/Tableau.Migration/Api/RefreshAuthenticationTokenDelegate.cs index 22e0006..30fda37 100644 --- a/src/Tableau.Migration/AsyncEventHandler.cs +++ b/src/Tableau.Migration/Api/RefreshAuthenticationTokenDelegate.cs @@ -17,12 +17,12 @@ using System.Threading; using System.Threading.Tasks; -namespace Tableau.Migration +namespace Tableau.Migration.Api { /// - /// Delegate for asynchronous events. + /// Delegate for refreshing an authentication token. /// - /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. - /// - public delegate Task AsyncEventHandler(CancellationToken cancel); + /// The cancellation token to obey. + /// The result of the token refresh operation. + public delegate Task> RefreshAuthenticationTokenDelegate(CancellationToken cancel); } diff --git a/src/Tableau.Migration/Api/Rest/Models/AdministratorLevels.cs b/src/Tableau.Migration/Api/Rest/Models/AdministratorLevels.cs index c5554e9..c0e82aa 100644 --- a/src/Tableau.Migration/Api/Rest/Models/AdministratorLevels.cs +++ b/src/Tableau.Migration/Api/Rest/Models/AdministratorLevels.cs @@ -22,12 +22,12 @@ namespace Tableau.Migration.Api.Rest.Models public class AdministratorLevels : StringEnum { /// - /// Name for the level the level when a user has Site administrator permissions.. + /// Name for the level when a user has Site administrator permissions. /// public const string Site = "Site"; /// - /// Name for the level the level when a user has no administrator permissions. + /// Name for the level when a user has no administrator permissions. /// public const string None = "None"; } diff --git a/src/Tableau.Migration/Api/Rest/Models/ExtractEncryptionModes.cs b/src/Tableau.Migration/Api/Rest/Models/ExtractEncryptionModes.cs new file mode 100644 index 0000000..8883d3e --- /dev/null +++ b/src/Tableau.Migration/Api/Rest/Models/ExtractEncryptionModes.cs @@ -0,0 +1,41 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +namespace Tableau.Migration.Api.Rest.Models +{ + /// + /// Class containing extract encryption mode constants. + /// See Tableau API Reference + /// for documentation. + /// + public class ExtractEncryptionModes : StringEnum + { + /// + /// The mode to enforce encryption of all extracts on the site. + /// + public const string Enforced = "enforced"; + + /// + /// The mode to allow users to specify to encrypt all extracts associated with specific published workbooks or data sources. + /// + public const string Enabled = "enabled"; + + /// + /// The mode to disable extract encryption on the site. + /// + public const string Disabled = "disabled"; + } +} diff --git a/src/Tableau.Migration/Api/Rest/Models/Requests/UpdateSiteRequest.cs b/src/Tableau.Migration/Api/Rest/Models/Requests/UpdateSiteRequest.cs new file mode 100644 index 0000000..c28fcd6 --- /dev/null +++ b/src/Tableau.Migration/Api/Rest/Models/Requests/UpdateSiteRequest.cs @@ -0,0 +1,69 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System.Xml.Serialization; +using Tableau.Migration.Content; + +namespace Tableau.Migration.Api.Rest.Models.Requests +{ + /// + /// + /// Class representing an update site request. + /// + /// + /// See Tableau API Reference for documentation. + /// + /// + [XmlType(XmlTypeName)] + public class UpdateSiteRequest : TableauServerRequest + { + /// + /// Creates a new object. + /// + public UpdateSiteRequest() + { } + + /// + /// Creates a new object. + /// + /// The settings to update. + public UpdateSiteRequest(ISiteSettingsUpdate update) + { + Site = new() + { + ExtractEncryptionMode = update.ExtractEncryptionMode + }; + } + + /// + /// Gets or sets the site for the request. + /// + [XmlElement("site")] + public SiteType? Site { get; set; } + + /// + /// The site type in the API request body. + /// + public class SiteType + { + /// + /// Gets or sets the extract encryption mode for the request. + /// + [XmlAttribute("extractEncryptionMode")] + public string? ExtractEncryptionMode { get; set; } + } + } +} diff --git a/src/Tableau.Migration/Api/Rest/Models/Responses/ServerSessionResponse.cs b/src/Tableau.Migration/Api/Rest/Models/Responses/ServerSessionResponse.cs new file mode 100644 index 0000000..ac0aeef --- /dev/null +++ b/src/Tableau.Migration/Api/Rest/Models/Responses/ServerSessionResponse.cs @@ -0,0 +1,129 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Xml.Serialization; + +namespace Tableau.Migration.Api.Rest.Models.Responses +{ + /// + /// + /// Class representing a server session response. + /// + /// + /// See Tableau API Reference for documentation + /// + /// + [XmlType(XmlTypeName)] + public class ServerSessionResponse : TableauServerResponse + { + /// + /// Gets or sets the session for the response. + /// + [XmlElement("session")] + public override SessionType? Item { get; set; } + + /// + /// Class representing a session response. + /// + public class SessionType + { + /// + /// Gets or sets the site for the response. + /// + [XmlElement("site")] + public SiteType? Site { get; set; } + + /// + /// Gets or sets the user for the response. + /// + [XmlElement("user")] + public UserType? User { get; set; } + + /// + /// Class representing a site response. + /// + public class SiteType + { + /// + /// Gets or sets the ID for the response. + /// + [XmlAttribute("id")] + public Guid Id { get; set; } + + /// + /// Gets or sets the name for the response. + /// + [XmlAttribute("name")] + public string? Name { get; set; } + + /// + /// Gets or sets the content URL for the response. + /// + [XmlAttribute("contentUrl")] + public string? ContentUrl { get; set; } + + /// + /// Gets or sets the extract encryption mode for the response. + /// + [XmlAttribute("extractEncryptionMode")] + public string? ExtractEncryptionMode { get; set; } + } + + /// + /// Class representing a user response. + /// + public class UserType + { + /// + /// Gets or sets the authentication setting for the response. + /// + [XmlAttribute("authSetting")] + public string? AuthSetting { get; set; } + + /// + /// Gets or sets the ID for the response. + /// + [XmlAttribute("id")] + public Guid Id { get; set; } + + /// + /// Gets or sets the external authentication ID for the response. + /// + [XmlAttribute("externalAuthUserId")] + public string? ExternalAuthUserId { get; set; } + + /// + /// Gets or sets the last login for the response. + /// + [XmlAttribute("lastLogin")] + public string? LastLogin { get; set; } + + /// + /// Gets or sets the name for the response. + /// + [XmlAttribute("name")] + public string? Name { get; set; } + + /// + /// Gets or sets the site role for the response. + /// + [XmlAttribute("siteRole")] + public string? SiteRole { get; set; } + } + } + } +} diff --git a/src/Tableau.Migration/Api/Rest/Models/Responses/SiteResponse.cs b/src/Tableau.Migration/Api/Rest/Models/Responses/SiteResponse.cs index 7b2d8c3..1b1679a 100644 --- a/src/Tableau.Migration/Api/Rest/Models/Responses/SiteResponse.cs +++ b/src/Tableau.Migration/Api/Rest/Models/Responses/SiteResponse.cs @@ -53,6 +53,12 @@ public class SiteType : IRestIdentifiable, IApiContentUrl /// [XmlAttribute("contentUrl")] public string? ContentUrl { get; set; } + + /// + /// Gets or sets the site extract encryption mode for the response. + /// + [XmlAttribute("extractEncryptionMode")] + public string? ExtractEncryptionMode { get; set; } } } } diff --git a/src/Tableau.Migration/Api/ServerSessionProvider.cs b/src/Tableau.Migration/Api/ServerSessionProvider.cs index e198916..2199d88 100644 --- a/src/Tableau.Migration/Api/ServerSessionProvider.cs +++ b/src/Tableau.Migration/Api/ServerSessionProvider.cs @@ -15,6 +15,8 @@ // using System; +using System.Threading; +using System.Threading.Tasks; using Tableau.Migration.Api.Models; namespace Tableau.Migration.Api @@ -24,12 +26,19 @@ internal sealed class ServerSessionProvider : IServerSessionProvider private readonly ITableauServerVersionProvider _versionProvider; private readonly IAuthenticationTokenProvider _tokenProvider; + internal async Task GetAuthenticationTokenAsync(CancellationToken cancel) + => await _tokenProvider.GetAsync(cancel).ConfigureAwait(false); + + /// public TableauServerVersion? Version => _versionProvider.Version; - public string? AuthenticationToken => _tokenProvider.Token; - public Guid? SiteId { get; private set; } + /// public string? SiteContentUrl { get; private set; } + /// + public Guid? SiteId { get; private set; } + + /// public Guid? UserId { get; private set; } public ServerSessionProvider( @@ -40,27 +49,31 @@ public ServerSessionProvider( _tokenProvider = tokenProvider; } - public void SetCurrentUserAndSite(ISignInResult signInResult) - => SetCurrentUserAndSite(signInResult.UserId, signInResult.SiteId, signInResult.SiteContentUrl, signInResult.Token); + /// + public async Task SetCurrentUserAndSiteAsync(ISignInResult signInResult, CancellationToken cancel) + => await SetCurrentUserAndSiteAsync(signInResult.UserId, signInResult.SiteId, signInResult.SiteContentUrl, signInResult.Token, cancel).ConfigureAwait(false); - public void SetCurrentUserAndSite(Guid userId, Guid siteId, string siteContentUrl, string authenticationToken) + /// + public async Task SetCurrentUserAndSiteAsync(Guid userId, Guid siteId, string siteContentUrl, string authenticationToken, CancellationToken cancel) { SiteId = siteId; SiteContentUrl = siteContentUrl; UserId = userId; - _tokenProvider.Set(authenticationToken); + await _tokenProvider.SetAsync(authenticationToken, cancel).ConfigureAwait(false); } - public void ClearCurrentUserAndSite() + /// + public async Task ClearCurrentUserAndSiteAsync(CancellationToken cancel) { SiteId = null; SiteContentUrl = null; UserId = null; - _tokenProvider.Clear(); + await _tokenProvider.ClearAsync(cancel).ConfigureAwait(false); } + /// public void SetVersion(TableauServerVersion version) => _versionProvider.Set(version); } } diff --git a/src/Tableau.Migration/Api/Simulation/Rest/Api/SitesRestApiSimulator.cs b/src/Tableau.Migration/Api/Simulation/Rest/Api/SitesRestApiSimulator.cs index 263e7e0..78b2ca7 100644 --- a/src/Tableau.Migration/Api/Simulation/Rest/Api/SitesRestApiSimulator.cs +++ b/src/Tableau.Migration/Api/Simulation/Rest/Api/SitesRestApiSimulator.cs @@ -14,9 +14,14 @@ // limitations under the License. // +using System; +using System.Linq; +using System.Net.Http; using System.Text.RegularExpressions; +using Tableau.Migration.Api.Rest.Models.Requests; using Tableau.Migration.Api.Rest.Models.Responses; using Tableau.Migration.Api.Simulation.Rest.Net; +using Tableau.Migration.Api.Simulation.Rest.Net.Requests; using Tableau.Migration.Net.Simulation; using static Tableau.Migration.Api.Simulation.Rest.Net.Requests.RestUrlPatterns; @@ -38,6 +43,11 @@ public sealed class SitesRestApiSimulator /// public MethodSimulator QuerySiteByContentUrl { get; } + /// + /// Gets the simulated update site API method. + /// + public MethodSimulator UpdateSite { get; } + /// /// Creates a new object. /// @@ -47,6 +57,36 @@ public SitesRestApiSimulator(TableauApiResponseSimulator simulator) QuerySiteById = simulator.SetupRestGetById(RestApiUrl($"sites/{SiteId}"), d => d.Sites); QuerySiteByContentUrl = simulator.SetupRestGetByContentUrl(RestApiUrl($"sites/{ContentUrlPattern}"), d => d.Sites, queryStringPatterns: new[] { ("key", new Regex("contentUrl")) }); + + UpdateSite = simulator.SetupRestPut(RestApiUrl($"sites/{SiteId}"), UpdateSiteFromRequest); + } + + private static SiteResponse.SiteType? UpdateSiteFromRequest(TableauData data, HttpRequestMessage request) + { + var id = request.GetIdAfterSegment("sites"); + if (id is null) + { + throw new InvalidOperationException("Site ID should not be null"); + } + + var site = data.Sites.SingleOrDefault(s => s.Id == id.Value); + if (site is null) + { + return null; + } + + var updateRequest = request.GetTableauServerRequest()?.Site; + if (updateRequest is null) + { + return null; + } + + if (updateRequest.ExtractEncryptionMode is not null) + { + site.ExtractEncryptionMode = updateRequest.ExtractEncryptionMode; + } + + return site; } } } diff --git a/src/Tableau.Migration/Api/Simulation/Rest/RestApiSimulator.cs b/src/Tableau.Migration/Api/Simulation/Rest/RestApiSimulator.cs index 3a6f05a..ac472ff 100644 --- a/src/Tableau.Migration/Api/Simulation/Rest/RestApiSimulator.cs +++ b/src/Tableau.Migration/Api/Simulation/Rest/RestApiSimulator.cs @@ -14,6 +14,8 @@ // limitations under the License. // +using System.Linq; +using Tableau.Migration.Api.Rest.Models; using Tableau.Migration.Api.Rest.Models.Responses; using Tableau.Migration.Api.Simulation.Rest.Api; using Tableau.Migration.Api.Simulation.Rest.Net; @@ -83,6 +85,42 @@ public sealed class RestApiSimulator /// public MethodSimulator QueryServerInfo { get; } + /// + /// Gets the simulated current server session query API method. + /// + public MethodSimulator GetCurrentServerSession { get; } + + private static ServerSessionResponse.SessionType BuildCurrentSession(TableauData data) + { + var user = data.Users.Single(u => u.Id == data.SignIn!.User!.Id); + var site = data.Sites.Single(s => s.Id == data.SignIn!.Site!.Id); + + var response = new ServerSessionResponse.SessionType + { + Site = new() + { + Id = site.Id, + ContentUrl = site.ContentUrl, + Name = site.Name + }, + User = new() + { + AuthSetting = user.AuthSetting, + Id = user.Id, + Name = user.Name, + SiteRole = user.SiteRole + } + }; + + var adminLevel = SiteRoleMapping.GetAdministratorLevel(user.SiteRole); + if(!AdministratorLevels.IsAMatch(adminLevel, AdministratorLevels.None)) + { + response.Site.ExtractEncryptionMode = site.ExtractEncryptionMode; + } + + return response; + } + /// /// Creates a new object. /// @@ -100,8 +138,8 @@ public RestApiSimulator(TableauApiResponseSimulator simulator) Files = new(simulator); Views = new(simulator); - QueryServerInfo = simulator.SetupRestGet(RestApiUrl("serverinfo"), d => d.ServerInfo, requiresAuthentication: false); + GetCurrentServerSession = simulator.SetupRestGet(RestApiUrl("sessions/current"), BuildCurrentSession); } } } diff --git a/src/Tableau.Migration/Api/Simulation/TableauData.cs b/src/Tableau.Migration/Api/Simulation/TableauData.cs index b27fbf1..ab985b7 100644 --- a/src/Tableau.Migration/Api/Simulation/TableauData.cs +++ b/src/Tableau.Migration/Api/Simulation/TableauData.cs @@ -362,7 +362,10 @@ private void AddSignInSite(Guid id) => Sites.Add( new() { - Id = id + Id = id, + ContentUrl = "", + Name = "Default", + ExtractEncryptionMode = ExtractEncryptionModes.Disabled }); private static GroupsResponse.GroupType CreateAllUsersGroup() diff --git a/src/Tableau.Migration/Api/SitesApiClient.cs b/src/Tableau.Migration/Api/SitesApiClient.cs index 5bbf17c..f447619 100644 --- a/src/Tableau.Migration/Api/SitesApiClient.cs +++ b/src/Tableau.Migration/Api/SitesApiClient.cs @@ -21,6 +21,7 @@ using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Tableau.Migration.Api.Permissions; +using Tableau.Migration.Api.Rest.Models.Requests; using Tableau.Migration.Api.Rest.Models.Responses; using Tableau.Migration.Api.Tags; using Tableau.Migration.Content; @@ -168,6 +169,21 @@ public async Task> GetSiteAsync(Guid siteId, CancellationToken ca public async Task> GetSiteAsync(string contentUrl, CancellationToken cancel) => await GetSiteAsync(r => r.WithSiteId(contentUrl).WithQuery(q => q.AddOrUpdate("key", "contentUrl")), cancel).ConfigureAwait(false); + /// + public async Task> UpdateSiteAsync(ISiteSettingsUpdate update, CancellationToken cancel) + { + var updateResult = await RestRequestBuilderFactory + .CreateUri("/") //"sites" URL segment added by WithSiteId + .WithSiteId(update.SiteId) + .ForPutRequest() + .WithXmlContent(new UpdateSiteRequest(update)) + .SendAsync(cancel) + .ToResultAsync(r => new Site(r), SharedResourcesLocalizer) + .ConfigureAwait(false); + + return updateResult; + } + #endregion #region - SignOutAsync - @@ -190,7 +206,7 @@ internal async Task SignOutAsync(CancellationToken cancel) .ToResultAsync(_serializer, SharedResourcesLocalizer, cancel) .ConfigureAwait(false); - _sessionProvider.ClearCurrentUserAndSite(); + await _sessionProvider.ClearCurrentUserAndSiteAsync(cancel).ConfigureAwait(false); return signOutResult; } diff --git a/src/Tableau.Migration/Config/PreflightOptions.cs b/src/Tableau.Migration/Config/PreflightOptions.cs new file mode 100644 index 0000000..732fcb8 --- /dev/null +++ b/src/Tableau.Migration/Config/PreflightOptions.cs @@ -0,0 +1,45 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +namespace Tableau.Migration.Config +{ + /// + /// Options releated to the preflight step before content is migrated. + /// + public class PreflightOptions + { + /// + /// Defaults for preflight options. + /// + public static class Defaults + { + /// + /// Whether to validate supported site settings during the preflight step. + /// + public const bool VALIDATE_SETTINGS = true; + } + + /// + /// Get or sets whether to validate supported site settings during the preflight step. Defaults to true. + /// + public bool ValidateSettings + { + get => _validateSettings ?? Defaults.VALIDATE_SETTINGS; + set => _validateSettings = value; + } + private bool? _validateSettings; + } +} diff --git a/src/Tableau.Migration/Config/ResilienceOptions.cs b/src/Tableau.Migration/Config/ResilienceOptions.cs index 2fd4131..72c4333 100644 --- a/src/Tableau.Migration/Config/ResilienceOptions.cs +++ b/src/Tableau.Migration/Config/ResilienceOptions.cs @@ -29,7 +29,7 @@ public class ResilienceOptions public static class Defaults { /// - /// The default Retry Flag - Enabled as Default. + /// The default Retry Flag. Enabled as Default. /// public const bool RETRY_ENABLED = true; @@ -47,27 +47,27 @@ public static class Defaults }; /// - /// The default Retry Override Response Codes, Empty as Default. + /// The default Retry Override Response Codes. Empty by Default. /// public readonly static int[] RETRY_OVERRIDE_RESPONSE_CODES = Array.Empty(); /// - /// The default Concurrent Requests Limit Flag - Disabled as Default. + /// The default Concurrent Requests Limit Flag. Disabled by Default. /// public const bool CONCURRENT_REQUESTS_LIMIT_ENABLED = false; /// - /// The default Maximum Concurrent Requests. + /// The default Maximum Concurrent Requests. Default is Processor count / 2 /// public readonly static int MAX_CONCURRENT_REQUESTS = Environment.ProcessorCount / 2; /// - /// The default Concurrent Waiting Requests on Queue. + /// The default Concurrent Waiting Requests on Queue. Default is Processor count / 4 /// public readonly static int CONCURRENT_WAITING_REQUESTS_QUEUE = Environment.ProcessorCount / 4; /// - /// The default Requests Client Throttle (Rate-Limit) Flag - Disabled as Default. + /// The default Requests Client Throttle (Rate-Limit) Flag. Disabled as Default. /// public const bool CLIENT_THROTTLE_ENABLED = false; @@ -95,24 +95,24 @@ public static class Defaults }; /// - /// The default Maximum Read Requests for the Client Throttle. + /// The default Maximum Read Requests for the Client Throttle. Default is 40000. /// public const int MAX_READ_REQUESTS = 40000; /// - /// The default interval for Read Requests Throttle. + /// The default interval for Read Requests Throttle. Default is 1 hour /// public readonly static TimeSpan MAX_READ_REQUESTS_INTERVAL = TimeSpan.FromHours(1); /// - /// The default Burst Read Requests for the Client Throttle. + /// The default Burst Read Requests for the Client Throttle. Default is 20. /// Without the burst configuration, it will be allowed just one request for each 90 milliseconds (1 hour / 40000 requests). /// This override the configuration and allow 20 requests in an interval of 90 milliseconds. /// public const int MAX_BURST_READ_REQUESTS = 20; /// - /// The default Maximum Publish Requests for the Client Throttle. + /// The default Maximum Publish Requests for the Client Throttle. Default is 5500. /// public const int MAX_PUBLISH_REQUESTS = 5500; @@ -122,25 +122,25 @@ public static class Defaults public readonly static TimeSpan MAX_PUBLISH_REQUESTS_INTERVAL = TimeSpan.FromDays(1); /// - /// The default Burst Publish Requests for the Client Throttle. + /// The default Burst Publish Requests for the Client Throttle. Default is 20. /// Without the burst configuration, it will be allowed just one request for each 16 seconds (1 day / 5500 requests). /// This overrides the configuration and allow 20 requests in an interval of 16 seconds. /// public const int MAX_BURST_PUBLISH_REQUESTS = 20; /// - /// The default Per-Request Timeout. 30 minutes as Default. + /// The default Per-Request Timeout. Default is 30 minutes. /// public readonly static TimeSpan REQUEST_TIMEOUT = TimeSpan.FromMinutes(30); /// - /// The default Per-FileTransferRequest Timeout. 12 hours as Default. + /// The default Per-FileTransferRequest Timeout. Default is 12 hours. /// public readonly static TimeSpan FILE_TRANSFER_REQUEST_TIMEOUT = TimeSpan.FromHours(12); } /// - /// Indicates if we retry requests in case of errors or not. The default is `true`. + /// Indicates if we retry requests in case of errors or not. /// public bool RetryEnabled { diff --git a/src/Tableau.Migration/Content/IServerSession.cs b/src/Tableau.Migration/Content/IServerSession.cs new file mode 100644 index 0000000..6f653ea --- /dev/null +++ b/src/Tableau.Migration/Content/IServerSession.cs @@ -0,0 +1,42 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System.Diagnostics.CodeAnalysis; + +namespace Tableau.Migration.Content +{ + /// + /// Interface for the current server session information. + /// + public interface IServerSession + { + /// + /// Gets the current session's site. + /// + IContentReference Site { get; } + + /// + /// Gets the site settings, or null if the user does not have access to the settings. + /// + ISiteSettings? Settings { get; } + + /// + /// Gets whether or not the current user has administrator access. + /// + [MemberNotNullWhen(true, nameof(Settings))] + bool IsAdministrator { get; } + } +} diff --git a/src/Tableau.Migration/Content/ISite.cs b/src/Tableau.Migration/Content/ISite.cs index b53c844..7e83dba 100644 --- a/src/Tableau.Migration/Content/ISite.cs +++ b/src/Tableau.Migration/Content/ISite.cs @@ -22,6 +22,7 @@ namespace Tableau.Migration.Content /// Interface for a site. /// public interface ISite //We don't implement IContentReference because sites aren't a true 'content type.' + : ISiteSettings { /// /// Gets the unique identifier of the site, diff --git a/src/Tableau.Migration/Content/ISiteSettings.cs b/src/Tableau.Migration/Content/ISiteSettings.cs new file mode 100644 index 0000000..cd3ab43 --- /dev/null +++ b/src/Tableau.Migration/Content/ISiteSettings.cs @@ -0,0 +1,29 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +namespace Tableau.Migration.Content +{ + /// + /// Interface for a site's settings. + /// + public interface ISiteSettings + { + /// + /// Gets the site's extract encryption mode. + /// + string ExtractEncryptionMode { get; } + } +} diff --git a/src/Tableau.Migration/Content/ISiteSettingsUpdate.cs b/src/Tableau.Migration/Content/ISiteSettingsUpdate.cs new file mode 100644 index 0000000..df3f25b --- /dev/null +++ b/src/Tableau.Migration/Content/ISiteSettingsUpdate.cs @@ -0,0 +1,42 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; + +namespace Tableau.Migration.Content +{ + /// + /// Interface for a site settings update operation. + /// + public interface ISiteSettingsUpdate + { + /// + /// Gets the ID of the site to update settings for. + /// + Guid SiteId { get; } + + /// + /// Gets the new extract encryption mode, or null to not update the setting. + /// + string? ExtractEncryptionMode { get; set; } + + /// + /// Finds whether any settings require updates. + /// + /// True if any setting has changes, otherwise false. + bool NeedsUpdate(); + } +} diff --git a/src/Tableau.Migration/Content/ServerSession.cs b/src/Tableau.Migration/Content/ServerSession.cs new file mode 100644 index 0000000..1d9226c --- /dev/null +++ b/src/Tableau.Migration/Content/ServerSession.cs @@ -0,0 +1,54 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using Tableau.Migration.Api.Rest.Models; +using Tableau.Migration.Api.Rest.Models.Responses; + +namespace Tableau.Migration.Content +{ + internal sealed class ServerSession : IServerSession + { + /// + public IContentReference Site { get; } + + /// + public ISiteSettings? Settings { get; } + + /// + public bool IsAdministrator { get; } + + public ServerSession(ServerSessionResponse response) + { + var session = Guard.AgainstNull(response.Item, () => response.Item); + + var site = Guard.AgainstNull(response.Item.Site, () => response.Item.Site); + + Site = new ContentReferenceStub(Guard.AgainstDefaultValue(site.Id, () => response.Item.Site.Id), + Guard.AgainstNull(site.ContentUrl, () => response.Item.Site.ContentUrl), + new(Guard.AgainstNullEmptyOrWhiteSpace(site.Name, () => response.Item.Site.Name))); + + var user = Guard.AgainstNull(response.Item.User, () => response.Item.User); + var siteRole = Guard.AgainstNullEmptyOrWhiteSpace(response.Item.User.SiteRole, () => response.Item.User.SiteRole); + var adminLevel = SiteRoleMapping.GetAdministratorLevel(siteRole); + IsAdministrator = !AdministratorLevels.IsAMatch(adminLevel, AdministratorLevels.None); + + if(IsAdministrator) + { + Settings = new ServerSessionSettings(response.Item.Site); + } + } + } +} diff --git a/src/Tableau.Migration/Content/ServerSessionSettings.cs b/src/Tableau.Migration/Content/ServerSessionSettings.cs new file mode 100644 index 0000000..c82f3f8 --- /dev/null +++ b/src/Tableau.Migration/Content/ServerSessionSettings.cs @@ -0,0 +1,31 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using Tableau.Migration.Api.Rest.Models.Responses; + +namespace Tableau.Migration.Content +{ + internal sealed class ServerSessionSettings : ISiteSettings + { + /// + public string ExtractEncryptionMode { get; } + + public ServerSessionSettings(ServerSessionResponse.SessionType.SiteType response) + { + ExtractEncryptionMode = Guard.AgainstNullEmptyOrWhiteSpace(response.ExtractEncryptionMode, () => response.ExtractEncryptionMode); + } + } +} diff --git a/src/Tableau.Migration/Content/Site.cs b/src/Tableau.Migration/Content/Site.cs index 7dd8f4a..e0d2531 100644 --- a/src/Tableau.Migration/Content/Site.cs +++ b/src/Tableau.Migration/Content/Site.cs @@ -40,6 +40,9 @@ internal sealed class Site : ISite /// public string ContentUrl { get; } + /// + public string ExtractEncryptionMode { get; set; } + public Site(SiteResponse response) { var site = Guard.AgainstNull(response.Item, () => response.Item); @@ -47,6 +50,7 @@ public Site(SiteResponse response) Id = Guard.AgainstDefaultValue(site.Id, () => response.Item.Id); Name = Guard.AgainstNullEmptyOrWhiteSpace(site.Name, () => response.Item.Name); ContentUrl = Guard.AgainstNull(site.ContentUrl, () => response.Item.ContentUrl); + ExtractEncryptionMode = Guard.AgainstNullEmptyOrWhiteSpace(site.ExtractEncryptionMode, () => response.Item.ExtractEncryptionMode); } } } diff --git a/src/Tableau.Migration/Content/SiteSettingsUpdate.cs b/src/Tableau.Migration/Content/SiteSettingsUpdate.cs new file mode 100644 index 0000000..86ac649 --- /dev/null +++ b/src/Tableau.Migration/Content/SiteSettingsUpdate.cs @@ -0,0 +1,41 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; + +namespace Tableau.Migration.Content +{ + internal sealed class SiteSettingsUpdate : ISiteSettingsUpdate + { + /// + /// Creates a new object. + /// + /// The ID of the site to update. + public SiteSettingsUpdate(Guid siteId) + { + SiteId = siteId; + } + + /// + public Guid SiteId { get; } + + /// + public string? ExtractEncryptionMode { get; set; } + + /// + public bool NeedsUpdate() => ExtractEncryptionMode is not null; + } +} diff --git a/src/Tableau.Migration/Engine/Actions/PreflightAction.cs b/src/Tableau.Migration/Engine/Actions/PreflightAction.cs index 7690245..e8c343e 100644 --- a/src/Tableau.Migration/Engine/Actions/PreflightAction.cs +++ b/src/Tableau.Migration/Engine/Actions/PreflightAction.cs @@ -16,6 +16,12 @@ using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using Tableau.Migration.Api.Rest.Models; +using Tableau.Migration.Config; +using Tableau.Migration.Content; +using Tableau.Migration.Resources; namespace Tableau.Migration.Engine.Actions { @@ -24,12 +30,104 @@ namespace Tableau.Migration.Engine.Actions /// public class PreflightAction : IMigrationAction { + private readonly PreflightOptions _options; + private readonly IMigration _migration; + private readonly ILogger _logger; + private readonly ISharedResourcesLocalizer _localizer; + + /// + /// Creates a new object. + /// + /// The preflight options. + /// The current migration. + /// A logger. + /// A localizer. + public PreflightAction(IOptions options, IMigration migration, + ILogger logger, ISharedResourcesLocalizer localizer) + { + _options = options.Value; + _migration = migration; + _logger = logger; + _localizer = localizer; + } + + private void ValidateExtractEncryptionSetting(ISiteSettings source, ISiteSettings destination) + { + /* Any destination value is valid if source has encryption disabled. + * There won't be extract migration errors and there may be destination extracts encrypted. + */ + if (ExtractEncryptionModes.IsAMatch(source.ExtractEncryptionMode, ExtractEncryptionModes.Disabled)) + { + return; + } + + /* We don't care if the destination site is enforced/enabled as long as it supports encrypted extracts. + * There won't be extract migration errors and the destination gets to keep its preference. + */ + if (!ExtractEncryptionModes.IsAMatch(destination.ExtractEncryptionMode, ExtractEncryptionModes.Disabled)) + { + return; + } + + // Warn the user about the potential failure of encrypted extract migration. + _logger.LogWarning(_localizer[SharedResourceKeys.SiteSettingsExtractEncryptionDisabledLogMessage]); + } + + private async ValueTask ManageSettingsAsync(CancellationToken cancel) + { + if(!_options.ValidateSettings) + { + _logger.LogDebug(_localizer[SharedResourceKeys.SiteSettingsSkippedDisabledLogMessage]); + return Result.Succeeded(); + } + + // Get the source and destination settings to compare concurrently. + var sourceSessionTask = _migration.Source.GetSessionAsync(cancel); + var destinationSessionTask = _migration.Destination.GetSessionAsync(cancel); + + await Task.WhenAll(sourceSessionTask, destinationSessionTask).ConfigureAwait(false); + + var sourceSessionResult = sourceSessionTask.Result; + var destinationSessionResult = destinationSessionTask.Result; + + if(!sourceSessionResult.Success || !destinationSessionResult.Success) + { + return new ResultBuilder().Add(sourceSessionResult, destinationSessionResult).Build(); + } + + // Find if we have access to validate settings. + var sourceSession = sourceSessionResult.Value; + var destinationSession = destinationSessionResult.Value; + + if(!sourceSession.IsAdministrator || !destinationSession.IsAdministrator) + { + _logger.LogDebug(_localizer[SharedResourceKeys.SiteSettingsSkippedNoAccessLogMessage]); + return Result.Succeeded(); + } + + // Validate supported settings. + + ValidateExtractEncryptionSetting(sourceSession.Settings, destinationSession.Settings); + + /* We currently don't update settings for the user because + * Tableau Cloud returns an error when site administrators update site settings, + * requiring server administrator access that Tableau Cloud users cannot have. + * + * If/when that gets addressed we can update the destination setting automatically. + */ + + return Result.Succeeded(); + } + /// - public Task ExecuteAsync(CancellationToken cancel) + public async Task ExecuteAsync(CancellationToken cancel) { //TODO (W-12586258): Preflight action should validate that hook factories return the right type. //TODO (W-12586258): Preflight action should validate endpoints beyond simple initialization. - return Task.FromResult((IMigrationActionResult)MigrationActionResult.Succeeded()); + + var settingsResult = await ManageSettingsAsync(cancel).ConfigureAwait(false); + + return MigrationActionResult.FromResult(settingsResult); } } } diff --git a/src/Tableau.Migration/Engine/Endpoints/IDestinationEndpoint.cs b/src/Tableau.Migration/Engine/Endpoints/IDestinationEndpoint.cs index 63237bb..b8c09d0 100644 --- a/src/Tableau.Migration/Engine/Endpoints/IDestinationEndpoint.cs +++ b/src/Tableau.Migration/Engine/Endpoints/IDestinationEndpoint.cs @@ -131,12 +131,20 @@ Task>> ListConnectionsAsync( /// The ID of the connection to be updated. /// The update connetion options. /// The cancellation token to obey. - /// + /// The result of the connection update operation. Task> UpdateConnectionAsync( Guid contentItemId, Guid connectionId, IUpdateConnectionOptions options, CancellationToken cancel) where TContent : IWithConnections; + + /// + /// Update the settings of a site. + /// + /// The site settings to update. + /// The cancellation token to obey. + /// The result of the site update operation. + Task> UpdateSiteSettingsAsync(ISiteSettingsUpdate newSiteSettings, CancellationToken cancel); } } diff --git a/src/Tableau.Migration/Engine/Endpoints/IMigrationEndpoint.cs b/src/Tableau.Migration/Engine/Endpoints/IMigrationEndpoint.cs index ea4af38..8942ba8 100644 --- a/src/Tableau.Migration/Engine/Endpoints/IMigrationEndpoint.cs +++ b/src/Tableau.Migration/Engine/Endpoints/IMigrationEndpoint.cs @@ -17,6 +17,7 @@ using System; using System.Threading; using System.Threading.Tasks; +using Tableau.Migration.Content; using Tableau.Migration.Paging; namespace Tableau.Migration.Engine.Endpoints @@ -40,5 +41,12 @@ public interface IMigrationEndpoint : IAsyncDisposable /// The page size to use. /// A pager to list content with. IPager GetPager(int pageSize); + + /// + /// Gets the current server session information. + /// + /// A cancellation token to obey. + /// An awaitable task with the server session result. + Task> GetSessionAsync(CancellationToken cancel); } } diff --git a/src/Tableau.Migration/Engine/Endpoints/TableauApiDestinationEndpoint.cs b/src/Tableau.Migration/Engine/Endpoints/TableauApiDestinationEndpoint.cs index ed97d7e..e3cf303 100644 --- a/src/Tableau.Migration/Engine/Endpoints/TableauApiDestinationEndpoint.cs +++ b/src/Tableau.Migration/Engine/Endpoints/TableauApiDestinationEndpoint.cs @@ -102,5 +102,11 @@ public async Task> UpdateConnectionAsync( var apiClient = SiteApi.GetConnectionsApiClient(); return await apiClient.UpdateConnectionAsync(contentItemId, connectionId, options, cancel).ConfigureAwait(false); } + + /// + public async Task> UpdateSiteSettingsAsync(ISiteSettingsUpdate newSiteSettings, CancellationToken cancel) + { + return await SiteApi.UpdateSiteAsync(newSiteSettings, cancel).ConfigureAwait(false); + } } } diff --git a/src/Tableau.Migration/Engine/Endpoints/TableauApiEndpointBase.cs b/src/Tableau.Migration/Engine/Endpoints/TableauApiEndpointBase.cs index 4ab85d3..937cc63 100644 --- a/src/Tableau.Migration/Engine/Endpoints/TableauApiEndpointBase.cs +++ b/src/Tableau.Migration/Engine/Endpoints/TableauApiEndpointBase.cs @@ -118,6 +118,10 @@ public IPager GetPager(int pageSize) return listApi.GetPager(pageSize); } + /// + public async Task> GetSessionAsync(CancellationToken cancel) + => await ServerApi.GetCurrentServerSessionAsync(cancel).ConfigureAwait(false); + /// public async Task> GetPermissionsAsync(IContentReference contentItem, CancellationToken cancel) where TContent : IPermissionsContent diff --git a/src/Tableau.Migration/Engine/Hooks/Transformers/Default/GroupUsersTransformer.cs b/src/Tableau.Migration/Engine/Hooks/Transformers/Default/GroupUsersTransformer.cs index 3a6f83c..8c1412a 100644 --- a/src/Tableau.Migration/Engine/Hooks/Transformers/Default/GroupUsersTransformer.cs +++ b/src/Tableau.Migration/Engine/Hooks/Transformers/Default/GroupUsersTransformer.cs @@ -50,27 +50,27 @@ public GroupUsersTransformer( /// public override async Task TransformAsync( - IPublishableGroup ctx, + IPublishableGroup sourceGroup, CancellationToken cancel) { - var contentFinder = _migrationPipeline.CreateDestinationFinder(); + var userFinder = _migrationPipeline.CreateDestinationFinder(); - foreach (var user in ctx.Users) + foreach (var user in sourceGroup.Users) { - var contentDestination = await contentFinder + var destinationUser = await userFinder .FindDestinationReferenceAsync(user.User.Location, cancel) .ConfigureAwait(false); - if (contentDestination is not null) + if (destinationUser is not null) { - user.User = contentDestination; + user.User = destinationUser; } else { - _logger.LogWarning(_localizer[SharedResourceKeys.GroupUsersTransformerCannotMapWarning], ctx.Name, user.User.Location); + _logger.LogWarning(_localizer[SharedResourceKeys.GroupUsersTransformerCannotAddUserWarning], sourceGroup.Name, user.User.Location); } } - return ctx; + return sourceGroup; } } } diff --git a/src/Tableau.Migration/Engine/Hooks/Transformers/Default/PermissionsTransformer.cs b/src/Tableau.Migration/Engine/Hooks/Transformers/Default/PermissionsTransformer.cs index 95f6eed..302db63 100644 --- a/src/Tableau.Migration/Engine/Hooks/Transformers/Default/PermissionsTransformer.cs +++ b/src/Tableau.Migration/Engine/Hooks/Transformers/Default/PermissionsTransformer.cs @@ -19,11 +19,14 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Localization; +using Microsoft.Extensions.Logging; using Tableau.Migration.Api.Rest.Models; using Tableau.Migration.Content; using Tableau.Migration.Content.Permissions; using Tableau.Migration.Engine.Endpoints.Search; using Tableau.Migration.Engine.Pipelines; +using Tableau.Migration.Resources; namespace Tableau.Migration.Engine.Hooks.Transformers.Default { @@ -34,15 +37,21 @@ public class PermissionsTransformer : IPermissionsTransformer { private readonly IMappedContentReferenceFinder _userContentFinder; private readonly IMappedContentReferenceFinder _groupContentFinder; + private readonly ILogger _logger; + private readonly ISharedResourcesLocalizer _localizer; /// /// Creates a new object. /// /// Destination content finder object. - public PermissionsTransformer(IMigrationPipeline migrationPipeline) + /// Default logger. + /// A string localizer. + public PermissionsTransformer(IMigrationPipeline migrationPipeline, ILogger logger, ISharedResourcesLocalizer localizer) { _userContentFinder = migrationPipeline.CreateDestinationFinder(); _groupContentFinder = migrationPipeline.CreateDestinationFinder(); + _logger = logger; + _localizer = localizer; } private static bool ShouldMigrateCapability(ICapability c) @@ -78,7 +87,7 @@ private static bool ShouldMigrateCapability(ICapability c) var groupsById = new HashSet(granteeCapabilities).GroupBy(c => c.GranteeId); foreach (var group in groupsById) - { + { var granteeType = group.First().GranteeType; IMappedContentReferenceFinder contentFinder = granteeType is GranteeType.User @@ -90,6 +99,7 @@ private static bool ShouldMigrateCapability(ICapability c) if (destinationGrantee is null) { + _logger.LogWarning(_localizer.GetString(SharedResourceKeys.PermissionsTransformerGranteeNotFoundWarning), granteeType.ToString(), group.Key); continue; } diff --git a/src/Tableau.Migration/Interop/Hooks/Mappings/SyncTableauCloudUsernameMapping.cs b/src/Tableau.Migration/Interop/Hooks/Mappings/SyncTableauCloudUsernameMapping.cs deleted file mode 100644 index 18fa9ba..0000000 --- a/src/Tableau.Migration/Interop/Hooks/Mappings/SyncTableauCloudUsernameMapping.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using Tableau.Migration.Content; -using Tableau.Migration.Engine.Hooks.Mappings; -using Tableau.Migration.Engine.Hooks.Mappings.Default; - -namespace Tableau.Migration.Interop.Hooks.Mappings -{ - /// - /// - /// - abstract public class SyncTableauCloudUsernameMapping : ITableauCloudUsernameMapping - { - /// - /// - /// - /// - /// - abstract public ContentMappingContext? Execute(ContentMappingContext ctx); - - /// - /// - /// - /// - /// - /// - public Task?> ExecuteAsync(ContentMappingContext ctx, CancellationToken cancel) => Task.FromResult(Execute(ctx)); - - } -} diff --git a/src/Tableau.Migration/Interop/IServiceCollectionExtensions.cs b/src/Tableau.Migration/Interop/IServiceCollectionExtensions.cs index 000d6bf..d15a816 100644 --- a/src/Tableau.Migration/Interop/IServiceCollectionExtensions.cs +++ b/src/Tableau.Migration/Interop/IServiceCollectionExtensions.cs @@ -15,9 +15,11 @@ // using System; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; using Microsoft.Extensions.Logging; +using Tableau.Migration.Config; using Tableau.Migration.Interop.Logging; namespace Tableau.Migration.Interop @@ -28,7 +30,8 @@ namespace Tableau.Migration.Interop public static class IServiceCollectionExtensions { /// - /// Add python support. This will clear all existing s. All other logger provider should be added after this call. + /// Add python support by adding python logging and configuration via environment variables. + /// This will clear all existing s. All other logger provider should be added after this call. /// /// /// @@ -39,7 +42,9 @@ public static IServiceCollection AddPythonSupport(this IServiceCollection servic // Replace the default IUserAgentSuffixProvider with the python one .Replace(new ServiceDescriptor(typeof(IUserAgentSuffixProvider), typeof(PythonUserAgentSuffixProvider), ServiceLifetime.Singleton)) // Add Python Logging - .AddLogging(b => b.AddPythonLogging(pythonProviderFactory)); + .AddLogging(b => b.AddPythonLogging(pythonProviderFactory)) + // Add environment variable configuration + .AddEnvironmentVariableConfiguration(); return services; } @@ -60,5 +65,23 @@ public static ILoggingBuilder AddPythonLogging(this ILoggingBuilder builder, Fun return builder; } + + /// + /// Adds support for setting configuration values via environment variables. + /// Environment variables start with "MigrationSDK__". + /// + /// + /// + public static IServiceCollection AddEnvironmentVariableConfiguration(this IServiceCollection services) + { + var configBuilder = + new ConfigurationBuilder() + .AddEnvironmentVariables("MigrationSDK__"); + var config = configBuilder.Build(); + + services.Configure(nameof(MigrationSdkOptions), config); + + return services; + } } } diff --git a/src/Tableau.Migration/Net/Handlers/AuthenticationHandler.cs b/src/Tableau.Migration/Net/Handlers/AuthenticationHandler.cs index 33596d0..4c78a32 100644 --- a/src/Tableau.Migration/Net/Handlers/AuthenticationHandler.cs +++ b/src/Tableau.Migration/Net/Handlers/AuthenticationHandler.cs @@ -34,29 +34,34 @@ public AuthenticationHandler(IAuthenticationTokenProvider tokenProvider) protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { - if (request.RequestUri.IsRest() && !request.RequestUri.IsRestSignIn()) + // Send request without auth token for non-REST API or sign in requests. + if (!request.RequestUri.IsRest() || request.RequestUri.IsRestSignIn()) { - // Use the current token - if (_tokenProvider.Token is not null) - request.SetRestAuthenticationToken(_tokenProvider.Token); - - var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); - - if (response.StatusCode == HttpStatusCode.Unauthorized) - { - await _tokenProvider.RequestRefreshAsync(cancellationToken).ConfigureAwait(false); - - // Use the new token - if (_tokenProvider.Token is not null) - request.SetRestAuthenticationToken(_tokenProvider.Token); - } - else - { - return response; - } + return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); } + + // Use the current token. + var requestAuthToken = await _tokenProvider.GetAsync(cancellationToken).ConfigureAwait(false); + if (requestAuthToken is not null) + request.SetRestAuthenticationToken(requestAuthToken); - return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + var response = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + + if (response.StatusCode is not HttpStatusCode.Unauthorized) + { + return response; + } + + // Refresh the authentication token. + await _tokenProvider.RequestRefreshAsync(requestAuthToken, cancellationToken).ConfigureAwait(false); + + // Set the new token for the retry. + var refreshedAuthToken = await _tokenProvider.GetAsync(cancellationToken).ConfigureAwait(false); + if (refreshedAuthToken is not null) + request.SetRestAuthenticationToken(refreshedAuthToken); + + // Re-send a single time, and rely on other resilience to retry more than that. + return await base.SendAsync(request, cancellationToken).ConfigureAwait(false); } } } diff --git a/src/Tableau.Migration/Net/NetworkTraceRedactor.cs b/src/Tableau.Migration/Net/NetworkTraceRedactor.cs index a3ae073..5a76f5e 100644 --- a/src/Tableau.Migration/Net/NetworkTraceRedactor.cs +++ b/src/Tableau.Migration/Net/NetworkTraceRedactor.cs @@ -60,7 +60,7 @@ public string ReplaceSensitiveData(string input) continue; } - input = regex.Replace(input, m => m.Groups["SENSITIVE_VALUE"].Success + input = regex.Replace(input, m => (m.Groups["SENSITIVE_VALUE"].Success && !m.Groups["SENSITIVE_VALUE"].Value.IsNullOrEmpty()) ? m.Value.Replace(m.Groups["SENSITIVE_VALUE"].Value, SENSITIVE_DATA_PLACEHOLDER, StringComparison.Ordinal) : m.Value); } diff --git a/src/Tableau.Migration/Resources/SharedResourceKeys.cs b/src/Tableau.Migration/Resources/SharedResourceKeys.cs index 0fbddf8..dac5753 100644 --- a/src/Tableau.Migration/Resources/SharedResourceKeys.cs +++ b/src/Tableau.Migration/Resources/SharedResourceKeys.cs @@ -71,7 +71,7 @@ internal static class SharedResourceKeys public const string PublishedDataSourceReferenceNotFoundLogMessage = "PublishedDataSourceReferenceNotFoundLogMessage"; public const string FailedJobExceptionContent = "FailedJobExceptionContent"; - + public const string TimeoutJobExceptionMessage = "TimeoutJobExceptionMessage"; public const string ContentFilterBaseDebugMessage = "ContentFilterBaseDebugMessage"; @@ -79,7 +79,15 @@ internal static class SharedResourceKeys public const string ContentMappingBaseDebugMessage = "ContentMappingBaseDebugMessage"; public const string ContentTransformerBaseDebugMessage = "ContentTransformerBaseDebugMessage"; - - public const string GroupUsersTransformerCannotMapWarning = "GroupUsersTransformerCannotMapWarning"; + + public const string GroupUsersTransformerCannotAddUserWarning = "GroupUsersTransformerCannotAddUserWarning"; + + public const string PermissionsTransformerGranteeNotFoundWarning = "PermissionsTransformerGranteeNotFoundWarning"; + + public const string SiteSettingsSkippedDisabledLogMessage = "SiteSettingsSkippedDisabledLogMessage"; + + public const string SiteSettingsSkippedNoAccessLogMessage = "SiteSettingsSkippedNoAccessLogMessage"; + + public const string SiteSettingsExtractEncryptionDisabledLogMessage = "SiteSettingsExtractEncryptionDisabledLogMessage"; } } diff --git a/src/Tableau.Migration/Resources/SharedResources.resx b/src/Tableau.Migration/Resources/SharedResources.resx index 371eb0b..075945c 100644 --- a/src/Tableau.Migration/Resources/SharedResources.resx +++ b/src/Tableau.Migration/Resources/SharedResources.resx @@ -141,11 +141,8 @@ File encryption is disabled and should be re-enabled for production migrations. - - Group {GroupName} cannot map {UserLocation} - - - Group {GroupName} cannot map {UserLocation} + + Could not add a user to the destination Group {GroupName}. Reason: Could not find the destination user for {UserLocation}. Migration transformer for publish type {0} is not supported by the migration pipeline. Did you mean {1}? Count: {2} @@ -157,10 +154,10 @@ An error occurred migrating {ContentType} item "{SourcePath}". Error: {Exception} - HTTP {Method} {RequestUri} failed. Error: "{ErrorMessage}".{Details} + HTTP {Method} {RequestUri} failed. Error: "{ErrorMessage}". Details: {Details} - HTTP {Method} {RequestUri} responded {ResponseStatus}.{Details} + HTTP {Method} {RequestUri} responded {ResponseStatus}. Details: {Details} <Not Displayed> @@ -202,6 +199,15 @@ Detail: {3} # Response Headers + + Destination site settings were not validated - disabled by configuration. + + + Destination site settings were not validated - administrator access is required in both source and destination sites. + + + The destination site has extract encryption disabled. This may cause migration errors for content with encrypted extracts. Enable extract encryption on the destination site, or register a transformer that disables extract encryption during migration. + Could not find destination user to map for source user {SourceUsername} (ID: {SourceUserId}). @@ -220,4 +226,7 @@ Detail: {3} Migration transformer for publish type {0} is not supported by the migration pipeline. Count: {1} + + Could not transform permissions. The {granteeType} for {UserOrGroupName} was not found at the destination. + \ No newline at end of file diff --git a/src/Tableau.Migration/Tableau.Migration.csproj b/src/Tableau.Migration/Tableau.Migration.csproj index e3b5095..8613f2b 100644 --- a/src/Tableau.Migration/Tableau.Migration.csproj +++ b/src/Tableau.Migration/Tableau.Migration.csproj @@ -43,6 +43,7 @@ Note: This SDK is specific for migrating from Tableau Server to Tableau Cloud. I + diff --git a/tests/Python.TestApplication/migration_testcomponents_mappings.py b/tests/Python.TestApplication/migration_testcomponents_mappings.py index 90703c4..f62e409 100644 --- a/tests/Python.TestApplication/migration_testcomponents_mappings.py +++ b/tests/Python.TestApplication/migration_testcomponents_mappings.py @@ -21,14 +21,14 @@ from Tableau.Migration import ContentLocation from Tableau.Migration.Api.Rest.Models import LicenseLevels from Tableau.Migration.Content import IUser -from Tableau.Migration.Interop.Hooks.Mappings import SyncTableauCloudUsernameMapping, ISyncContentMapping +from Tableau.Migration.Interop.Hooks.Mappings import ISyncContentMapping from Tableau.Migration.Engine.Hooks.Mappings.Default import ITableauCloudUsernameMapping -class PyTestTableauCloudUsernameMapping(SyncTableauCloudUsernameMapping): +class PyTestTableauCloudUsernameMapping(ISyncContentMapping[IUser]): """Mapping that takes a base email and appends the source item name to the email username.""" __namespace__ = "Python.TestApplication" - _dotnet_base = ITableauCloudUsernameMapping + _dotnet_base = ISyncContentMapping[IUser] def __init__(self): """Default init to set up logging.""" diff --git a/tests/Tableau.Migration.Tests/Simulation/Tests/Api/ApiClientTests.cs b/tests/Tableau.Migration.Tests/Simulation/Tests/Api/ApiClientTests.cs index 1136986..d423306 100644 --- a/tests/Tableau.Migration.Tests/Simulation/Tests/Api/ApiClientTests.cs +++ b/tests/Tableau.Migration.Tests/Simulation/Tests/Api/ApiClientTests.cs @@ -43,10 +43,11 @@ protected void AssertSession(Action assert) assert(sessionProvider); } - protected void AssertAuthenticationToken(Action assert) + protected async Task AssertAuthenticationTokenAsync(Action assert) { var tokenProvider = ServiceProvider.GetRequiredService(); - assert(tokenProvider); + var token = await tokenProvider.GetAsync(Cancel); + assert(token); } } @@ -116,9 +117,9 @@ public async Task Returns_site_client_on_success() Assert.Equal(signIn.Site.ContentUrl, p.SiteContentUrl); }); - AssertAuthenticationToken(p => + await AssertAuthenticationTokenAsync(token => { - Assert.Equal(signIn.Token, p.Token); + Assert.Equal(signIn.Token, token); }); } @@ -142,9 +143,9 @@ public async Task Returns_error_on_failure() Assert.Null(p.SiteContentUrl); }); - AssertAuthenticationToken(p => + await AssertAuthenticationTokenAsync(token => { - Assert.Null(p.Token); + Assert.Null(token); }); } @@ -178,9 +179,9 @@ public async Task Returns_error_on_invalid_credentials() Assert.Null(p.SiteContentUrl); }); - AssertAuthenticationToken(p => + await AssertAuthenticationTokenAsync(token => { - Assert.Null(p.Token); + Assert.Null(token); }); } } @@ -198,9 +199,9 @@ public async Task Returns_success() Assert.True(result.Success); - AssertAuthenticationToken(p => + await AssertAuthenticationTokenAsync(token => { - Assert.Null(p.Token); + Assert.Null(token); }); } @@ -220,9 +221,9 @@ public async Task Returns_error_on_failure() var error = Assert.Single(result.Errors); Assert.IsType(error); - AssertAuthenticationToken(p => + await AssertAuthenticationTokenAsync(token => { - Assert.Null(p.Token); + Assert.Null(token); }); } } diff --git a/tests/Tableau.Migration.Tests/Unit/Api/ApiClientTests.cs b/tests/Tableau.Migration.Tests/Unit/Api/ApiClientTests.cs index 8d69c54..b156a9c 100644 --- a/tests/Tableau.Migration.Tests/Unit/Api/ApiClientTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Api/ApiClientTests.cs @@ -108,7 +108,7 @@ public async Task Returns_error() request.AssertUri(SiteConnectionConfiguration.ServerUrl, $"/api/{TableauServerVersion.RestApiVersion}/serverinfo"); - MockSessionProvider.Verify(p => p.SetCurrentUserAndSite(It.IsAny()), Times.Never); + MockSessionProvider.Verify(p => p.SetCurrentUserAndSiteAsync(It.IsAny(), Cancel), Times.Never); } } @@ -141,11 +141,11 @@ await request.AssertContentAsync( Assert.Equal(r.Credentials.PersonalAccessTokenSecret, SiteConnectionConfiguration.AccessToken); }); - MockSessionProvider.Verify(p => p.SetCurrentUserAndSite(It.Is(r => + MockSessionProvider.Verify(p => p.SetCurrentUserAndSiteAsync(It.Is(r => r.SiteId == signInResponse.Item!.Site!.Id && r.SiteContentUrl == signInResponse.Item.Site.ContentUrl && r.UserId == signInResponse.Item.User!.Id && - r.Token == signInResponse.Item.Token)), + r.Token == signInResponse.Item.Token), Cancel), Times.Once); } @@ -196,5 +196,40 @@ public async Task Returns_error() request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/auth/signin"); } } + + public class GetCurrentServerSessionAsync : ApiClientTest + { + [Fact] + public async Task ReturnsServerSessionAsync() + { + var sessionResponse = AutoFixture.CreateResponse(); + var mockResponse = new MockHttpResponseMessage(sessionResponse); + MockHttpClient.SetupResponse(mockResponse); + + var result = await ApiClient.GetCurrentServerSessionAsync(Cancel); + + Assert.True(result.Success); + + var request = MockHttpClient.AssertSingleRequest(); + + request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/sessions/current"); + } + + [Fact] + public async Task Returns_error() + { + var sessionResponse = AutoFixture.CreateErrorResponse(); + var mockResponse = new MockHttpResponseMessage(sessionResponse); + MockHttpClient.SetupResponse(mockResponse); + + var result = await ApiClient.GetCurrentServerSessionAsync(Cancel); + + Assert.False(result.Success); + + var request = MockHttpClient.AssertSingleRequest(); + + request.AssertUri(SiteConnectionConfiguration.ServerUrl, $"/api/{TableauServerVersion.RestApiVersion}/sessions/current"); + } + } } } diff --git a/tests/Tableau.Migration.Tests/Unit/Api/AuthenticationTokenProviderTests.cs b/tests/Tableau.Migration.Tests/Unit/Api/AuthenticationTokenProviderTests.cs index f68e9fe..a638c8b 100644 --- a/tests/Tableau.Migration.Tests/Unit/Api/AuthenticationTokenProviderTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Api/AuthenticationTokenProviderTests.cs @@ -14,6 +14,10 @@ // limitations under the License. // +using System.Collections.Immutable; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Threading; using System.Threading.Tasks; using Tableau.Migration.Api; using Xunit; @@ -30,56 +34,90 @@ public abstract class AuthenticationTokenProviderTest : AutoFixtureTestBase public class RequestRefreshAsync : AuthenticationTokenProviderTest { [Fact] - public async Task Handles_null_event() + public async Task HandlesNullEventAsync() { // Does not throw - await Provider.RequestRefreshAsync(default); + await Provider.RequestRefreshAsync(null, default); } [Fact] - public async Task Calls_refresh() + public async Task CallsRefreshAsync() { var count = 0; Provider.RefreshRequestedAsync += _ => { count++; - return Task.CompletedTask; + return Task.FromResult>(Result.Succeeded(Create())); }; - await Provider.RequestRefreshAsync(default); + await Provider.RequestRefreshAsync(null, Cancel); Assert.Equal(1, count); } + + [Fact] + public async Task SingleRefreshWithConcurrentCallsAsync() + { + var refreshCount = 0; + + Provider.RefreshRequestedAsync += _ => + { + refreshCount++; + return Task.FromResult>(Result.Succeeded(Create())); + }; + + var oldToken = Create(); + await Provider.SetAsync(oldToken, Cancel); + + var syncWait = new ManualResetEventSlim(); + + async Task RefreshTokenAsync() + { + syncWait.Wait(Cancel); + + await Provider.RequestRefreshAsync(oldToken, Cancel); + } + + var tasks = Enumerable.Range(0, 20) + .Select(i => Task.Run(RefreshTokenAsync)) + .ToImmutableArray(); + + syncWait.Set(); + + await Task.WhenAll(tasks); + + Assert.Equal(1, refreshCount); + } } public class Set : AuthenticationTokenProviderTest { [Fact] - public void Sets_token() + public async Task SetsTokenAsync() { var token = Create(); - Provider.Set(token); + await Provider.SetAsync(token, Cancel); - Assert.Equal(token, Provider.Token); + Assert.Equal(token, await Provider.GetAsync(Cancel)); } } public class Clear : AuthenticationTokenProviderTest { [Fact] - public void Clears_token() + public async Task ClearsTokenAsync() { var token = Create(); - Provider.Set(token); + await Provider.SetAsync(token, Cancel); - Assert.Equal(token, Provider.Token); + Assert.Equal(token, await Provider.GetAsync(Cancel)); - Provider.Clear(); + await Provider.ClearAsync(Cancel); - Assert.Null(Provider.Token); + Assert.Null(await Provider.GetAsync(Cancel)); } } } diff --git a/tests/Tableau.Migration.Tests/Unit/Api/Rest/Models/Requests/UpdateSiteRequestTests.cs b/tests/Tableau.Migration.Tests/Unit/Api/Rest/Models/Requests/UpdateSiteRequestTests.cs new file mode 100644 index 0000000..b80e19c --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/Api/Rest/Models/Requests/UpdateSiteRequestTests.cs @@ -0,0 +1,39 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using Tableau.Migration.Api.Rest.Models.Requests; +using Tableau.Migration.Content; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.Api.Rest.Models.Requests +{ + public class UpdateSiteRequestTests + { + public class Ctor : AutoFixtureTestBase + { + [Fact] + public void Initializes() + { + var update = Create(); + + var req = new UpdateSiteRequest(update); + + Assert.NotNull(req.Site); + Assert.Equal(update.ExtractEncryptionMode, req.Site.ExtractEncryptionMode); + } + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/Api/ServerSessionProviderTests.cs b/tests/Tableau.Migration.Tests/Unit/Api/ServerSessionProviderTests.cs index 55a49d5..7578409 100644 --- a/tests/Tableau.Migration.Tests/Unit/Api/ServerSessionProviderTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Api/ServerSessionProviderTests.cs @@ -15,6 +15,7 @@ // using System; +using System.Threading.Tasks; using Moq; using Tableau.Migration.Api; using Tableau.Migration.Api.Models; @@ -37,16 +38,16 @@ public ServerSessionProviderTest() } } - public class AuthenticationToken : ServerSessionProviderTest + public class GetAuthenticationTokenAsync : ServerSessionProviderTest { [Fact] - public void Returns_token() + public async Task GetsTokenAsync() { var token = Create(); - MockTokenProvider.SetupGet(p => p.Token).Returns(token); + MockTokenProvider.Setup(p => p.GetAsync(Cancel)).ReturnsAsync(token); - Assert.Equal(token, Provider.AuthenticationToken); + Assert.Equal(token, await Provider.GetAuthenticationTokenAsync(Cancel)); } } @@ -63,62 +64,56 @@ public void Returns_version() } } - public class SetCurrentUserAndSite + public class SetCurrentUserAndSiteAsync : ServerSessionProviderTest { - public class ISignInResult_Overload : ServerSessionProviderTest + [Fact] + public async Task SetsUserAndSiteWithSignInResultAsync() { - [Fact] - public void Sets_user_and_site() - { - var signInResult = Create(); + var signInResult = Create(); - Provider.SetCurrentUserAndSite(signInResult); + await Provider.SetCurrentUserAndSiteAsync(signInResult, Cancel); - Assert.Equal(signInResult.UserId, Provider.UserId); - Assert.Equal(signInResult.SiteContentUrl, Provider.SiteContentUrl); - Assert.Equal(signInResult.SiteId, Provider.SiteId); + Assert.Equal(signInResult.UserId, Provider.UserId); + Assert.Equal(signInResult.SiteContentUrl, Provider.SiteContentUrl); + Assert.Equal(signInResult.SiteId, Provider.SiteId); - MockTokenProvider.Verify(p => p.Set(signInResult.Token), Times.Once); - } + MockTokenProvider.Verify(p => p.SetAsync(signInResult.Token, Cancel), Times.Once); } - public class Values_Overload : ServerSessionProviderTest + [Fact] + public async Task SetsUserAndSiteFromValuesAsync() { - [Fact] - public void Sets_user_and_site() - { - var userId = Create(); - var siteContentUrl = Create(); - var siteId = Create(); - var token = Create(); - - Provider.SetCurrentUserAndSite(userId, siteId, siteContentUrl, token); - - Assert.Equal(userId, Provider.UserId); - Assert.Equal(siteContentUrl, Provider.SiteContentUrl); - Assert.Equal(siteId, Provider.SiteId); - - MockTokenProvider.Verify(p => p.Set(token), Times.Once); - } + var userId = Create(); + var siteContentUrl = Create(); + var siteId = Create(); + var token = Create(); + + await Provider.SetCurrentUserAndSiteAsync(userId, siteId, siteContentUrl, token, Cancel); + + Assert.Equal(userId, Provider.UserId); + Assert.Equal(siteContentUrl, Provider.SiteContentUrl); + Assert.Equal(siteId, Provider.SiteId); + + MockTokenProvider.Verify(p => p.SetAsync(token, Cancel), Times.Once); } } public class ClearCurrentUserAndSite : ServerSessionProviderTest { [Fact] - public void Clears_user() + public async Task ClearsUserAsync() { var signInResult = Create(); - Provider.SetCurrentUserAndSite(signInResult); + await Provider.SetCurrentUserAndSiteAsync(signInResult, Cancel); - Provider.ClearCurrentUserAndSite(); + await Provider.ClearCurrentUserAndSiteAsync(Cancel); Assert.Null(Provider.UserId); Assert.Null(Provider.SiteContentUrl); Assert.Null(Provider.SiteId); - MockTokenProvider.Verify(p => p.Clear(), Times.Once); + MockTokenProvider.Verify(p => p.ClearAsync(Cancel), Times.Once); } } diff --git a/tests/Tableau.Migration.Tests/Unit/Api/SitesApiClientTests.cs b/tests/Tableau.Migration.Tests/Unit/Api/SitesApiClientTests.cs index eaca4d3..f9e23c6 100644 --- a/tests/Tableau.Migration.Tests/Unit/Api/SitesApiClientTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Api/SitesApiClientTests.cs @@ -22,6 +22,7 @@ using Moq; using Tableau.Migration.Api; using Tableau.Migration.Api.Rest.Models.Responses; +using Tableau.Migration.Content; using Xunit; namespace Tableau.Migration.Tests.Unit.Api @@ -213,7 +214,7 @@ public async Task Returns_success() request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/auth/signout"); - MockSessionProvider.Verify(p => p.ClearCurrentUserAndSite(), Times.Once); + MockSessionProvider.Verify(p => p.ClearCurrentUserAndSiteAsync(Cancel), Times.Once); } [Fact] @@ -236,7 +237,7 @@ public async Task Returns_failure() request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/auth/signout"); - MockSessionProvider.Verify(p => p.ClearCurrentUserAndSite(), Times.Once); + MockSessionProvider.Verify(p => p.ClearCurrentUserAndSiteAsync(Cancel), Times.Once); } [Fact] @@ -291,5 +292,73 @@ public async Task Catches_errors() } #endregion + + #region - UpdateSiteAsync - + + public class UpdateSiteAsync : SitesApiClientTest + { + [Fact] + public async Task ErrorAsync() + { + var exception = new Exception(); + + var mockResponse = new MockHttpResponseMessage(HttpStatusCode.InternalServerError, null); + mockResponse.Setup(r => r.EnsureSuccessStatusCode()).Throws(exception); + + MockHttpClient.SetupResponse(mockResponse); + + var update = new SiteSettingsUpdate(Guid.NewGuid()); + var result = await SitesApiClient.UpdateSiteAsync(update, Cancel); + + result.AssertFailure(); + + var resultError = Assert.Single(result.Errors); + Assert.Same(exception, resultError); + + var request = MockHttpClient.AssertSingleRequest(); + request.AssertHttpMethod(HttpMethod.Put); + request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/sites/{update.SiteId}"); + } + + [Fact] + public async Task FailureResponseAsync() + { + var mockResponse = new MockHttpResponseMessage(HttpStatusCode.NotFound, null); + MockHttpClient.SetupResponse(mockResponse); + + var update = new SiteSettingsUpdate(Guid.NewGuid()); + var result = await SitesApiClient.UpdateSiteAsync(update, Cancel); + + result.AssertFailure(); + + Assert.Null(result.Value); + Assert.Single(result.Errors); + + var request = MockHttpClient.AssertSingleRequest(); + request.AssertHttpMethod(HttpMethod.Put); + request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/sites/{update.SiteId}"); + } + + [Fact] + public async Task SuccessAsync() + { + var siteResponse = AutoFixture.CreateResponse(); + + var mockResponse = new MockHttpResponseMessage(siteResponse); + MockHttpClient.SetupResponse(mockResponse); + + var update = new SiteSettingsUpdate(Guid.NewGuid()); + var result = await SitesApiClient.UpdateSiteAsync(update, Cancel); + + result.AssertSuccess(); + Assert.NotNull(result.Value); + + var request = MockHttpClient.AssertSingleRequest(); + request.AssertHttpMethod(HttpMethod.Put); + request.AssertRelativeUri($"/api/{TableauServerVersion.RestApiVersion}/sites/{update.SiteId}"); + } + } + + #endregion } } diff --git a/tests/Tableau.Migration.Tests/Unit/AsyncEventHandlerTests.cs b/tests/Tableau.Migration.Tests/Unit/Config/PreflightOptionsTests.cs similarity index 52% rename from tests/Tableau.Migration.Tests/Unit/AsyncEventHandlerTests.cs rename to tests/Tableau.Migration.Tests/Unit/Config/PreflightOptionsTests.cs index 5211f43..9a875b3 100644 --- a/tests/Tableau.Migration.Tests/Unit/AsyncEventHandlerTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Config/PreflightOptionsTests.cs @@ -14,40 +14,31 @@ // limitations under the License. // -using System.Threading.Tasks; +using Tableau.Migration.Config; using Xunit; -namespace Tableau.Migration.Tests.Unit +namespace Tableau.Migration.Tests.Unit.Config { - public class AsyncEventHandlerTests + public class PreflightOptionsTests { - private class TestClass + public class ValidateSettings { - public event AsyncEventHandler? EventTriggered; - - public async void TriggerEvent() + [Fact] + public void FallsBackToDefault() { - if (EventTriggered is not null) - await EventTriggered.Invoke(default); + var opts = new PreflightOptions(); + Assert.Equal(PreflightOptions.Defaults.VALIDATE_SETTINGS, opts.ValidateSettings); } - } - - [Fact] - public void Calls_handler() - { - var obj = new TestClass(); - var count = 0; - - obj.EventTriggered += async (c) => + [Fact] + public void CustomizedValue() { - count++; - await Task.CompletedTask; - }; - - obj.TriggerEvent(); - - Assert.Equal(1, count); + var opts = new PreflightOptions + { + ValidateSettings = false + }; + Assert.False(opts.ValidateSettings); + } } } } diff --git a/tests/Tableau.Migration.Tests/Unit/Content/ServerSessionSettingsTests.cs b/tests/Tableau.Migration.Tests/Unit/Content/ServerSessionSettingsTests.cs new file mode 100644 index 0000000..7ee6621 --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/Content/ServerSessionSettingsTests.cs @@ -0,0 +1,40 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using Tableau.Migration.Api.Rest.Models.Responses; +using Tableau.Migration.Content; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.Content +{ + public class ServerSessionSettingsTests + { + public class Ctor + { + [Theory] + [NullEmptyWhiteSpaceData] + public void RequiresExtractEncryptionMode(string? s) + { + var response = new ServerSessionResponse.SessionType.SiteType() + { + ExtractEncryptionMode = s + }; + Assert.Throws(() => new ServerSessionSettings(response)); + } + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/Content/ServerSessionTests.cs b/tests/Tableau.Migration.Tests/Unit/Content/ServerSessionTests.cs new file mode 100644 index 0000000..19b0e03 --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/Content/ServerSessionTests.cs @@ -0,0 +1,110 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using Tableau.Migration.Api.Rest.Models; +using Tableau.Migration.Api.Rest.Models.Responses; +using Tableau.Migration.Content; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.Content +{ + public class ServerSessionTests + { + public class Ctor : AutoFixtureTestBase + { + [Fact] + public void RequiresResponseItem() + { + var req = new ServerSessionResponse(); + + Assert.Throws(() => new ServerSession(req)); + } + + [Fact] + public void RequiresSite() + { + var req = new ServerSessionResponse() + { + Item = new() + }; + + Assert.Throws(() => new ServerSession(req)); + } + + [Fact] + public void RequiresSiteId() + { + var req = Create(); + req.Item!.Site!.Id = Guid.Empty; + + Assert.Throws(() => new ServerSession(req)); + } + + [Fact] + public void RequiresSiteContentUrl() + { + var req = Create(); + req.Item!.Site!.ContentUrl = null; + + Assert.Throws(() => new ServerSession(req)); + } + + [Theory] + [NullEmptyWhiteSpaceData] + public void RequiresSiteName(string? s) + { + var req = Create(); + req.Item!.Site!.Name = s; + + Assert.Throws(() => new ServerSession(req)); + } + + [Fact] + public void RequiresUserItem() + { + var req = Create(); + req.Item!.User = null; + + Assert.Throws(() => new ServerSession(req)); + } + + [Fact] + public void NonAdministrator() + { + var req = Create(); + req.Item!.User!.SiteRole = SiteRoles.ExplorerCanPublish; + + var s = new ServerSession(req); + + Assert.False(s.IsAdministrator); + Assert.Null(s.Settings); + } + + [Fact] + public void Administrator() + { + var req = Create(); + req.Item!.User!.SiteRole = SiteRoles.SiteAdministratorExplorer; + + var s = new ServerSession(req); + + Assert.True(s.IsAdministrator); + Assert.NotNull(s.Settings); + } + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/Content/SiteSettingUpdateTests.cs b/tests/Tableau.Migration.Tests/Unit/Content/SiteSettingUpdateTests.cs new file mode 100644 index 0000000..824b89c --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/Content/SiteSettingUpdateTests.cs @@ -0,0 +1,59 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using Tableau.Migration.Content; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.Content +{ + public class SiteSettingUpdateTests + { + public class Ctor + { + [Fact] + public void Initializes() + { + var id = Guid.NewGuid(); + var u = new SiteSettingsUpdate(id); + + Assert.Equal(id, u.SiteId); + } + } + + public class NeedsUpdate : AutoFixtureTestBase + { + [Fact] + public void NoUpdates() + { + var u = new SiteSettingsUpdate(Guid.NewGuid()); + + Assert.False(u.NeedsUpdate()); + } + + [Fact] + public void ExtractEncryptionModeUpdated() + { + var u = new SiteSettingsUpdate(Guid.NewGuid()) + { + ExtractEncryptionMode = Create() + }; + + Assert.True(u.NeedsUpdate()); + } + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/Content/SiteTests.cs b/tests/Tableau.Migration.Tests/Unit/Content/SiteTests.cs index 0c34bbb..2d29c89 100644 --- a/tests/Tableau.Migration.Tests/Unit/Content/SiteTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Content/SiteTests.cs @@ -15,6 +15,7 @@ // using System; +using Tableau.Migration.Api.Rest.Models; using Tableau.Migration.Api.Rest.Models.Responses; using Tableau.Migration.Content; using Xunit; @@ -33,7 +34,8 @@ protected SiteResponse CreateTestResponse() { Id = Create(), Name = Create(), - ContentUrl = Create() + ContentUrl = Create(), + ExtractEncryptionMode = ExtractEncryptionModes.Disabled } }; } @@ -93,6 +95,16 @@ public void ContentUrlShouldNotBeNull(string? contentUrl) var site = new Site(response); } } + + [Theory] + [NullEmptyWhiteSpaceData] + public void ExtractEncryptionModeRequired(string? s) + { + var response = CreateTestResponse(); + response.Item!.ExtractEncryptionMode = s; + + Assert.Throws(() => new Site(response)); + } } } } diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Actions/PreflightActionTests.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Actions/PreflightActionTests.cs new file mode 100644 index 0000000..5a23f0e --- /dev/null +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Actions/PreflightActionTests.cs @@ -0,0 +1,272 @@ +// Copyright (c) 2023, Salesforce, Inc. +// SPDX-License-Identifier: Apache-2 +// +// Licensed under the Apache License, Version 2.0 (the ""License"") +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an ""AS IS"" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +using System; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Moq; +using Tableau.Migration.Api.Rest.Models; +using Tableau.Migration.Config; +using Tableau.Migration.Content; +using Tableau.Migration.Engine.Actions; +using Tableau.Migration.Engine.Endpoints; +using Xunit; + +namespace Tableau.Migration.Tests.Unit.Engine.Actions +{ + public class PreflightActionTests + { + public class ExecuteAsync : AutoFixtureTestBase + { + protected PreflightOptions Options { get; } + + protected Mock MockSourceSession { get; set; } + + protected Func> SourceSessionResult { get; set; } + + protected Mock MockSource { get; set; } + + protected Mock MockDestinationSession { get; set; } + + protected Func> DestinationSessionResult { get; set; } + + protected Mock MockDestination { get; set; } + + protected IResult UpdateResult { get; set; } + + protected Mock> MockLogger { get; } + + public ExecuteAsync() + { + Options = Freeze(); + Options.ValidateSettings = PreflightOptions.Defaults.VALIDATE_SETTINGS; + + MockSourceSession = Create>(); + MockSourceSession.SetupGet(x => x.IsAdministrator).Returns(true); + + SourceSessionResult = () => Result.Succeeded(MockSourceSession.Object); + + MockSource = Freeze>(); + MockSource.Setup(x => x.GetSessionAsync(Cancel)).ReturnsAsync(() => SourceSessionResult()); + + MockDestinationSession = Create>(); + MockDestinationSession.SetupGet(x => x.IsAdministrator).Returns(true); + + DestinationSessionResult = () => Result.Succeeded(MockDestinationSession.Object); + + MockDestination = Freeze>(); + MockDestination.Setup(x => x.GetSessionAsync(Cancel)).ReturnsAsync(() => DestinationSessionResult()); + + UpdateResult = Result.Succeeded(Create()); + MockDestination.Setup(x => x.UpdateSiteSettingsAsync(It.IsAny(), Cancel)) + .ReturnsAsync(() => UpdateResult); + + MockLogger = Freeze>>(); + } + + [Fact] + public async Task SettingValidationDisabledAsync() + { + Options.ValidateSettings = false; + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertSuccess(); + Assert.True(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Never); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Never); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task SourceSessionFailedAsync() + { + SourceSessionResult = () => Result.Failed(new Exception()); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertFailure(); + Assert.False(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task DestinationSessionFailedAsync() + { + DestinationSessionResult = () => Result.Failed(new Exception()); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertFailure(); + Assert.False(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task CombinesEndpointErrorsAsync() + { + var sourceEx = new Exception(); + var destinationEx = new Exception(); + + SourceSessionResult = () => Result.Failed(sourceEx); + DestinationSessionResult = () => Result.Failed(destinationEx); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertFailure(); + Assert.False(result.PerformNextAction); + + Assert.Contains(sourceEx, result.Errors); + Assert.Contains(destinationEx, result.Errors); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task NotSourceAdminAsync() + { + MockSourceSession.SetupGet(x => x.IsAdministrator).Returns(false); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertSuccess(); + Assert.True(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task NotDestinationAdminAsync() + { + MockDestinationSession.SetupGet(x => x.IsAdministrator).Returns(false); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertSuccess(); + Assert.True(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task ValidSameSettingAsync() + { + MockSourceSession.SetupGet(x => x.Settings!.ExtractEncryptionMode).Returns(ExtractEncryptionModes.Enabled); + MockDestinationSession.SetupGet(x => x.Settings!.ExtractEncryptionMode).Returns(ExtractEncryptionModes.Enabled); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertSuccess(); + Assert.True(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task ExtractEncryptionValidSourceDisabledAsync() + { + MockSourceSession.SetupGet(x => x.Settings!.ExtractEncryptionMode).Returns(ExtractEncryptionModes.Disabled); + MockDestinationSession.SetupGet(x => x.Settings!.ExtractEncryptionMode).Returns(ExtractEncryptionModes.Enabled); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertSuccess(); + Assert.True(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task ExtractEncryptionValidCompatibleAsync() + { + MockSourceSession.SetupGet(x => x.Settings!.ExtractEncryptionMode).Returns(ExtractEncryptionModes.Enforced); + MockDestinationSession.SetupGet(x => x.Settings!.ExtractEncryptionMode).Returns(ExtractEncryptionModes.Enabled); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertSuccess(); + Assert.True(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Never); + } + + [Fact] + public async Task InvalidExtractEncryptionModeWarnsAsync() + { + MockSourceSession.SetupGet(x => x.Settings!.ExtractEncryptionMode).Returns(ExtractEncryptionModes.Enforced); + MockDestinationSession.SetupGet(x => x.Settings!.ExtractEncryptionMode).Returns(ExtractEncryptionModes.Disabled); + + var action = Create(); + + var result = await action.ExecuteAsync(Cancel); + + result.AssertSuccess(); + Assert.True(result.PerformNextAction); + + MockSource.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + MockDestination.Verify(x => x.GetSessionAsync(Cancel), Times.Once); + + MockLogger.VerifyWarnings(Times.Once); + } + } + } +} diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Endpoints/TableauApiDestinationEndpointTests.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Endpoints/TableauApiDestinationEndpointTests.cs index e79b2eb..3bed22e 100644 --- a/tests/Tableau.Migration.Tests/Unit/Engine/Endpoints/TableauApiDestinationEndpointTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Endpoints/TableauApiDestinationEndpointTests.cs @@ -14,10 +14,12 @@ // limitations under the License. // +using System.Net; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Moq; using Tableau.Migration.Api; +using Tableau.Migration.Content; using Tableau.Migration.Content.Files; using Tableau.Migration.Engine.Endpoints; using Tableau.Migration.Engine.Endpoints.Search; @@ -104,5 +106,29 @@ public async Task UsesBatchPublishClientAsync() } #endregion + + #region - UpdateSiteSettingsAsync - + + public class UpdateSiteSettingsAsync : TableauApiDestinationEndpointTest + { + [Fact] + public async Task UpdatesSiteSettingsAsync() + { + await Endpoint.InitializeAsync(Cancel); + + var update = Create(); + var apiResult = Create>(); + + MockSiteApi.Setup(x => x.UpdateSiteAsync(update, Cancel)) + .ReturnsAsync(apiResult); + + var result = await Endpoint.UpdateSiteSettingsAsync(update, Cancel); + + Assert.Same(apiResult, result); + MockSiteApi.Verify(x => x.UpdateSiteAsync(update, Cancel), Times.Once); + } + } + + #endregion } } diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Endpoints/TableauApiEndpointBaseTests.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Endpoints/TableauApiEndpointBaseTests.cs index 390a635..5d237dc 100644 --- a/tests/Tableau.Migration.Tests/Unit/Engine/Endpoints/TableauApiEndpointBaseTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Endpoints/TableauApiEndpointBaseTests.cs @@ -20,6 +20,7 @@ using Microsoft.Extensions.DependencyInjection; using Moq; using Tableau.Migration.Api; +using Tableau.Migration.Content; using Tableau.Migration.Content.Files; using Tableau.Migration.Content.Search; using Tableau.Migration.Engine; @@ -201,5 +202,29 @@ public async Task GetsPager() } #endregion + + #region - GetSessionAsync - + + public class GetSessionAsync : TableauApiEndpointBaseTest + { + [Fact] + public async Task GetsSessionAsync() + { + await Endpoint.InitializeAsync(Cancel); + + var session = Create(); + var apiResult = Result.Succeeded(session); + + MockServerApi.Setup(x => x.GetCurrentServerSessionAsync(Cancel)) + .ReturnsAsync(apiResult); + + var result = await Endpoint.GetSessionAsync(Cancel); + + Assert.Same(apiResult, result); + MockServerApi.Verify(x => x.GetCurrentServerSessionAsync(Cancel), Times.Once); + } + } + + #endregion } } diff --git a/tests/Tableau.Migration.Tests/Unit/Engine/Hooks/Transformers/Default/PermissionsTransformerTests.cs b/tests/Tableau.Migration.Tests/Unit/Engine/Hooks/Transformers/Default/PermissionsTransformerTests.cs index 3367251..06cd5f4 100644 --- a/tests/Tableau.Migration.Tests/Unit/Engine/Hooks/Transformers/Default/PermissionsTransformerTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Engine/Hooks/Transformers/Default/PermissionsTransformerTests.cs @@ -20,6 +20,7 @@ using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; using Moq; using Tableau.Migration.Api.Rest.Models; using Tableau.Migration.Api.Rest.Models.Types; @@ -28,6 +29,7 @@ using Tableau.Migration.Engine.Endpoints.Search; using Tableau.Migration.Engine.Hooks.Transformers.Default; using Tableau.Migration.Engine.Pipelines; +using Tableau.Migration.Resources; using Tableau.Migration.Tests.Content.Permissions; using Xunit; @@ -45,12 +47,15 @@ public abstract class PermissionsTransformerTest : AutoFixtureTestBase protected readonly PermissionsTransformer Transformer; + protected readonly Mock> MockLogger = new(); + protected readonly MockSharedResourcesLocalizer MockLocalizer = new(); + public PermissionsTransformerTest() { MockMigrationPipeline.Setup(p => p.CreateDestinationFinder()).Returns(MockUserContentFinder.Object); MockMigrationPipeline.Setup(p => p.CreateDestinationFinder()).Returns(MockGroupContentFinder.Object); - Transformer = new(MockMigrationPipeline.Object); + Transformer = new(MockMigrationPipeline.Object, MockLogger.Object, MockLocalizer.Object); } } @@ -247,6 +252,8 @@ public async Task Excludes_unfound_source_grantees() Assert.NotNull(result); Assert.DoesNotContain(unfoundGranteeCapability, result); + + MockLocalizer.Verify(x => x[SharedResourceKeys.PermissionsTransformerGranteeNotFoundWarning], Times.Once); } [Fact] diff --git a/tests/Tableau.Migration.Tests/Unit/Net/Handlers/AuthenticationHandlerTests.cs b/tests/Tableau.Migration.Tests/Unit/Net/Handlers/AuthenticationHandlerTests.cs index 703d31c..b37dfe1 100644 --- a/tests/Tableau.Migration.Tests/Unit/Net/Handlers/AuthenticationHandlerTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Net/Handlers/AuthenticationHandlerTests.cs @@ -95,7 +95,7 @@ public async Task Sets_token() { var token = Create(); - MockTokenProvider.SetupGet(p => p.Token).Returns(token); + MockTokenProvider.Setup(p => p.GetAsync(Cancel)).ReturnsAsync(token); var mockHandler = CreateMockDelegatingHandler( HttpStatusCode.OK, @@ -110,7 +110,7 @@ public async Task Sets_token() await invoker.SendAsync(new HttpRequestMessage(HttpMethod.Get, "http://localhost/api/3.20/test"), Cancel); - MockTokenProvider.Verify(p => p.RequestRefreshAsync(It.IsAny()), Times.Never()); + MockTokenProvider.Verify(p => p.RequestRefreshAsync(It.IsAny(), It.IsAny()), Times.Never()); } [Fact] @@ -118,7 +118,7 @@ public async Task Overwrites_token_with_provider_value() { var token = Create(); - MockTokenProvider.SetupGet(p => p.Token).Returns(token); + MockTokenProvider.Setup(p => p.GetAsync(Cancel)).ReturnsAsync(token); var mockHandler = CreateMockDelegatingHandler( HttpStatusCode.OK, @@ -143,11 +143,11 @@ public async Task Refreshes_token() var token1 = Create(); var token2 = Create(); - MockTokenProvider.SetupGet(p => p.Token).Returns(token1); + MockTokenProvider.Setup(p => p.GetAsync(Cancel)).ReturnsAsync(token1); MockTokenProvider - .Setup(p => p.RequestRefreshAsync(Cancel)) - .Callback(() => MockTokenProvider.SetupGet(p => p.Token).Returns(token2)); + .Setup(p => p.RequestRefreshAsync(It.IsAny(), Cancel)) + .Callback(() => MockTokenProvider.Setup(p => p.GetAsync(Cancel)).ReturnsAsync(token2)); var mockHandler = CreateMockDelegatingHandler( HttpStatusCode.Unauthorized, @@ -162,7 +162,7 @@ public async Task Refreshes_token() await invoker.SendAsync(new HttpRequestMessage(HttpMethod.Get, "http://localhost/api/3.20/test"), Cancel); - MockTokenProvider.Verify(p => p.RequestRefreshAsync(It.IsAny()), Times.Once()); + MockTokenProvider.Verify(p => p.RequestRefreshAsync(It.IsAny(), It.IsAny()), Times.Once()); } [Fact] diff --git a/tests/Tableau.Migration.Tests/Unit/Net/NetworkTraceRedactorTests.cs b/tests/Tableau.Migration.Tests/Unit/Net/NetworkTraceRedactorTests.cs index a99762a..5dc8d34 100644 --- a/tests/Tableau.Migration.Tests/Unit/Net/NetworkTraceRedactorTests.cs +++ b/tests/Tableau.Migration.Tests/Unit/Net/NetworkTraceRedactorTests.cs @@ -80,15 +80,19 @@ public void IncorrectElementTags( [InlineData("password", "test")] [InlineData("password", "te"st")] [InlineData("password", "te'st")] + [InlineData("password", "")] [InlineData("token", "test")] [InlineData("token", "te"st")] [InlineData("token", "te'st")] + [InlineData("token", "")] [InlineData("jwt", "test")] [InlineData("jwt", "te"st")] [InlineData("jwt", "te'st")] + [InlineData("jwt", "")] [InlineData("personalAccessTokenSecret", "test")] [InlineData("personalAccessTokenSecret", "te"st")] [InlineData("personalAccessTokenSecret", "te'st")] + [InlineData("personalAccessTokenSecret", "")] public void ReplaceXmlAttributes( string attributeName, string secret) @@ -102,12 +106,15 @@ public void ReplaceXmlAttributes( [InlineData("authenticity_token", "test")] [InlineData("authenticity_token", "te\\\"st")] [InlineData("authenticity_token", "te\\'st")] + [InlineData("authenticity_token", "")] [InlineData("modulus", "test")] [InlineData("modulus", "te\\\"st")] [InlineData("modulus", "te\\'st")] + [InlineData("modulus", "")] [InlineData("exponent", "test")] [InlineData("exponent", "te\\\"st")] [InlineData("exponent", "te\\'st")] + [InlineData("exponent", "")] public void ReplaceXmlElement( string elementName, string secret) @@ -121,6 +128,7 @@ public void ReplaceXmlElement( [InlineData("test")] [InlineData("te"st")] [InlineData("te'st")] + [InlineData("")] public void ReplaceMultipleSecrets(string secret) { TestValue( @@ -142,10 +150,15 @@ private void TestValue( string sensitiveValue) { // Arrange - var expectedResult = input.Replace( - sensitiveValue, - NetworkTraceRedactor.SENSITIVE_DATA_PLACEHOLDER, - StringComparison.Ordinal); + var expectedResult = input; + + if (!sensitiveValue.IsNullOrEmpty()) + { + expectedResult = input.Replace( + sensitiveValue, + NetworkTraceRedactor.SENSITIVE_DATA_PLACEHOLDER, + StringComparison.Ordinal); + } // Act var result = _networkTraceRedactor.ReplaceSensitiveData(input);