Release 4.1.0

tableau · Jun 28, 2024 · ece3695 · ece3695
1 parent 353414e
commit ece3695
Show file tree

Hide file tree

Showing 259 changed files with 2,600 additions and 1,747 deletions.
diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json
@@ -3,7 +3,7 @@
   "isRoot": true,
   "tools": {
     "docfx": {
-      "version": "2.75.3",
+      "version": "2.76.0",
       "commands": [
         "docfx"
       ]

diff --git a/.github/workflows/dotnet-test.yml b/.github/workflows/dotnet-test.yml
@@ -26,7 +26,7 @@ jobs:
       run: dotnet build '${{ vars.BUILD_SOLUTION }}' -c ${{ matrix.config }}
     - name: Test solution with ${{ matrix.config }} configuration
       run: |
-        dotnet test '${{ vars.BUILD_SOLUTION }}' --no-build -c ${{ matrix.config }} --verbosity normal --logger trx --results-directory "TestResults-${{ matrix.os }}-${{ matrix.config }}" -- RunConfiguration.TestSessionTimeout=${{ vars.MIGRATIONSDK_TEST_CANCELLATION_TIMEOUT_MILLISECONDS }}
+        dotnet test '${{ vars.BUILD_SOLUTION }}' --no-build -c ${{ matrix.config }} --verbosity normal --logger junit --results-directory "TestResults-${{ matrix.os }}-${{ matrix.config }}" -- RunConfiguration.TestSessionTimeout=${{ vars.MIGRATIONSDK_TEST_CANCELLATION_TIMEOUT_MILLISECONDS }}
     - name: Upload test results
       # Use always() to always run this step to publish test results when there are test failures
       if: ${{ always() }}
@@ -35,3 +35,9 @@ jobs:
         name: dotnet-results-${{ matrix.os }}-${{ matrix.config }}
         path: TestResults-${{ matrix.os }}-${{ matrix.config }}
         if-no-files-found: error
+    - name: Publish Unit Test Results
+      uses: test-summary/action@v2
+      with:
+        paths: TestResults-${{ matrix.os }}-${{ matrix.config }}/*.xml
+        show: "fail, skip"
+
diff --git a/.github/workflows/publishdocs-dryrun.yml b/.github/workflows/publishdocs-dryrun.yml
@@ -23,7 +23,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: ./.github/actions/setup-dotnet
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
     - name: Generate API Reference Docs..

diff --git a/.github/workflows/publishdocs.yml b/.github/workflows/publishdocs.yml
@@ -37,7 +37,7 @@ jobs:
     - uses: actions/checkout@v4
     - uses: ./.github/actions/setup-dotnet
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ inputs.python-version }}
     - name: Generate API Reference Docs..

diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -43,7 +43,7 @@ jobs:
     - uses: ./.github/actions/setup-dotnet
       if: ${{ env.PUBLISH_PACKAGE_KEY != '' && inputs.runs-on-config != 'self-hosted' }}
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       if: ${{ env.PUBLISH_PACKAGE_KEY != '' && inputs.runs-on-config != 'self-hosted' }}
       with:
         python-version: ${{ vars.PYTHON_TEST_VERSIONS }}

diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml
@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-dotnet
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ vars.PYTHON_TEST_VERSIONS }}
           cache: 'pip' # caching pip dependencies

diff --git a/Directory.Build.props b/Directory.Build.props
@@ -4,7 +4,7 @@
 		<Nullable>enable</Nullable>
 		<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
 		<EnforceCodeStyleInBuild>true</EnforceCodeStyleInBuild>
-		<Version>4.0.0</Version>
+		<Version>4.1.0</Version>
 		<Authors>Salesforce, Inc.</Authors>
 		<Company>Salesforce, Inc.</Company>
 		<Copyright>Copyright (c) 2024, Salesforce, Inc. and its licensors</Copyright>

diff --git a/Migration SDK.sln b/Migration SDK.sln
@@ -52,6 +52,8 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "scripts", "scripts", "{92044843-B4D7-4062-B1D5-DE7596072E33}"
 	ProjectSection(SolutionItems) = preProject
 		scripts\generate-docs.ps1 = scripts\generate-docs.ps1
+		scripts\sync-release-settings.json = scripts\sync-release-settings.json
+		scripts\sync-release.ps1 = scripts\sync-release.ps1
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DependencyInjection.ExampleApplication", "examples\DependencyInjection.ExampleApplication\DependencyInjection.ExampleApplication.csproj", "{99DA12FB-BB16-4EE1-9C9C-047755210255}"

diff --git a/README.md b/README.md
@@ -11,11 +11,11 @@ The Tableau Migration SDK is a client library to build an application to facilit
 Quickstart:
 - Install a [.NET Runtime](https://dotnet.microsoft.com/en-us/download).
 - For Python install using PIP:
-   - [PIP CLI](https://pip.pypa.io/en/stable/cli/pip_install): `pip install tableau-migration`
+   - [PIP CLI](https://pip.pypa.io/en/stable/cli/pip_install): `pip install tableau_migration`
 - For C# install using NuGet:
-   - [dotnet CLI](https://learn.microsoft.com/en-us/nuget/quickstart/install-and-use-a-package-using-the-dotnet-cli): `dotnet add package Tableau.Migration --version 1.0.0`
+   - [dotnet CLI](https://learn.microsoft.com/en-us/nuget/quickstart/install-and-use-a-package-using-the-dotnet-cli): `dotnet add package Tableau.Migration`
    - [Nuget Package Manager](https://learn.microsoft.com/en-us/nuget/quickstart/install-and-use-a-package-in-visual-studio): Search for `Tableau.Migration`.
-- Then check out our [code samples](https://tableau.github.io/migration-sdk/samples/intro.html).
+- Then check out our [code samples](https://tableau.github.io/migration-sdk/samples/index.html).
 
 To look at source code and delve deeper, first clone the repo:
 

diff --git a/examples/Csharp.ExampleApplication/Csharp.ExampleApplication.csproj b/examples/Csharp.ExampleApplication/Csharp.ExampleApplication.csproj
@@ -4,7 +4,7 @@
 		<TargetFrameworks>net6.0;net8.0</TargetFrameworks>
 		<!-- Don't warn on ConfigureAwait on console application, Don't require license headers for sample code -->
 		<NoWarn>CA2007,IDE0073</NoWarn>
-		<UserSecretsId>7d7631f1-dc4a-49de-89d5-a194544705c1</UserSecretsId>
+		<UserSecretsId>8368baab-103b-45f6-bfb1-f89a537f4f3c</UserSecretsId>
 	</PropertyGroup>
 	<ItemGroup>
 		<ProjectReference Include="../../src/Tableau.Migration/Tableau.Migration.csproj" />		

diff --git a/examples/Csharp.ExampleApplication/Hooks/Transformers/EncryptExtractTransformer.cs b/examples/Csharp.ExampleApplication/Hooks/Transformers/EncryptExtractTransformer.cs
@@ -15,7 +15,7 @@ public class EncryptExtractsTransformer<T> : ContentTransformerBase<T> where T :
     {
         private readonly ILogger<IContentTransformer<T>>? _logger;
 
-        public EncryptExtractsTransformer(ISharedResourcesLocalizer? localizer, ILogger<IContentTransformer<T>>? logger) : base(localizer, logger)
+        public EncryptExtractsTransformer(ISharedResourcesLocalizer localizer, ILogger<IContentTransformer<T>> logger) : base(localizer, logger)
         {
             _logger = logger;
         }

diff --git a/examples/Csharp.ExampleApplication/Hooks/Transformers/MigratedTagTransformer.cs b/examples/Csharp.ExampleApplication/Hooks/Transformers/MigratedTagTransformer.cs
@@ -15,7 +15,7 @@ public class MigratedTagTransformer<T> : ContentTransformerBase<T> where T : ICo
     {
         private readonly ILogger<IContentTransformer<T>>? _logger;
 
-        public MigratedTagTransformer(ISharedResourcesLocalizer? localizer, ILogger<IContentTransformer<T>>? logger) : base(localizer, logger)
+        public MigratedTagTransformer(ISharedResourcesLocalizer localizer, ILogger<IContentTransformer<T>> logger) : base(localizer, logger)
         {
             _logger = logger;
         }

diff --git a/examples/Csharp.ExampleApplication/MyMigrationApplication.cs b/examples/Csharp.ExampleApplication/MyMigrationApplication.cs
@@ -15,6 +15,7 @@
 using Microsoft.Extensions.Options;
 using Tableau.Migration;
 using Tableau.Migration.Content;
+using Tableau.Migration.Content.Schedules.Cloud;
 using Tableau.Migration.Engine.Pipelines;
 
 #region namespace
@@ -120,6 +121,10 @@ public async Task StartAsync(CancellationToken cancel)
             _planBuilder.Transformers.Add<EncryptExtractsTransformer<IPublishableWorkbook>, IPublishableWorkbook>();
             #endregion
 
+            #region StartAtTransformer-Registration
+            _planBuilder.Transformers.Add<SimpleScheduleStartAtTransformer<ICloudExtractRefreshTask>, ICloudExtractRefreshTask>();
+            #endregion
+
             // Add migration action completed hooks
             #region LogMigrationActionsHook-Registration
             _planBuilder.Hooks.Add<LogMigrationActionsHook>();
@@ -131,6 +136,7 @@ public async Task StartAsync(CancellationToken cancel)
             _planBuilder.Hooks.Add<LogMigrationBatchesHook<IProject>>();
             _planBuilder.Hooks.Add<LogMigrationBatchesHook<IDataSource>>();
             _planBuilder.Hooks.Add<LogMigrationBatchesHook<IWorkbook>>();
+            _planBuilder.Hooks.Add<LogMigrationBatchesHook<ICloudExtractRefreshTask>>();
             #endregion
 
             // Build the plan

diff --git a/examples/Csharp.ExampleApplication/Program.cs b/examples/Csharp.ExampleApplication/Program.cs
@@ -87,6 +87,10 @@ public static IServiceCollection AddCustomizations(this IServiceCollection servi
             services.AddScoped<EncryptExtractsTransformer<IPublishableWorkbook>>();
             #endregion
 
+            #region StartAtTransformer-DI
+            services.AddScoped(typeof(SimpleScheduleStartAtTransformer<>));
+            #endregion
+
             #region LogMigrationActionsHook-DI
             services.AddScoped<LogMigrationActionsHook>();
             #endregion

diff --git a/examples/Csharp.ExampleApplication/appsettings.json b/examples/Csharp.ExampleApplication/appsettings.json
@@ -32,6 +32,10 @@
         {
           "type": "workbook",
           "batchSize": 90
+        },
+        {
+          "type": "serverextractrefreshtask",
+          "batchSize": 100
         }
       ]
     },

diff --git a/examples/Python.ExampleApplication/Hooks/mappings/email_domain_mapping.py b/examples/Python.ExampleApplication/Hooks/mappings/email_domain_mapping.py
@@ -11,7 +11,7 @@ def map(self, ctx: ContentMappingContext[IUser]) -> ContentMappingContext[IUser]
         _tableau_user_domain = ctx.mapped_location.parent()
 
         # Re-use an existing email if it already exists. 
-        if not ctx.content_item.email:
+        if ctx.content_item.email:
             return ctx.map_to(_tableau_user_domain.append(ctx.content_item.email))
 
         # Takes the existing username and appends the domain to build the email

diff --git a/examples/Python.ExampleApplication/Python.ExampleApplication.pyproj b/examples/Python.ExampleApplication/Python.ExampleApplication.pyproj
@@ -32,6 +32,7 @@
     <Compile Include="hooks\post_publish\bulk_logging_hook.py" />
     <Compile Include="hooks\transformers\encrypt_extracts_transformer.py" />
     <Compile Include="hooks\transformers\migrated_tag_transformer.py" />
+    <Compile Include="Hooks\transformers\schedule_startat_transformer.py" />
     <Compile Include="print_result.py" />
     <Compile Include="Python.ExampleApplication.py" />
   </ItemGroup>

diff --git a/examples/Python.ExampleApplication/requirements.txt b/examples/Python.ExampleApplication/requirements.txt
@@ -1,10 +1,7 @@
---index-url https://artifactory.prod.tableautools.com/artifactory/api/pypi/tabpypi/simple
---pre
-pip==24.0.0
-setuptools==69.1.0
-configparser==6.0.0
+setuptools==70.1.1
+configparser==7.0.0
 tableau_migration
 cffi==1.16.0
-pycparser==2.21
+pycparser==2.22
 pythonnet==3.0.3
-typing_extensions==4.9.0
+typing_extensions==4.12.2
diff --git a/global.json b/global.json
@@ -1,6 +1,6 @@
 {
   "sdk": {
-    "version": "8.0.201",
+    "version": "8.0.302",
     "rollForward": "latestMajor"
   }
 }
diff --git a/scripts/generate-docs.ps1 b/scripts/generate-docs.ps1
@@ -49,12 +49,12 @@ $main_docs_dir = Join-Path $root_dir "src/Documentation"
 $sphinx_output_dir = Join-Path $python_dir "Documentation/generated"
 
 # Directory for Python wrapper files. Includes an index file and a directory for autogenerated files.
-$python_wrapper_dir = Join-Path $main_docs_dir "api-python";
+$python_apiref_dir = Join-Path $main_docs_dir "api-python";
 
 $python_reference_dir_name = "reference";
 
 # Directory where DocFX looks for markdown files to render into our 'Python Wrapper' section.
-$python_md_destination = Join-Path $python_wrapper_dir $python_reference_dir_name;
+$python_md_destination = Join-Path $python_apiref_dir $python_reference_dir_name;
 
 function Run-Command {
 	param([string]$Cmd)
@@ -152,7 +152,7 @@ function Write-Python-docs {
 	Run-Command ("python -m pip install -q --upgrade pip");
 	Run-Command ("python -m pip install -q hatch");
 	Run-Command ("Clear-Directory -Path $sphinx_output_dir");	
-	Run-Command ("python -m hatch run docs:sphinx-build -M markdown $sphinx_build_dir $sphinx_output_dir -q");
+	Run-Command ("python -m hatch run docs:sphinx-build -M markdown $sphinx_build_dir $sphinx_output_dir -Q");
 	Run-Command ("Pop-Location");
 	Write-Host-With-Timestamp "Finished: Generating python docs.";
 
@@ -269,7 +269,7 @@ function Write-Python-Docs-Toc {
 		}		
 	}
 
-	$tocPath = Join-Path -Path $python_wrapper_dir -ChildPath toc.yml;
+	$tocPath = Join-Path -Path $python_apiref_dir -ChildPath toc.yml;
 
 	Run-Command ("Out-File -FilePath '$tocPath' -InputObject '$($fileContent | Out-String)'");
 
@@ -290,7 +290,7 @@ function Write-Final-Docs {
 
 	# Docfx related paths
 	$docfx_config_path = Join-Path $main_docs_dir "docfx.json";
-	$docfx_cmd = "dotnet docfx $docfx_config_path -t statictoc,templates\tableau --logLevel warning";
+	$docfx_cmd = "dotnet docfx $docfx_config_path -t default,statictoc,templates\tableau --logLevel warning";
 
 	# Run the docfx command to generate the final output	
 	if ($Serve) {

diff --git a/src/Documentation/.gitignore b/src/Documentation/.gitignore
@@ -6,5 +6,4 @@
 /**/packages/
 /**/bin/
 /**/obj/
-_site
 migration_sdk_metadata.json
diff --git a/src/Documentation/README.md b/src/Documentation/README.md
@@ -148,7 +148,7 @@ Content in Tab 3
 ##### Using regions (Preferred method)
 
 ```markdown
-[!code-csharp[CS](../../../../examples/Csharp.ExampleApplication/MyMigrationApplication.cs#DefaultProjectsFilter-Registration)]
+[!code-csharp[](../../../../examples/Csharp.ExampleApplication/MyMigrationApplication.cs#DefaultProjectsFilter-Registration)]
 ```
 
 ##### Using line numbers
@@ -157,7 +157,7 @@ Content in Tab 3
 > Use these very sparingly for C#. When the code is edited, it can change line numbers and mess up code snippets.
 
 ```markdown
-[!code-csharp[CS](../../../../examples/Csharp.ExampleApplication/Hooks/Mappings/EmailDomainMapping.cs#L8-)]
+[!code-csharp[](../../../../examples/Csharp.ExampleApplication/Hooks/Mappings/EmailDomainMapping.cs#L8-)]
 ```
 
 > [!IMPORTANT]

diff --git a/src/Documentation/api-csharp/index.md b/src/Documentation/api-csharp/index.md
@@ -4,21 +4,21 @@ Welcome to the C# API Reference for the Migration SDK.
 
 ## Examples to get started
 
-The following code samples are for writing a simple migration app using the Migration SDK. For details on configuring and customizing the Migration SDK to your specific needs, see [Articles](~/articles/intro.md) and [Code Samples](~/samples/intro.md).
+The following code samples are for writing a simple migration app using the Migration SDK. For details on configuring and customizing the Migration SDK to your specific needs, see [Articles](~/articles/index.md) and [Code Samples](~/samples/index.md).
 
 ### [Program.cs](#tab/program-cs)
 
-[!code-csharp[CS](../../../examples/Csharp.ExampleApplication/Program.cs#namespace)]
+[!code-csharp[](../../../examples/Csharp.ExampleApplication/Program.cs#namespace)]
 
 ### [Startup code](#tab/startup-cde)
 
-[!code-csharp[CS](../../../examples/Csharp.ExampleApplication/MyMigrationApplication.cs#namespace)]
+[!code-csharp[](../../../examples/Csharp.ExampleApplication/MyMigrationApplication.cs#namespace)]
 
 ### [Config classes](#tab/config-classes)
 
-[!code-csharp[CS](../../../examples/Csharp.ExampleApplication/Config/MyMigrationApplicationOptions.cs#namespace)]
+[!code-csharp[](../../../examples/Csharp.ExampleApplication/Config/MyMigrationApplicationOptions.cs#namespace)]
 
-[!code-csharp[CS](../../../examples/Csharp.ExampleApplication/Config/EndpointOptions.cs#namespace)]
+[!code-csharp[](../../../examples/Csharp.ExampleApplication/Config/EndpointOptions.cs#namespace)]
 
 ### [Config file](#tab/appsettings)
 
@@ -43,5 +43,5 @@ The following code samples are for writing a simple migration app using the Migr
 
 ## Suggested Reading
 
-- [Code Samples](~/samples/intro.md)
-- [Articles](~/articles/intro.md)
+- [Code Samples](~/samples/index.md)
+- [Articles](~/articles/index.md)
diff --git a/src/Documentation/api-python/index.md b/src/Documentation/api-python/index.md
@@ -22,7 +22,7 @@ There are advanced features of the Migration SDK that the Python Wrapper cannot
 
 ## Examples to get started
 
-The following code samples are for writing a simple migration app using the Migration SDK. For details on configuring and customizing the Migration SDK to your specific needs, see [Articles](~/articles/intro.md) and [Code Samples](~/samples/intro.md).
+The following code samples are for writing a simple migration app using the Migration SDK. For details on configuring and customizing the Migration SDK to your specific needs, see [Articles](~/articles/index.md) and [Code Samples](~/samples/index.md).
 
 ### [Startup Script](#tab/startup)
 

diff --git a/src/Documentation/articles/configuration.md b/src/Documentation/articles/configuration.md
@@ -2,7 +2,7 @@
 
 The Migration SDK uses two sources of configuration in two blocks: the [Migration Plan](#migration-plan) that contains configuration for a specific migration run, and [Migration SDK Options](#migration-sdk-options) for configuration that is unlikely to change between migration runs.
 
-![Configuration Blocks](../images/configuration.png)
+![Configuration Blocks](../images/configuration.svg){width=65%}
 
 ## Migration Plan
 
@@ -79,7 +79,10 @@ This is an array of [`MigrationSdkOptions.ContentTypesOptions`](xref:Tableau.Mig
 > The [type](xref:Tableau.Migration.Config.ContentTypesOptions.Type) values are case-insensitive.
 > Duplicate [type](xref:Tableau.Migration.Config.ContentTypesOptions.Type) key values will result in an exception.
 
-In the following `json` example config file, a `BatchSize` of `201` is applied to the content type `User`. The same setting for `Project` is `203`.
+In the following `json` example config file, 
+- A `BatchSize` of `201` is applied to the content type `User`. 
+- A `BatchSize` of `203` for `Project`.
+- A `BatchSize` of `200` for `ServerExtractRefreshTask`.
 
 ```JSON
 {
@@ -92,6 +95,10 @@ In the following `json` example config file, a `BatchSize` of `201` is applied t
         {
             "type":"Project",
             "batchSize": 203            
+        },
+        {
+            "type":"ServerExtractRefreshTask",
+            "batchSize": 200
         }
         ],
     }
@@ -146,6 +153,7 @@ The following sections describe each setting. They should always be set per cont
 *Description:* The Migration SDK uses the **BatchPublishingEnabled** property to select the mode it will publish a given content type. Disabled by default, with this configuration, the SDK will publish the content by using individual REST API calls for each item. When this option is enabled, it is possible to publish content in a batch of items (just for some supported content types).
 
 Supported Content Types:
+
 - [User](xref:Tableau.Migration.Content.IUser) by using the method [Import Users to Site from CSV](https://help.tableau.com/current/api/rest_api/en-us/REST/rest_api_ref_users_and_groups.htm#import_users_to_site_from_csv);
 
 ### MigrationParallelism