chore(pipelines)!: improvements on CI-Release and CD-Containers

BREAKING CHANGE: split all tasks in small pipeline templates that can be reused

cmf-cli/resources/template_feed/init/Builds/.tasks/clean-agent-directories.yml

@@ -1,16 +1,17 @@
+# Clean Agent Directories
 steps:
-- pwsh: Remove-Item $(Build.SourcesDirectory)\* -Recurse -Force -Verbose
-  displayName: Clean Sources Directory
-  condition: always()
+  - pwsh: Remove-Item $(Build.SourcesDirectory)\* -Recurse -Force -Verbose
+    displayName: Clean Sources Directory
+    condition: always()
 
-- pwsh: Remove-Item $(Build.ArtifactStagingDirectory)\* -Recurse -Force -Verbose
-  displayName: Clean Artifact Staging Directory
-  condition: always()
+  - pwsh: Remove-Item $(Build.ArtifactStagingDirectory)\* -Recurse -Force -Verbose
+    displayName: Clean Artifact Staging Directory
+    condition: always()
 
-- pwsh: Remove-Item $(Build.BinariesDirectory)\* -Recurse -Force -Verbose
-  displayName: Clean Binaries Directory
-  condition: always()
+  - pwsh: Remove-Item $(Build.BinariesDirectory)\* -Recurse -Force -Verbose
+    displayName: Clean Binaries Directory
+    condition: always()
 
-- pwsh: Remove-Item $(Common.TestResultsDirectory)\* -Recurse -Force -Verbose
-  displayName: Clean Test Results Directory
-  condition: always()
+  - pwsh: Remove-Item $(Common.TestResultsDirectory)\* -Recurse -Force -Verbose
+    displayName: Clean Test Results Directory
+    condition: always()

cmf-cli/resources/template_feed/init/Builds/.tasks/copy-and-mount-iso.yml

@@ -0,0 +1,24 @@
+# Copy and mount MES ISO
+steps:
+- powershell: |
+    $iso = Get-Item "$(ISOImagePath)"
+    $destPath = "$(CandidatePackages)/../isos/"
+    $destIso = "$destPath/" + $iso.Name
+
+    if (-Not (Test-Path $destIso)) {
+        New-Item -Type Directory -Path $destPath -Force -Verbose
+        Copy-Item -Path $iso -Destination $destIso -Force -Verbose
+    }
+
+    $ImagePath = $destIso
+
+    # Mount the ISO, without having a drive letter auto-assigned
+    if (!(Get-DiskImage -ImagePath $ImagePath).Attached) {
+        Mount-DiskImage -ImagePath $ImagePath -PassThru
+    }
+
+    $driveLetter = (Get-DiskImage -ImagePath $ImagePath | Get-Volume).DriveLetter + ":/"
+    Set-Location -Path $driveLetter
+
+    Write-Output "##vso[task.setvariable variable=DriveLetter]$driveLetter"
+  displayName: Copy and mount MES ISO

cmf-cli/resources/template_feed/init/Builds/.tasks/copy-configurations-files.yml

@@ -0,0 +1,58 @@
+parameters:
+  configurationDir: $(Build.ArtifactStagingDirectory)/Configurations
+  packageDir: ''
+
+steps:
+# Copy dependencies.json to Configurations~
+- task: CopyFiles@2
+  displayName: Copy dependencies.json to Configurations
+  inputs:
+    SourceFolder: ${{ parameters.packageDir }}
+    Contents: |
+      dependencies.json
+    FlattenFolders: true
+    TargetFolder: ${{ parameters.configurationDir }}
+
+# Remove dependencies.json
+- task: DeleteFiles@1
+  displayName: Remove dependencies.json from source
+  inputs:
+    SourceFolder: ${{ parameters.packageDir }}
+    Contents: dependencies.json
+
+# Copy global.json and cmfpackage.json to Configurations
+- task: CopyFiles@2
+  displayName: Copy global.json and cmfpackage.json to Configurations
+  inputs:
+    Contents: |
+      global.json
+      cmfpackage.json
+    FlattenFolders: true
+    TargetFolder: ${{ parameters.configurationDir }}
+
+# Copy Runsettings to Configurations
+- task: CopyFiles@2
+  displayName: Copy Runsettings to Configurations
+  inputs:
+    Contents: |
+      **/*.runsettings
+    TargetFolder: ${{ parameters.configurationDir }}/Runsettings
+    flattenFolders: true
+
+# Copy EnvironmentConfigs to Configurations
+- task: CopyFiles@2
+  displayName: Copy EnvironmentConfigs to Configurations
+  inputs:
+    SourceFolder: EnvironmentConfigs
+    Contents: |
+      *.json
+    TargetFolder: $(Build.ArtifactStagingDirectory)/Configurations/EnvironmentConfigs
+
+# Copy LBOs to Configurations
+- task: CopyFiles@2
+  displayName: Copy LBOs to Configurations
+  inputs:
+    SourceFolder: Libs/LBOs/NetStandard
+    Contents: |
+      Cmf.LightBusinessObjects.dll
+    TargetFolder: $(Build.ArtifactStagingDirectory)/Configurations/LBOs

cmf-cli/resources/template_feed/init/Builds/.tasks/download-cd-artifacts.yml

@@ -0,0 +1,50 @@
+steps:
+# Clean deploy folder
+- task: DeleteFiles@1
+  displayName: Clean $(CandidatePackages) folder
+  inputs:
+    SourceFolder: $(CandidatePackages)
+    Contents: "**"
+
+# Download Configurations Artifacts
+- task: DownloadBuildArtifacts@0
+  displayName: Download Configurations
+  inputs:
+    buildType: specific
+    project: $(System.TeamProject)
+    pipeline: CI-Publish
+    specificBuildWithTriggering: true
+    buildVersionToDownload: specific
+    buildId: $(resources.pipeline.CIPublish.runID)
+    downloadType: single
+    artifactName: Configurations
+    downloadPath: $(CandidatePackages)
+
+# Download Package Artifacts
+- task: DownloadBuildArtifacts@0
+  displayName: Download Package
+  inputs:
+    buildType: specific
+    project: $(System.TeamProject)
+    pipeline: CI-Publish
+    specificBuildWithTriggering: true
+    buildVersionToDownload: specific
+    buildId: $(resources.pipeline.CIPublish.runID)
+    downloadType: single
+    artifactName: Package
+    downloadPath: $(CandidatePackages)
+
+# Download Tests Artifacts
+- task: DownloadBuildArtifacts@0
+  displayName: Download Tests
+  inputs:
+    buildType: specific
+    project: $(System.TeamProject)
+    pipeline: CI-Publish
+    specificBuildWithTriggering: true
+    buildVersionToDownload: specific
+    buildId: $(resources.pipeline.CIPublish.runID)
+    downloadType: single
+    artifactName: Tests
+    downloadPath: $(CandidatePackages)
+  continueOnError: true

cmf-cli/resources/template_feed/init/Builds/.tasks/drop-db.yml

@@ -0,0 +1,32 @@
+# Drop Database
+parameters:
+<<<<<<< Updated upstream
+    DB: ''
+=======
+  DB: ''
+  DBName: $(EnvConfig.Product.SystemName)
+  ServerName: $(EnvConfig.PackageProduct.Database.${{parameters.DB}}.Database.Server)
+
+>>>>>>> Stashed changes
+steps:
+- task: SqlDacpacDeploymentOnMachineGroup@0
+  displayName: Drop ${{parameters.DB}} database
+  inputs:
+    TaskType: sqlInline
+    InlineSql: |
+      IF EXISTS (SELECT name from sys.databases WHERE (name = N'${{parameters.DBName}}'))
+      BEGIN
+        EXEC msdb.dbo.sp_delete_database_backuphistory @database_name = N'${{parameters.DBName}}'
+        ALTER DATABASE [${{parameters.DBName}}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE
+        DROP DATABASE IF EXISTS ${{parameters.DBName}}
+      END;
+    ServerName: ${{parameters.ServerName}}
+    DatabaseName: master
+    AuthScheme: sqlServerAuthentication
+<<<<<<< Updated upstream
+    SqlUsername: $(EnvConfig.PackageProduct.Database.${{ parameters.DB}}.Database.User)
+    SqlPassword: $(EnvConfig.PackageProduct.Database.${{ parameters.DB}}.Database.Password)
+=======
+    SqlUsername: $(Database${{parameters.DB}}User)
+    SqlPassword: $(Database${{parameters.DB}}Password)
+>>>>>>> Stashed changes

cmf-cli/resources/template_feed/init/Builds/.tasks/install-cli.yml

@@ -1,4 +1,7 @@
 steps:
+# set node version
+- template: use-node-version.yml
+
 - pwsh: npm install --no-save @criticalmanufacturing/cli@$(CmfCliVersion) --registry $(CmfCliRepository)
   displayName: Install cmf-cli@$(CmfCliVersion)
   workingDirectory: $(Agent.TempDirectory)

cmf-cli/resources/template_feed/init/Builds/.tasks/install-cmf-pipeline.yml

@@ -1,4 +1,7 @@
 steps:
+# set node version
+- template: use-node-version.yml
+
 - pwsh: npm install --no-save @criticalmanufacturing/cmf-pipeline@$(CmfPipelineVersion) --registry $(CmfPipelineRepository)
   displayName: Install cmf-pipeline@$(CmfPipelineVersion)
   workingDirectory: $(Agent.TempDirectory)

cmf-cli/resources/template_feed/init/Builds/.tasks/install-cmf-portal.yml

@@ -0,0 +1,7 @@
+steps:
+# set node version
+- template: use-node-version.yml
+
+- pwsh: npm install --no-save @criticalmanufacturing/portal --registry ${{ variables.CmfCliRepository }}
+  displayName: Install cmf-portal
+  workingDirectory: $(Agent.TempDirectory)

cmf-cli/resources/template_feed/init/Builds/.tasks/load-config.yml

@@ -0,0 +1,46 @@
+# Load EnvironmentConfig and Package Info
+steps:
+- pwsh: |
+    # Load PackageInfo
+    $PackageId = "${{ variables.RootPackageIdToInstall }}"
+    if($PackageId)
+    {
+        $PackageVersion = (Get-Item "$(CandidatePackages)/Package/$PackageId*").BaseName -replace "$PackageId."
+    }
+    else
+    {
+        $CmfPackageJsonFile = Get-Item "$(CandidatePackages)//Configurations/cmfpackage.json"
+        $CmfPackageJson = Get-Content -Raw -Path $CmfPackageJsonFile | ConvertFrom-Json
+        $PackageId = $CmfPackageJson."packageId"
+        $PackageVersion = $CmfPackageJson."version"
+    }
+    $PackageToInstall = "$PackageId@$PackageVersion"
+
+    # Load file location to variable
+    $file = "$(CandidatePackages)/Configurations/EnvironmentConfigs/$(EnvironmentConfigName)"
+    Write-Output "##vso[task.setvariable variable=ConfigPath;isOutput=true]$file"
+
+    # Load file to variable
+    $configString = Get-Content "$file" | out-string
+    Write-Output "##vso[task.setvariable variable=ConfigString;isOutput=true]$configString"
+
+    # Load each property to variable
+    if (Test-Path -Path $file -PathType Leaf) {
+        $envConfig = $configString | ConvertFrom-Json
+        $envConfig.PSObject.Properties | ForEach-Object {
+        if (-not($_.Value -eq "" -or $_.Value -eq $nil -or
+            ($_.Value.GetType().FullName -eq "System.Management.Automation.PSCustomObject"))) {
+                # we need to remove the [] to avoid errors when trying to access the variable
+                $var = ($_.Name).Replace("[","").Replace("]","");
+                Write-Output "##vso[task.setvariable variable=$($var);isOutput=true]$($_.Value)"
+            }
+        }
+    } else {
+        Write-Output "$file not found"
+    }
+
+    Write-Output "##vso[task.setvariable variable=PackageToInstall;isOutput=true]$PackageToInstall"
+    Write-Output "##vso[task.setvariable variable=PackageId;isOutput=true]$PackageId"
+    Write-Output "##vso[task.setvariable variable=PackageVersion;isOutput=true]$PackageVersion"
+  name: EnvConfig
+  displayName: Load EnvironmentConfig and PackageInfo

cmf-cli/resources/template_feed/init/Builds/.tasks/load-root-package-details.yml

@@ -0,0 +1,13 @@
+steps:
+  - pwsh: |
+      $CmfPackageJsonFile = Get-Item ./cmfpackage.json
+      $CmfPackageJson = Get-Content -Raw -Path $CmfPackageJsonFile | ConvertFrom-Json
+      $PackageId = $CmfPackageJson."packageId"
+      $PackageVersion = $CmfPackageJson."version"
+      $Package = "$PackageId@$PackageVersion"
+
+      Write-Output "##vso[task.setvariable variable=Name;isOutput=true]$Package"
+      Write-Output "##vso[task.setvariable variable=Id;isOutput=true]$PackageId"
+      Write-Output "##vso[task.setvariable variable=Version;isOutput=true]$PackageVersion"
+    name: Package
+    displayName: Get package details

cmf-cli/resources/template_feed/init/Builds/.tasks/portal-deploy-customization.yml

@@ -0,0 +1,17 @@
+steps:
+- pwsh: |
+    $envConfig = Get-Content "$(EnvConfig.ConfigPath)" | Out-String | ConvertFrom-Json
+    $envConfig.ENV_MANAGER_BOOT_PACKAGE = $(EnvConfig.PackageToInstall)
+    $envConfig | ConvertTo-Json -Depth 10 | set-content "$(Agent.TempDirectory)/install-package.json"
+
+    $(Agent.TempDirectory)/node_modules/@criticalmanufacturing/portal/bin/cmf-portal.exe deploy `
+    --name "$(CustomerEnvironmentName)" `
+    --customer-infrastructure-name "DS Integration Environments" `
+    --parameters "$(Agent.TempDirectory)/install-package.json" `
+    --license "$(LicenseName)" `
+    --site "$(SiteName)" `
+    --target "$(DeploymentTarget)" `
+    --package="$(DeploymentPackage)" `
+    --terminateOtherVersions --verbose
+  displayName: Customization Installation
+

cmf-cli/resources/template_feed/init/Builds/.tasks/portal-deploy-masterdata.yml

@@ -0,0 +1,30 @@
+steps:
+- pwsh: |
+    $testMasterDataFilter = "$(TestMasterDataFilter)"
+    if(!$testMasterDataFilter)
+    {
+      $testMasterDataFilter = "*MasterData*"
+    }
+    $masterDataFiles = "$(CandidatePackages)/Tests/$testMasterDataFilter.zip"
+
+    Get-ChildItem $masterDataFiles | foreach {
+      $PackageId = [regex]::split($_.BaseName, ".[0-9]+.[0-9]+.[0-9]+")[0];
+      $PackageVersion = ($_.BaseName-replace "$PackageId." );
+      $PackageToInstall = "$PackageId@$PackageVersion"
+
+      $envConfig = Get-Content "$(EnvConfig.ConfigPath)" | Out-String | ConvertFrom-Json
+      $envConfig.ENV_MANAGER_BOOT_PACKAGE = $(EnvConfig.PackageToInstall)
+      $envConfig | ConvertTo-Json -Depth 10 | set-content "$(Agent.TempDirectory)/install-package.json"
+
+      $(Agent.TempDirectory)/node_modules/@criticalmanufacturing/portal/bin/cmf-portal.exe deploy `
+      --name "$(CustomerEnvironmentName)" `
+      --customer-infrastructure-name "DS Integration Environments" `
+      --parameters "$(Agent.TempDirectory)/install-package.json" `
+      --license "$(LicenseName)" `
+      --site "$(SiteName)" `
+      --target "$(DeploymentTarget)" `
+      --package="$(DeploymentPackage)" `
+      --terminateOtherVersions --verbose
+    }
+  displayName: MasterData Loading
+

cmf-cli/resources/template_feed/init/Builds/.tasks/portal-login.yml

@@ -0,0 +1,3 @@
+steps:
+- pwsh: $(Agent.TempDirectory)/node_modules/@criticalmanufacturing/portal/bin/cmf-portal.exe login --token $(CustomerPortalPAT) --verbose
+  displayName: Portal Login 

cmf-cli/resources/template_feed/init/Builds/.tasks/publish-tests.yml

@@ -1,21 +1,23 @@
 parameters:
-- name: path # name of the parameter; required
-  type: string # data type of the parameter; required
+- name: path
+  type: string
   default: false
 
 steps:
-- task: PowerShell@2
-  displayName: 'Check if reportgenerator is to be executed'
-  inputs:
-    targetType: 'inline'
-    script: |
-      $count = (Get-ChildItem -Path ".\**\coverage.cobertura.xml" -Recurse | Measure-Object).Count
-      if ($count -gt 0) {
-        Write-Host "##vso[task.setvariable variable=generateCoverageReport]Yes" #set variable doThing to Yes
-      }
+- pwsh: |
+    $count = (Get-ChildItem -Path ".\**\coverage.cobertura.xml" -Recurse | Measure-Object).Count
+    if ($count -gt 0) {
+      Write-Host "##vso[task.setvariable variable=generateCoverageReport]Yes" #set variable doThing to Yes
+    }
+  displayName: Check if reportgenerator is to be executed
+
+# install dotnet tools
+- pwsh: dotnet tool restore
+  displayName: Restore .NET tools
 
-- script: 'dotnet tool run dotnet-coverage merge -o "${{ parameters.path }}/Merged.Cobertura.xml" -f "Cobertura" -r "coverage.cobertura.xml"'
-  displayName: 'Run reportgenerator'
+# run dotnet-coverage
+- pwsh: dotnet tool run dotnet-coverage merge -o "${{ parameters.path }}/Merged.Cobertura.xml" -f "Cobertura" -r "coverage.cobertura.xml"
+  displayName: Run reportgenerator
   condition: and(succeeded(), eq(variables.generateCoverageReport, 'Yes'))
 
 # Publish test results
@@ -31,4 +33,4 @@ steps:
   displayName: Publishing Code Coverage
   inputs:
     codeCoverageTool: Cobertura
-    summaryFileLocation: '${{ parameters.path }}/Merged.Cobertura.xml'
+    summaryFileLocation: '${{ parameters.path }}/Merged.Cobertura.xml'