End to end tests #228
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: End to end tests | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| ref: | |
| description: ref on which to run the end-2-end tests (default is head_sha on main) | |
| required: false | |
| default: '' | |
| githubOwner: | |
| description: GitHub organization set as owner for the temp repositories (default is to use your personal account - CANNOT be a free account) | |
| required: false | |
| default: '' | |
| runTestMatrix: | |
| description: Run the end to end test scenario for the full test matrix | |
| type: boolean | |
| default: true | |
| runScenarios: | |
| description: Run the end to end scenario tests | |
| type: boolean | |
| default: true | |
| runUpgradeTests: | |
| description: Run the end to end upgrade tests | |
| type: boolean | |
| default: true | |
| bcContainerHelperVersion: | |
| description: Which version of BcContainerHelper to use? (latest, preview, private, a specific version number or a direct download URL like https://github.com/freddydk/navcontainerhelper/archive/master.zip - leave empty to use latest) | |
| required: false | |
| default: 'preview' | |
| defaults: | |
| run: | |
| shell: pwsh | |
| jobs: | |
| Check: | |
| runs-on: [ ubuntu-latest ] | |
| outputs: | |
| maxParallel: ${{ steps.check.outputs.maxParallel }} | |
| githubOwner: ${{ steps.check.outputs.githubOwner }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Check secrets | |
| id: check | |
| env: | |
| githubOwner: ${{ github.event.inputs.githubOwner }} | |
| run: | | |
| $err = $false | |
| if ('${{ Secrets.E2EPAT }}' -eq '') { | |
| Write-Host "::Error::In order to run end to end tests, you need a Secret called E2EPAT containing a valid Personal Access Token with the following permissions: admin:org, delete_repo, repo, workflow, packages:write" | |
| $err = $true | |
| } | |
| if ('${{ Secrets.adminCenterApiCredentials }}' -eq '') { | |
| Write-Host "::Error::In order to run end to end tests, you need a Secret called adminCenterApiCredentials containing API access credentials for BC Admin Center" | |
| $err = $true | |
| } | |
| if ('${{ Secrets.licenseFileUrl }}' -eq '') { | |
| Write-Host "::Error::In order to run end to end tests, you need a Secret called licenseFileUrl containing a secure Url to a build license file" | |
| $err = $true | |
| } | |
| if ($err) { | |
| exit 1 | |
| } | |
| $githubOwner = $ENV:githubOwner | |
| $maxParallel = 99 | |
| if (!($githubOwner)) { | |
| $githubOwner = "$ENV:GITHUB_REPOSITORY_OWNER" | |
| } | |
| $orgmap = Get-Content -path (Join-Path "." "e2eTests\orgmap.json") -encoding UTF8 -raw | ConvertFrom-Json | |
| if ($orgmap.PSObject.Properties.Name -eq $githubOwner) { | |
| $githubOwner = $orgmap."$githubOwner" | |
| } | |
| if ($githubOwner -eq $ENV:GITHUB_REPOSITORY_OWNER) { | |
| $maxParallel = 8 | |
| } | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "maxParallel=$maxParallel" | |
| Write-Host "maxParallel=$maxParallel" | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "githubOwner=$githubOwner" | |
| Write-Host "githubOwner=$githubOwner" | |
| SetupRepositories: | |
| runs-on: [ ubuntu-latest ] | |
| needs: [ Check ] | |
| outputs: | |
| actionsRepo: ${{ steps.setup.outputs.actionsRepo }} | |
| perTenantExtensionRepo: ${{ steps.setup.outputs.perTenantExtensionRepo }} | |
| appSourceAppRepo: ${{ steps.setup.outputs.appSourceAppRepo }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Setup Repositories | |
| id: setup | |
| run: | | |
| $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 | |
| . (Join-Path "." "e2eTests/SetupRepositories.ps1") -githubOwner '${{ needs.Check.outputs.githubowner }}' -token '${{ Secrets.E2EPAT }}' -bcContainerHelperVersion '${{ github.event.inputs.bcContainerHelperVersion }}' | |
| Analyze: | |
| runs-on: [ ubuntu-latest ] | |
| needs: [ Check ] | |
| outputs: | |
| testruns: ${{ steps.Analyze.outputs.testruns }} | |
| releases: ${{ steps.Analyze.outputs.releases }} | |
| scenarios: ${{ steps.Analyze.outputs.scenarios }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Analyze | |
| id: Analyze | |
| run: | | |
| $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 | |
| $modulePath = Join-Path "." "e2eTests\e2eTestHelper.psm1" -resolve | |
| Import-Module $modulePath -DisableNameChecking | |
| SetTokenAndRepository -github -githubOwner '${{ needs.Check.outputs.githubowner }}' -token '${{ Secrets.E2EPAT }}' -repository "microsoft/AL-Go" | |
| $maxParallel = [int]'${{ needs.Check.outputs.maxParallel }}' | |
| $testrunsJson = @{ | |
| "matrix" = @{ | |
| "include" = @( | |
| @('private','public') | ForEach-Object { | |
| $visibility = $_ | |
| @('appSourceApp','PTE') | ForEach-Object { | |
| $type = $_ | |
| @('linux','windows') | ForEach-Object { | |
| $os = $_ | |
| @('multiProject','singleProject') | ForEach-Object { | |
| $style = $_ | |
| @{ "visibility" = $visibility; "type" = $type; "os" = $os; "style" = $style; "Compiler" = "Container" } | |
| if ($visibility -eq "public" -and $type -eq "PTE") { | |
| # Run end 2 end tests using CompilerFolder with Windows+Linux and single/multiproject | |
| @{ "visibility" = $visibility; "type" = $type; "os" = $os; "style" = $style; "Compiler" = "CompilerFolder" } | |
| } | |
| } | |
| } | |
| } | |
| } | |
| ) | |
| } | |
| "max-parallel" = $maxParallel | |
| "fail-fast" = $false | |
| } | ConvertTo-Json -depth 99 -compress | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "testruns=$testrunsJson" | |
| Write-Host "testruns=$testrunsJson" | |
| $releases = @(gh release list --repo microsoft/AL-Go | ForEach-Object { $_.split("`t")[0] }) | |
| $releasesJson = @{ | |
| "matrix" = @{ | |
| "include" = @($releases | ForEach-Object { @{ "Release" = $_; "type" = 'appSourceApp' }; @{ "Release" = $_; "type" = 'PTE' } } ) | |
| }; | |
| "max-parallel" = $maxParallel | |
| "fail-fast" = $false | |
| } | ConvertTo-Json -depth 99 -compress | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "releases=$releasesJson" | |
| Write-Host "releases=$releasesJson" | |
| $scenariosJson = @{ | |
| "matrix" = @{ | |
| "include" = @(Get-ChildItem -path (Join-Path $ENV:GITHUB_WORKSPACE "e2eTests/scenarios/*/runtest.ps1") | ForEach-Object { @{ "Scenario" = $_.Directory.Name } } ) | |
| }; | |
| "max-parallel" = $maxParallel | |
| "fail-fast" = $false | |
| } | ConvertTo-Json -depth 99 -compress | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "scenarios=$scenariosJson" | |
| Write-Host "scenarios=$scenariosJson" | |
| Scenario: | |
| runs-on: [ ubuntu-latest ] | |
| needs: [ Check, SetupRepositories, Analyze ] | |
| if: github.event.inputs.runScenarios == 'true' | |
| strategy: ${{ fromJson(needs.Analyze.outputs.scenarios) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Calculate parameters | |
| id: calculateParams | |
| run: | | |
| $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 | |
| $reponame = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName()) | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "repoName='$repoName'" | |
| Write-Host "repoName='$repoName'" | |
| Write-Host "Repo URL: https://github.com/${{ needs.Check.outputs.githubowner }}/$repoName" | |
| - name: Run test | |
| run: | | |
| try { | |
| . (Join-Path "." "e2eTests/scenarios/${{ matrix.scenario }}/runtest.ps1") -github -githubOwner '${{ needs.Check.outputs.githubowner }}' -repoName ${{ steps.calculateParams.outputs.repoName }} -token '${{ Secrets.E2EPAT }}' -pteTemplate '${{ needs.Check.outputs.githubowner }}/${{ needs.SetupRepositories.outputs.perTenantExtensionRepo }}' -appSourceTemplate '${{ needs.Check.outputs.githubowner }}/${{ needs.SetupRepositories.outputs.appSourceAppRepo }}' -adminCenterApiToken '${{ Secrets.adminCenterApiCredentials }}' -licenseFileUrl '${{ Secrets.licenseFileUrl }}' | |
| } | |
| catch { | |
| Write-Host $_.Exception.Message | |
| Write-Host $_.ScriptStackTrace | |
| Write-Host "::Error::$($_.Exception.Message)" | |
| $host.SetShouldExit(1) | |
| } | |
| TestAlGo: | |
| runs-on: [ ubuntu-latest ] | |
| needs: [ Check, SetupRepositories, Analyze ] | |
| if: github.event.inputs.runTestMatrix == 'true' | |
| strategy: ${{ fromJson(needs.Analyze.outputs.testruns) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Calculate parameters | |
| id: calculateParams | |
| run: | | |
| $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 | |
| $adminCenterApiCredentials = '' | |
| $licenseFileUrl = '' | |
| if ('${{ matrix.type }}' -eq 'PTE' -and '${{ matrix.style }}' -eq 'singleProject' -and '${{ matrix.os }}' -eq 'windows') { | |
| $adminCenterApiCredentials = '${{ Secrets.adminCenterApiCredentials }}' | |
| } | |
| if ('${{ matrix.type }}' -eq 'appSourceApp') { | |
| $licenseFileUrl='${{ secrets.licenseFileUrl }}' | |
| $template = '${{ needs.Check.outputs.githubowner }}/${{ needs.SetupRepositories.outputs.appSourceAppRepo }}' | |
| } | |
| else { | |
| $template = '${{ needs.Check.outputs.githubowner }}/${{ needs.SetupRepositories.outputs.perTenantExtensionRepo }}' | |
| } | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "adminCenterApiCredentials='$adminCenterApiCredentials'" | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "licenseFileUrl='$licenseFileUrl'" | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "template='$template'" | |
| $reponame = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName()) | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "repoName='$repoName'" | |
| Write-Host "repoName='$repoName'" | |
| Write-Host "Repo URL: https://github.com/${{ needs.Check.outputs.githubowner }}/$repoName" | |
| - name: Run tests | |
| run: | | |
| try { | |
| . (Join-Path "." "e2eTests/Test-AL-Go.ps1") -github -githubOwner '${{ needs.Check.outputs.githubowner }}' -repoName ${{ steps.calculateParams.outputs.repoName }} -token '${{ Secrets.E2EPAT }}' -template ${{ steps.calculateParams.outputs.template }} -adminCenterApiToken ${{ steps.calculateParams.outputs.adminCenterApiCredentials }} -licenseFileUrl ${{ steps.calculateParams.outputs.licenseFileUrl }} -multiProject:('${{ matrix.style }}' -eq 'multiProject') -appSource:('${{ matrix.type }}' -eq 'appSourceApp') -linux:('${{ matrix.os }}' -eq 'linux') -private:('${{ matrix.visibility }}' -eq 'private') -useCompilerFolder:('${{ matrix.Compiler }}' -eq 'CompilerFolder') | |
| } | |
| catch { | |
| Write-Host $_.Exception.Message | |
| Write-Host $_.ScriptStackTrace | |
| Write-Host "::Error::$($_.Exception.Message)" | |
| $host.SetShouldExit(1) | |
| } | |
| TestAlGoUpgrade: | |
| runs-on: [ ubuntu-latest ] | |
| needs: [ Check, SetupRepositories, Analyze ] | |
| if: github.event.inputs.runUpgradeTests == 'true' | |
| strategy: ${{ fromJson(needs.Analyze.outputs.releases) }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Calculate parameters | |
| id: calculateParams | |
| run: | | |
| $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 | |
| if ('${{ matrix.type }}' -eq 'appSourceApp') { | |
| $licenseFileUrl = '${{ secrets.licenseFileUrl }}' | |
| $template = '${{ needs.Check.outputs.githubowner }}/${{ needs.SetupRepositories.outputs.appSourceAppRepo }}' | |
| $contentPath = 'appsourceapp' | |
| } | |
| else { | |
| $licenseFileUrl = '' | |
| $template = '${{ needs.Check.outputs.githubowner }}/${{ needs.SetupRepositories.outputs.perTenantExtensionRepo }}' | |
| $contentPath = 'pte' | |
| } | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "licenseFileUrl='$licenseFileUrl'" | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "template='$template'" | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "contentPath='$contentPath'" | |
| $reponame = [System.IO.Path]::GetFileNameWithoutExtension([System.IO.Path]::GetTempFileName()) | |
| Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "repoName='$repoName'" | |
| Write-Host "repoName='$repoName'" | |
| Write-Host "Repo URL: https://github.com/${{ needs.Check.outputs.githubowner }}/$repoName" | |
| - name: Run tests | |
| run: | | |
| try { | |
| . (Join-Path "." "e2eTests/Test-AL-Go-Upgrade.ps1") -github -githubOwner '${{ needs.Check.outputs.githubowner }}' -repoName ${{ steps.calculateParams.outputs.repoName }} -token '${{ Secrets.E2EPAT }}' -template ${{ steps.calculateParams.outputs.template }} -licenseFileUrl ${{ steps.calculateParams.outputs.licenseFileUrl }} -appSource:('${{ matrix.type }}' -eq 'appSourceApp') -release '${{ matrix.release }}' -contentPath ${{ steps.calculateParams.outputs.contentPath }} | |
| } | |
| catch { | |
| Write-Host $_.Exception.Message | |
| Write-Host $_.ScriptStackTrace | |
| Write-Host "::Error::$($_.Exception.Message)" | |
| $host.SetShouldExit(1) | |
| } | |
| RemoveRepositories: | |
| runs-on: [ ubuntu-latest ] | |
| needs: [ Check, SetupRepositories, TestAlGo, TestAlGoUpgrade, Scenario ] | |
| if: always() && (!Cancelled()) && (needs.SetupRepositories.result == 'Success') && (needs.TestAlGo.result == 'Success' || needs.TestAlGo.result == 'Skipped') && (needs.TestAlGoUpgrade.result == 'Success' || needs.TestAlGoUpgrade.result == 'Skipped') && (needs.Scenario.result == 'Success' || needs.Scenario.result == 'Skipped') | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.inputs.ref }} | |
| - name: Remove Repositories | |
| run: | | |
| $errorActionPreference = "Stop"; $ProgressPreference = "SilentlyContinue"; Set-StrictMode -Version 2.0 | |
| . (Join-Path "." "e2eTests/RemoveRepositories.ps1") -github -githubOwner '${{ needs.Check.outputs.githubowner }}' -token '${{ Secrets.E2EPAT }}' -actionsRepo '${{ needs.SetupRepositories.outputs.actionsRepo }}' -perTenantExtensionRepo '${{ needs.SetupRepositories.outputs.perTenantExtensionRepo }}' -appSourceAppRepo '${{ needs.SetupRepositories.outputs.appSourceAppRepo }}' |