setupOpenLibertyAks.yml

name: Setup OpenLiberty on AKS

on:
  workflow_dispatch:
    inputs:
      disambiguationSuffix:
        description: "Included in names to disambiguate. Get from another pipeline execution"
        required: true
      region:
        description: "Azure region for resources"
        required: true
        default: westus
      deleteResource:
        description: "True to delete all the Azure resources after the deployments. False to keep Azure resources."
        default: "true"
        required: true
  repository_dispatch:
  schedule:
    - cron: '0 0 27 * *' # run the workflow at the end of 27th monthly.

env:
  refArmttk: c11a62d4ae011ee96fdecc76d76d811c5b5a99ce
  refJavaee: 931ce903b85a4bf1fc79c763a92ac3666f10a1e0
  appInsightsName: appinsights${{ github.run_id }}${{ github.run_number }}
  azCliVersion: 2.60.0
  azureCredentials: ${{ secrets.AZURE_CREDENTIALS }}
  dbName: libertydb${{ github.run_id }}${{ github.run_number }}
  dbAdminUser: liberty
  dbPassword: ${{ secrets.DB_PASSWORD }}
  dbServerName: libertydbs
  logAnalyticsName: loga${{ github.run_id }}${{ github.run_number }}
  namespace: default
  replicas: 3
  resourceGroupForDB: liberty-db-${{ github.run_id }}-${{ github.run_number }}
  aksRepoUserName: WASdev
  aksRepoBranchName: 048e776e9efe2ffed8368812e198c1007ba94b2c

jobs:
  # Make it so the bicep file that causes Liberty on AKS to be deployed is available to this workflow.
  preflight:
    runs-on: ubuntu-20.04
    outputs:
        deleteResource: ${{steps.setup-env-variables.outputs.deleteResource}}
        location: ${{ steps.setup-env-variables.outputs.location }}
        resourceGroupForOpenLibertyAks: ol-aks-${{ steps.setup-env-variables.outputs.disambiguationSuffix }}-${{ github.run_number }}
    steps:      
      - name: Set up ENV variables
        id: setup-env-variables
        run: |
          deleteResource=${{ github.event.inputs.deleteResource }}
          if [ -z ${deleteResource} ]; then
            deleteResource='true'
          fi
          echo "deleteResource: ${deleteResource}"

          location=${{ github.event.inputs.region }}
          if [[ -z "${location}" ]];then
            location=eastus
          fi

          disambiguationSuffix=${{ github.event.inputs.disambiguationSuffix }}
          if [[ -z "${disambiguationSuffix}" ]];then
            disambiguationSuffix=$(date +%m%d)
          fi

          echo "deleteResource=${deleteResource}" >> $GITHUB_OUTPUT
          echo "location=${location}" >> $GITHUB_OUTPUT
          echo "disambiguationSuffix=${disambiguationSuffix}" >> $GITHUB_OUTPUT
      - name: Set up bicep
        run: |
          curl -Lo bicep https://github.com/Azure/bicep/releases/download/v0.29.47/bicep-linux-x64
          chmod +x ./bicep
          sudo mv ./bicep /usr/local/bin/bicep
          bicep --version
      - name: Checkout arm-ttk
        uses: actions/checkout@v2
        with:
          repository: Azure/arm-ttk
          path: arm-ttk
          ref: ${{ env.refArmttk }}
      # Check out the repository containing the code to deploy Liberty on AKS.
      - name: Checkout ${{ env.aksRepoUserName }}/azure.liberty.aks
        uses: actions/checkout@v2
        with:
          repository: ${{ env.aksRepoUserName }}/azure.liberty.aks
          path: azure.liberty.aks
          ref: ${{ env.aksRepoBranchName }}
      - name: Set up JDK 17
        uses: actions/setup-java@v1
        with:
          java-version: 17
          server-username: PACKAGES_ACCESS_USERNAME
          server-password: PACKAGES_ACCESS_TOKEN
      # Generate the Liberty on AKS bicep template so it can be invoked later. The bicep template is enclosed in a zip file
      - name: Build and test ${{ env.aksRepoUserName }}/azure.liberty.aks
        env:
          PACKAGES_ACCESS_USERNAME: github
          PACKAGES_ACCESS_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          cd azure.liberty.aks
          mvn -Pbicep -Ddev -Passembly -Ptemplate-validation-tests clean install
      - name: Generate artifact file name and path
        id: artifact_file
        run: |
          version=$(mvn -q -Dexec.executable=echo -Dexec.args='${version.azure.liberty.aks}' --file azure.liberty.aks/pom.xml  --non-recursive exec:exec)
          artifactName=azure.liberty.aks-$version-arm-assembly
          unzip azure.liberty.aks/target/$artifactName.zip -d azure.liberty.aks/target/$artifactName
          echo "artifactName=${artifactName}" >> $GITHUB_OUTPUT
          echo "artifactPath=azure.liberty.aks/target/$artifactName" >> $GITHUB_OUTPUT
      # Make the contents of the zip file available for use later in the workflow.
      - name: Archive azure.liberty.aks template
        uses: actions/upload-artifact@v1
        if: success()
        with:
          name: ${{steps.artifact_file.outputs.artifactName}}
          path: ${{steps.artifact_file.outputs.artifactPath}}
  # Set up database for use by application.
  deploy-db:
    needs: preflight
    runs-on: ubuntu-latest
    steps:
      - name: Set variables
        run: |
          echo "location=${{ needs.preflight.outputs.location }}" >> $GITHUB_ENV
      - uses: azure/login@v1
        id: azure-login
        with:
          creds: ${{ env.azureCredentials }}
      - name: Create Resource Group
        uses: azure/CLI@v1
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            echo "create resource group" ${{ env.resourceGroupForDB }}
            az group create --verbose --name ${{ env.resourceGroupForDB }} --location ${location}
      - name: Set Up Azure Postgresql to Test dbTemplate
        id: setup-postgresql
        uses: azure/CLI@v1
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            echo "Deploy DB with name " ${{ env.dbName }}
            az postgres flexible-server create \
            --resource-group ${{ env.resourceGroupForDB }} \
            --name ${{ env.dbName }} \
            --location ${location} \
            --admin-user ${{ env.dbAdminUser }} \
            --admin-password ${{ env.dbPassword }} \
            --version 16 \
            --public-access 0.0.0.0 \
            --tier Burstable \
            --sku-name Standard_B1ms \
            --yes
            az postgres flexible-server db create \
              --resource-group ${{ env.resourceGroupForDB }} \
              --server-name ${{ env.dbName }} \
              --database-name ${{ env.dbServerName }}
            sleep 1m
            echo "Allow Access To Azure Services"
            az postgres flexible-server firewall-rule create \
              -g ${{ env.resourceGroupForDB }} \
              -n ${{ env.dbName }} \
              -r "AllowAllWindowsAzureIps" \
              --start-ip-address "0.0.0.0" \
              --end-ip-address "0.0.0.0"

            az postgres flexible-server parameter set \
              --name max_prepared_transactions \
              --value 10 \
              -g ${{ env.resourceGroupForDB }} \
              --server-name ${{ env.dbName }}

            az postgres flexible-server restart \
              -g ${{ env.resourceGroupForDB }} \
              --name ${{ env.dbName }}
            
  # Invoke the solution template, passing the necessary parameters to identify the configuration of AKS.
  deploy-openliberty-on-aks:
    needs: preflight
    runs-on: ubuntu-20.04
    steps:
      - name: Set variables
        run: |
          echo "location=${{ needs.preflight.outputs.location }}" >> $GITHUB_ENV
          echo "resourceGroupForOpenLibertyAks=${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}" >> $GITHUB_ENV
      - name: Checkout ${{ env.aksRepoUserName }}/azure.liberty.aks
        uses: actions/checkout@v2
        with:
          repository: ${{ env.aksRepoUserName }}/azure.liberty.aks
          path: azure.liberty.aks
          ref: ${{ env.aksRepoBranchName }}
      - name: Get version information from azure.liberty.aks/pom.xml
        id: version
        run: |
          version=$(mvn -q -Dexec.executable=echo -Dexec.args='${version.azure.liberty.aks}' --file azure.liberty.aks/pom.xml  --non-recursive exec:exec)
          echo "version=${version}" >> $GITHUB_ENV
      - name: Output artifact name for Download action
        id: artifact_file
        run: |
          artifactName=azure.liberty.aks-$version-arm-assembly
          echo "artifactName=${artifactName}" >> $GITHUB_ENV
          echo "artifactName=${artifactName}" >> $GITHUB_OUTPUT
      - name: Download artifact for deployment
        uses: actions/download-artifact@v1
        with:
          name: ${{steps.artifact_file.outputs.artifactName}}
      - uses: azure/login@v1
        id: azure-login
        with:
          creds: ${{ env.azureCredentials }}
      - name: Create Resource Group
        uses: azure/CLI@v1
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            echo "create resource group" ${resourceGroupForOpenLibertyAks}
            az group create --verbose --name ${resourceGroupForOpenLibertyAks} --location ${location}
      - name: Checkout cargotracker
        uses: actions/checkout@v2
        with:
          path: cargotracker
      - name: Prepare parameter file
        run: |
          echo "replace placeholders using real parameter"
          sed -i "s/#location#/${location}/g; \
                  s/#testbranchName#/${aksRepoBranchName}/g; \
                  s/#gitUserName#/${aksRepoUserName}/g" \
                  cargotracker/src/test/aks/parameters.json
      - name: Archive parameters.json
        uses: actions/upload-artifact@v1
        with:
          name: archivefiles
          path: cargotracker/src/test/aks/parameters.json
      # Validate the parameters file in the context of the solution template to be invoked. This will catch some errors before taking the time to start the full deployment.
      - name: Validate Deploy of Open Liberty Server Cluster Domain offer
        id: validate-openliberty-cluster-deployment
        uses: azure/CLI@v1
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            az deployment group validate \
            --debug \
            --resource-group ${resourceGroupForOpenLibertyAks} \
            --name openliberty-on-aks \
            --parameters @cargotracker/src/test/aks/parameters.json \
            --template-file ${artifactName}/mainTemplate.json
      - name: Deploy Open Liberty Server Cluster Domain offer
        id: deploy-openliberty-cluster
        uses: azure/CLI@v1
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            az deployment group create \
            --verbose \
            --resource-group ${resourceGroupForOpenLibertyAks} \
            --name openliberty-on-aks \
            --parameters @cargotracker/src/test/aks/parameters.json \
            --template-file ${artifactName}/mainTemplate.json
  # Set up monitoring resources
  deploy-azure-monitor:
    needs: [preflight, deploy-openliberty-on-aks]
    runs-on: ubuntu-20.04
    steps:
      - name: Set variables
        run: |
          echo "location=${{ needs.preflight.outputs.location }}" >> $GITHUB_ENV
          echo "resourceGroupForOpenLibertyAks=${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}" >> $GITHUB_ENV
      - uses: azure/login@v1
        id: azure-login
        with:
          creds: ${{ env.azureCredentials }}
      - name: Deploy Log Analytics Workspace
        id: deploy-log-analytics
        uses: azure/CLI@v1
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            az monitor log-analytics workspace create \
              --resource-group ${resourceGroupForOpenLibertyAks} \
              --workspace-name ${{ env.logAnalyticsName }} \
              --location ${location}
      - name: Enable Container Insights
        id: enable-container-insights
        uses: azure/CLI@v1
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            workspaceId=$(az monitor log-analytics workspace list -g ${resourceGroupForOpenLibertyAks} --query '[0].id' -o tsv)
            aksClusterName=$(az aks list  -g ${resourceGroupForOpenLibertyAks} --query [0].name -o tsv)

            az aks enable-addons \
              --addons monitoring \
              --name ${aksClusterName} \
              --resource-group ${resourceGroupForOpenLibertyAks} \
              --workspace-resource-id ${workspaceId}
      - name: Provision Application Insights
        id: provision-app-insights
        uses: azure/CLI@v1
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            az extension add --upgrade -n application-insights

            workspaceId=$(az monitor log-analytics workspace list -g ${resourceGroupForOpenLibertyAks} --query '[0].id' -o tsv)
            az monitor app-insights component create \
              --resource-group ${resourceGroupForOpenLibertyAks} \
              --app ${{ env.appInsightsName }} \
              --location ${location} \
              --workspace ${workspaceId}
  # Build app, push to ACR and apply it to Open Liberty servers running on AKS.
  deploy-cargo-tracker:
    needs: [preflight, deploy-db,deploy-azure-monitor]
    runs-on: ubuntu-20.04
    steps:
      - name: Set variables
        run: |
          echo "location=${{ needs.preflight.outputs.location }}" >> $GITHUB_ENV
          echo "resourceGroupForOpenLibertyAks=${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}" >> $GITHUB_ENV
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          distribution: 'microsoft'
          java-version: '17'
      - name: Install jq
        run: |
          sudo apt-get install jq -y
      - uses: azure/login@v1
        id: azure-login
        with:
          creds: ${{ env.azureCredentials }}
      # Obtain AKS and ACR resource properties.
      - name: Prepare variables
        id: prepare_variables
        run: |
          echo "Query AKS cluster name"
          aksClusterName=$(az aks list  -g ${resourceGroupForOpenLibertyAks} --query [0].name -o tsv)
          if [ -z "${aksClusterName}" ] ; then
            echo "Failed to obtain AKS. Exiting."
            exit 1
          fi
          echo "aksClusterName=${aksClusterName}" >> $GITHUB_ENV

          acrName=$(az acr list  -g ${resourceGroupForOpenLibertyAks} --query [0].name -o tsv)
          if [ -z "${acrName}" ] ; then
            echo "Falied to obtain ACR. Exiting."
            exit 1
          fi

          azureACRServer=$(az acr show -n $acrName -g ${resourceGroupForOpenLibertyAks} --query 'loginServer' -o tsv)
          azureACRUserName=$(az acr credential show -n $acrName -g ${resourceGroupForOpenLibertyAks} --query 'username' -o tsv)
          azureACRPassword=$(az acr credential show -n $acrName -g ${resourceGroupForOpenLibertyAks} --query 'passwords[0].value' -o tsv)
          
          az extension add --upgrade -n application-insights
          appInsightsConnectionString=$(az monitor app-insights component show \
            --resource-group ${resourceGroupForOpenLibertyAks} \
            --query '[0].connectionString' -o tsv)

          echo "azureACRServer=${azureACRServer}" >> $GITHUB_OUTPUT
          echo "azureACRUserName=${azureACRUserName}" >> $GITHUB_OUTPUT
          echo "azureACRPassword=${azureACRPassword}" >> $GITHUB_OUTPUT
          echo "appInsightsConnectionString=${appInsightsConnectionString}" >> $GITHUB_OUTPUT
      - name: Checkout cargotracker
        uses: actions/checkout@v2
        with:
          path: cargotracker
      # Build Cargo Tracker with DB and JMS configuration
      - name: Build the app
        run: |
          echo "build the Cargo Tracker web app"
          echo "setup environment variables"
          export LOGIN_SERVER=${{steps.prepare_variables.outputs.azureACRServer}}
          export USER_NAME=${{steps.prepare_variables.outputs.azureACRUserName}}
          export PASSWORD=${{steps.prepare_variables.outputs.azureACRPassword}}
          export DB_SERVER_NAME=${{ env.dbName }}.postgres.database.azure.com
          export DB_PORT_NUMBER=5432
          export DB_NAME=${{env.dbServerName}}
          export DB_USER=${{ env.dbAdminUser }}
          export DB_PASSWORD=${{ env.dbPassword }}
          export NAMESPACE=${{ env.namespace }}
          export APPLICATIONINSIGHTS_CONNECTION_STRING=${{steps.prepare_variables.outputs.appInsightsConnectionString}}
          mvn clean install -PopenLibertyOnAks --file cargotracker/pom.xml
      - name: Query version string for deployment verification
        run: |
          PROPERTY_FILE="cargotracker/target/cargo-tracker/WEB-INF/classes/org/eclipse/cargotracker/messages.properties"
          PROP_KEY=versionString
          deployVersion=$(cat $PROPERTY_FILE | grep "$PROP_KEY" | cut -d '=' -f 2)
          echo "deployVersion=${deployVersion}" >> $GITHUB_ENV
      - name: Build image and upload to ACR
        run: |
          echo "get image name and version"
          cd cargotracker/
          IMAGE_NAME=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.artifactId}' --non-recursive exec:exec) 
          IMAGE_VERSION=$(mvn -q -Dexec.executable=echo -Dexec.args='${project.version}' --non-recursive exec:exec)

          echo "build image and upload"
          cd target/
          docker build -t ${IMAGE_NAME}:${IMAGE_VERSION} --pull --file=Dockerfile .
          docker tag ${IMAGE_NAME}:${IMAGE_VERSION} ${{steps.prepare_variables.outputs.azureACRServer}}/${IMAGE_NAME}:${IMAGE_VERSION}
          docker login -u ${{steps.prepare_variables.outputs.azureACRUserName}} -p ${{steps.prepare_variables.outputs.azureACRPassword}} ${{steps.prepare_variables.outputs.azureACRServer}}

          docker push ${{steps.prepare_variables.outputs.azureACRServer}}/${IMAGE_NAME}:${IMAGE_VERSION}
      - name: Connect to AKS cluster
        run: |
          echo "connect to cluster"
          az aks get-credentials --resource-group ${resourceGroupForOpenLibertyAks} --name ${aksClusterName}
      - name: Apply deployment files
        run: |
          echo "record time before restart"
          timestampBeforePatchingDomain=$(date +%s)
          cd cargotracker/target/
          kubectl apply -f db-secret.yaml
          kubectl apply -f app-insight.yaml
          kubectl apply -f openlibertyapplication.yaml
          # restart every time to make sure
          sleep 30
          kubectl rollout restart deployment/cargo-tracker-cluster
          echo "timestampBeforePatchingDomain=${timestampBeforePatchingDomain}" >> $GITHUB_ENV
      # Make sure all the pods are running.
      - name: Verify pods are ready
        run: |
          # interval of checking pod status.
          checkPodStatusInterval=20
          # max attempt to check pod status.
          checkPodStatusMaxAttempts=30
          updatedPodNum=0
          attempt=0
          echo $timestampBeforePatchingDomain ${{ env.replicas }} $checkPodStatusMaxAttempts $checkPodStatusInterval
          while [[ ${updatedPodNum} -ne ${{ env.replicas }} ]] && [[ $attempt -le ${checkPodStatusMaxAttempts} ]]; do
              echo "attempts ${attempt}"
              ret=$(kubectl get pods -n ${{ env.namespace }} -o json | jq -r '.items[] | select(.metadata.name|test("cargo-tracker-cluster.")) | .metadata.creationTimestamp' | tr -d "\"")
              counter=0
              for item in $ret; do
                  podCreateTimeStamp=$(date -u -d "${item}" +"%s")
                  echo "pod create time: $podCreateTimeStamp, base time: ${timestampBeforePatchingDomain}"
                  if [[ ${podCreateTimeStamp} -gt ${timestampBeforePatchingDomain} ]]; then
                      counter=$((counter + 1))
                  fi
              done
              updatedPodNum=$counter
              echo "Number of new pod: ${updatedPodNum}"
              attempt=$((attempt + 1))
              sleep ${checkPodStatusInterval}
          done
          if [[ ${attempt} -gt ${checkPodStatusMaxAttempts} ]]; then
              echo "Failed to restart all pods. "
              exit 1
          fi
      - name: Query Application URL
        run: |
          gatewayPublicIPId=$(az network application-gateway list \
            --resource-group ${resourceGroupForOpenLibertyAks} \
            --query '[0].frontendIPConfigurations[0].publicIPAddress.id' -o tsv)

          gatewayUrl=$(az network public-ip show --ids ${gatewayPublicIPId} --query 'dnsSettings.fqdn' -o tsv)

          appUrl="http://${gatewayUrl}/cargo-tracker/"

          echo "appURL=${appUrl}" >> $GITHUB_ENV
      # Make REST API calls to cause metrics, which will be shown in the Application Insights.
      - name: Make REST API calls
        run: |
          # wait for application ready
          attempt=0
          maxAttempt=50
          statusCode=404

          while [[ $attempt -le ${maxAttempt} ]] && [[ "${statusCode}" != "200" ]]; do
            echo "attempt: ${attempt}"
            statusCode=$(curl -s -o /dev/null -w "%{http_code}" ${appURL})
            echo "http code: ${statusCode}"
            attempt=$((attempt + 1))
            sleep 20
          done

          if [[ ${attempt} -gt ${maxAttempt} ]]; then
              echo "Failed to access ${appURL}. "
              exit 1
          fi
         
          # get request
          curl -X GET -H "Accept: application/json" "${appURL}rest/graph-traversal/shortest-path?origin=CNHKG&destination=USNYC"
          
          # post request
          currentDateTime=$(date +'%m/%d/%Y %I:%M %p')
          cat <<EOF >data.json
          {
            "completionTime": "${currentDateTime}",
            "trackingId": "ABC123",
            "eventType": "UNLOAD",
            "unLocode": "USNYC",
            "voyageNumber": "0200T"
          }
          EOF
          curl -X POST -d "@data.json" -H "Content-Type: application/json" ${appURL}rest/handling/reports
          
          # Datetime format Failure
          currentDateTime=$(date +'%m/%d/%Y %H:%M:%S')
          cat <<EOF >data.json
          {
            "completionTime": "${currentDateTime}",
            "trackingId": "ABC123",
            "eventType": "UNLOAD",
            "unLocode": "USNYC",
            "voyageNumber": "0200T"
          }
          EOF
          curl -X POST -d "@data.json" -H "Content-Type: application/json" ${appURL}rest/handling/reports
      
      # Print app URL to the pipeline summary page.
      - name: Print app URL
        run: |
          echo "${appURL}" >> $GITHUB_STEP_SUMMARY
 # Delete azure resources
  cleanup:
    if: always()
    needs: [preflight, deploy-db, deploy-openliberty-on-aks, deploy-azure-monitor, deploy-cargo-tracker]
    runs-on: ubuntu-latest
    steps:
      - uses: azure/login@v1
        id: azure-login
        with:
          creds: ${{ env.azureCredentials }}
      - name: Delete Azure resources.
        uses: azure/CLI@v1
        if: ${{ needs.preflight.outputs.deleteResource == 'true' }}
        with:
          azcliversion: ${{ env.azCliVersion }}
          inlineScript: |
            echo "delete... " ${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}
            az group delete --yes --no-wait --verbose --name ${{ needs.preflight.outputs.resourceGroupForOpenLibertyAks }}

            echo "delete... " ${{ env.resourceGroupForDB }}
            az group delete --yes --no-wait --verbose --name ${{ env.resourceGroupForDB }}
# File error message to specified Teams channel.
  alert:
    if: failure()
    needs: [preflight, deploy-db, deploy-openliberty-on-aks, deploy-azure-monitor, deploy-cargo-tracker, cleanup]
    runs-on: ubuntu-latest
    steps:
      - name: Send Teams message
        if: ${{ github.repository_owner == 'azure-javaee' }}
        run: |
          echo "Job failed, send notification to Teams"
          # https://learn.microsoft.com/en-us/microsoftteams/platform/webhooks-and-connectors/how-to/add-incoming-webhook?tabs=dotnet
          curl ${{ secrets.MSTEAMS_WEBHOOK }} \
            -H 'Content-Type: application/json' \
            --data-binary @- << EOF
            {
            "@context":"http://schema.org/extensions",
            "@type":"MessageCard",
            "text":"Workflow failed in cargotracker-liberty-aks repository, please take a look at: https://github.com/${GITHUB_REPOSITORY}/actions/runs/${{ github.run_id }}"
            }
          EOF