daily-full-build-2201.11.x.yml

name: Daily Full Build Pipeline (2201.11.x)

on:
  workflow_dispatch:
  schedule:
    - cron: '30 18 * * *'   # 00:00 in LK time (GMT+5:30)

env:
  PATCH_LEVEL: 2201.11.x

jobs:
  build-lang:
    name: Build ballerina-lang
    runs-on: ubuntu-latest
    timeout-minutes: 150
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.x'
      - name: Install Python Packages
        run: |
          pip install requests
          pip install httplib2
          pip install colorama
          pip install configobj
          pip install requests
          pip install PyGithub
          pip install cryptography
      - name: Set up JDK 21.0.3
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '21.0.3'
      - name: Setup GraalVM
        uses: graalvm/setup-graalvm@v1
        with:
          java-version: '21.0.3'
          distribution: 'graalvm'
          components: 'native-image'
          github-token: ${{ secrets.GITHUB_TOKEN }}
          set-java-home: 'false'
      - name: Build ballerina-lang
        run: python -u dependabot/full_build_pipeline.py ./ --lang-branch $PATCH_LEVEL --build-level 0
        env:
          packageUser: ${{ secrets.BALLERINA_BOT_USERNAME }}
          packagePAT: ${{ secrets.BALLERINA_BOT_TOKEN }}
          devCentralToken: ${{ secrets.BALLERINA_DEV_CENTRAL_ACCESS_TOKEN }}
          TEST_MODE_ACTIVE: true
      - name: Notify Build Failure
        if: ${{ failure() }}
        shell: bash
        run:
          FILE="failed_modules.txt"

          while read module; do
            python3 dependabot/notify_full_build_failure.py $module $PATCH_LEVEL;
          done < ${FILE}
        env:
          CHAT_ID: ${{ secrets.NOTIFICATIONS_CHAT_ID }}
          CHAT_KEY: ${{ secrets.NOTIFICATIONS_CHAT_KEY }}
          CHAT_TOKEN: ${{ secrets.NOTIFICATIONS_CHAT_TOKEN }}
          ENV_USER_ENCRYPTION_KEY: ${{secrets.USER_ENCRYPTION_KEY}}
          BALLERINA_BOT_TOKEN: ${{ secrets.BALLERINA_BOT_TOKEN }}
      - name: Get lang version
        id: lang-version
        run: |
          VERSION=$((grep -w "version" | cut -d= -f2 | xargs) < ballerina-lang/gradle.properties)
          SPEC_VERSION=$((grep -w "specVersion" | cut -d= -f2 | xargs) < ballerina-lang/gradle.properties)
          echo "::set-output name=version::$VERSION"
          echo "::set-output name=specVersion::$SPEC_VERSION"
      - name: Archive Lang Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Ballerina Lang Artifacts
          path: ~/.m2/
          include-hidden-files: true
    outputs:
      lang-version: ${{ steps.lang-version.outputs.version }}
      spec-version: ${{ steps.lang-version.outputs.specVersion }}

  build-stdlibs:
    name: Build Standard Libraries & Tools
    needs: build-lang
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.x'
      - name: Install Python Packages
        run: |
          pip install requests
          pip install httplib2
          pip install colorama
          pip install configobj
          pip install requests
          pip install PyGithub
          pip install cryptography
      - name: Set up JDK 21.0.3
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '21.0.3'
      - name: Setup NodeJs
        uses: actions/setup-node@v3
        with:
          node-version: 10.22.1
      - name: Download Ballerina Lang Artifacts
        uses: actions/download-artifact@v4
        with:
          name: Ballerina Lang Artifacts
          path: ~/.m2/
      - name: Build Standard Libraries & Tools
        run: |
          python -u dependabot/full_build_pipeline.py ./ --lang-version ${{ needs.build-lang.outputs.lang-version }} \
          --update-stdlib-dependencies --skip-build-distribution --continue-on-error --remove-after-build
        env:
          packageUser: ${{ secrets.BALLERINA_BOT_USERNAME }}
          packagePAT: ${{ secrets.BALLERINA_BOT_TOKEN }}
          devCentralToken: ${{ secrets.BALLERINA_DEV_CENTRAL_ACCESS_TOKEN }}
          CLIENT_ID: ${{ secrets.CLIENT_ID }}
          CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
          REFRESH_TOKEN: ${{ secrets.REFRESH_TOKEN }}
          TEST_MODE_ACTIVE: true
      - name: Archive Standard Library Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Standard Library Artifacts
          path: ~/.m2/
          include-hidden-files: true
      - name: Notify Build Failure
        if: ${{ failure() }}
        shell: bash
        run:
          FILE="failed_modules.txt"

          while read module; do
            python3 dependabot/notify_full_build_failure.py $module $PATCH_LEVEL;
          done < ${FILE}
        env:
          CHAT_ID: ${{ secrets.NOTIFICATIONS_CHAT_ID }}
          CHAT_KEY: ${{ secrets.NOTIFICATIONS_CHAT_KEY }}
          CHAT_TOKEN: ${{ secrets.NOTIFICATIONS_CHAT_TOKEN }}
          ENV_USER_ENCRYPTION_KEY: ${{secrets.USER_ENCRYPTION_KEY}}
          BALLERINA_BOT_TOKEN: ${{ secrets.BALLERINA_BOT_TOKEN }}

  build-distribution:
    name: Build ballerina-distribution
    needs: [build-lang, build-stdlibs]
    runs-on: ubuntu-latest
    timeout-minutes: 150
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
      - name: Setup Python
        uses: actions/setup-python@v2
        with:
          python-version: '3.x'
      - name: Install Python Packages
        run: |
          pip install requests
          pip install httplib2
          pip install colorama
          pip install configobj
          pip install requests
          pip install PyGithub
          pip install cryptography
      - name: Set up JDK 21.0.3
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '21.0.3'
      - name: Download Ballerina Lang Artifacts
        uses: actions/download-artifact@v4
        with:
          name: Ballerina Lang Artifacts
          path: ~/.m2/
      - name: Download Standard Library Artifacts
        uses: actions/download-artifact@v4
        with:
          name: Standard Library Artifacts
          path: ~/.m2/
      - name: Build ballerina-distribution
        run: |
          python -u dependabot/full_build_pipeline.py ./ --lang-version ${{ needs.build-lang.outputs.lang-version }} \
          --update-stdlib-dependencies --only-build-distribution --continue-on-error
        env:
          packageUser: ${{ secrets.BALLERINA_BOT_USERNAME }}
          packagePAT: ${{ secrets.BALLERINA_BOT_TOKEN }}
          devCentralToken: ${{ secrets.BALLERINA_DEV_CENTRAL_ACCESS_TOKEN }}
          TEST_MODE_ACTIVE: true
      - name: Get project version
        id: project-version
        run: |
          SHORT_VERSION=$((grep -w "version" | cut -d= -f2 | cut -d- -f1 | xargs) < ballerina-distribution/gradle.properties)
          DIST_VERSION=$((grep -w "version" | cut -d= -f2 | xargs) < ballerina-distribution/gradle.properties)
          CODE_NAME=$((grep -w 'codeName' | cut -d= -f2 | xargs) < ballerina-distribution/gradle.properties)
          RELEASE_VERSION=$DIST_VERSION-$CODE_NAME
          echo "::set-output name=version::$RELEASE_VERSION"
          echo "::set-output name=sVersion::$SHORT_VERSION"
      - name: Archive Ballerina ZIP
        uses: actions/upload-artifact@v4
        with:
          name: Ballerina ZIP
          path: ballerina-distribution/ballerina/build/distributions/ballerina-*-swan-lake.zip
      - name: Archive Ballerina Short Name ZIP
        uses: actions/upload-artifact@v4
        with:
          name: Ballerina Short Name ZIP
          path: ballerina-distribution/ballerina/build/distributions/ballerina-22*.zip
      - name: Archive MacOS Installer ZIP
        uses: actions/upload-artifact@v4
        with:
          name: MacOS Installer ZIP
          path: ballerina-distribution/ballerina/build/distributions/ballerina-*-macos.zip
      - name: Archive MacOS-ARM Installer ZIP
        uses: actions/upload-artifact@v4
        with:
          name: MacOS-ARM Installer ZIP
          path: ballerina-distribution/ballerina/build/distributions/ballerina-*-macos-arm.zip
      - name: Archive Windows Installer ZIP
        uses: actions/upload-artifact@v4
        with:
          name: Windows Installer ZIP
          path: ballerina-distribution/ballerina/build/distributions/ballerina-*-windows.zip
      - name: Create linux-deb
        id: run_installers_deb
        working-directory: ballerina-distribution/installers/linux-deb
        run: ./build-ballerina-linux-deb-x64.sh -v ${{ steps.project-version.outputs.version }} -p ./../../ballerina/build/distributions
      - name: Create linux-rpm
        id: run_installers_rpm
        working-directory: ballerina-distribution/installers/linux-rpm
        run: ./build-ballerina-linux-rpm-x64.sh -v ${{ steps.project-version.outputs.version }} -p ./../../ballerina/build/distributions
      - name: Archive Linux deb
        uses: actions/upload-artifact@v4
        with:
          name: Linux Installer deb
          path: ballerina-distribution/installers/linux-deb/target/ballerina-*-linux-x64.deb
      - name: Archive Linux rpm
        uses: actions/upload-artifact@v4
        with:
          name: Linux Installer rpm
          path: ballerina-distribution/installers/linux-rpm/rpmbuild/RPMS/x86_64/ballerina-*-linux-x64.rpm
      - name: Install Ballerina DEB
        run: sudo dpkg -i ballerina-distribution/installers/linux-deb/target/ballerina-*-linux-x64.deb
      - name: Update Installer Test Configs
        run: |
          DISPLAY_TEXT=${{ steps.project-version.outputs.sVersion }}
          SWAN_LAKE_LATEST_VERSION="swan-lake-"+$DISPLAY_TEXT
          SPEC_VERSION=${{ needs.build-lang.outputs.spec-version }}
          perl -pi -e "s/^\s*swan-lake-latest-version-display-text=.*/swan-lake-latest-version-display-text=$DISPLAY_TEXT/" ballerina-distribution/ballerina-test-automation/gradle.properties
          perl -pi -e "s/^\s*swan-lake-latest-version=.*/swan-lake-latest-version=$SWAN_LAKE_LATEST_VERSION/" ballerina-distribution/ballerina-test-automation/gradle.properties
          perl -pi -e "s/^\s*swan-lake-latest-spec-version=.*/swan-lake-latest-spec-version=$SPEC_VERSION/" ballerina-distribution/ballerina-test-automation/gradle.properties
      - name: Run Installer Tests
        working-directory: ./ballerina-distribution/ballerina-test-automation/installer-test
        run: ./../gradlew build --stacktrace -scan --console=plain --no-daemon -DballerinaInstalled=true
        env:
          TEST_MODE_ACTIVE: true
      - name: Notify Build Failure
        if: ${{ failure() }}
        shell: bash
        run:
          FILE="failed_modules.txt"

          while read module; do
            python3 dependabot/notify_full_build_failure.py $module $PATCH_LEVEL;
          done < ${FILE}
        env:
          CHAT_ID: ${{ secrets.NOTIFICATIONS_CHAT_ID }}
          CHAT_KEY: ${{ secrets.NOTIFICATIONS_CHAT_KEY }}
          CHAT_TOKEN: ${{ secrets.NOTIFICATIONS_CHAT_TOKEN }}
          ENV_USER_ENCRYPTION_KEY: ${{secrets.USER_ENCRYPTION_KEY}}
          BALLERINA_BOT_TOKEN: ${{ secrets.BALLERINA_BOT_TOKEN }}
    outputs:
      project-version: ${{ steps.project-version.outputs.version }}
      short-version: ${{ steps.project-version.outputs.sVersion }}

  ubuntu-rpm-installer-test:
    needs: [build-lang, build-distribution]
    name: Ubuntu rpm Installer
    runs-on: ubuntu-latest
    container: centos:latest

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v2
      - name: Set up JDK 21.0.3
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '21.0.3'
      - name: Setup Files
        run: |
          cd /etc/yum.repos.d/
          sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
          sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
      - name: Install Git
        run: |
          yum install sudo -y
          sudo dnf install git -y
      - name: Download Ballerina Distribution
        run: |
          git clone https://github.com/ballerina-platform/ballerina-distribution.git
          cd ballerina-distribution
          git checkout $PATCH_LEVEL
      - name: Download Ballerina rpm Installer
        uses: actions/download-artifact@v4
        with:
          name: Linux Installer rpm
      - name: Install Ballerina RPM
        run: rpm -ivh ballerina-*-linux-x64.rpm
      - name: Update Installer Test Configs
        run: |
          DISPLAY_TEXT=${{ needs.build-distribution.outputs.short-version }}
          SWAN_LAKE_LATEST_VERSION="swan-lake-"+$DISPLAY_TEXT
          SPEC_VERSION=${{ needs.build-lang.outputs.spec-version }}
          perl -pi -e "s/^\s*swan-lake-latest-version-display-text=.*/swan-lake-latest-version-display-text=$DISPLAY_TEXT/" ballerina-distribution/ballerina-test-automation/gradle.properties
          perl -pi -e "s/^\s*swan-lake-latest-version=.*/swan-lake-latest-version=$SWAN_LAKE_LATEST_VERSION/" ballerina-distribution/ballerina-test-automation/gradle.properties
          perl -pi -e "s/^\s*swan-lake-latest-spec-version=.*/swan-lake-latest-spec-version=$SPEC_VERSION/" ballerina-distribution/ballerina-test-automation/gradle.properties
      - name: Run Installer Tests
        working-directory: ./ballerina-distribution/ballerina-test-automation/installer-test
        run: ./../gradlew build --stacktrace -scan --console=plain --no-daemon -DballerinaInstalled=true
        env:
          TEST_MODE_ACTIVE: true

  macos-installer-build:
    needs: [build-lang, build-distribution]
    name: Build MacOS Installer
    runs-on: macos-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v2
      - name: Set up JDK 21.0.3
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '21.0.3'
      - name: Download Ballerina Distribution
        run: |
          git clone https://github.com/ballerina-platform/ballerina-distribution.git
          cd ballerina-distribution
          git checkout $PATCH_LEVEL
      - name: Download MacOS Intaller Zip
        uses: actions/download-artifact@v4
        with:
          name: MacOS Installer ZIP
      - name: Create macos-pkg
        id: run_installers_pkg
        working-directory: ballerina-distribution/installers/mac
        run: ./build-ballerina-macos-x64.sh -v ${{ needs.build-distribution.outputs.project-version }} -p ./../../../
      - name: Archive MacOS pkg
        uses: actions/upload-artifact@v4
        with:
          name: MacOS Installer pkg
          path: ballerina-distribution/installers/mac/target/pkg/ballerina-*-macos-x64.pkg
      - name: Install Ballerina PKG
        run: sudo installer -pkg ballerina-distribution/installers/mac/target/pkg/ballerina-*-macos-x64.pkg -target /
      - name: Update Installer Test Configs
        run: |
          DISPLAY_TEXT=${{ needs.build-distribution.outputs.short-version }}
          SWAN_LAKE_LATEST_VERSION="swan-lake-"+$DISPLAY_TEXT
          SPEC_VERSION=${{ needs.build-lang.outputs.spec-version }}
          perl -pi -e "s/^\s*swan-lake-latest-version-display-text=.*/swan-lake-latest-version-display-text=$DISPLAY_TEXT/" ballerina-distribution/ballerina-test-automation/gradle.properties
          perl -pi -e "s/^\s*swan-lake-latest-version=.*/swan-lake-latest-version=$SWAN_LAKE_LATEST_VERSION/" ballerina-distribution/ballerina-test-automation/gradle.properties
          perl -pi -e "s/^\s*swan-lake-latest-spec-version=.*/swan-lake-latest-spec-version=$SPEC_VERSION/" ballerina-distribution/ballerina-test-automation/gradle.properties
      - name: Run Installer Tests
        working-directory: ./ballerina-distribution/ballerina-test-automation/installer-test
        run: ./../gradlew build --stacktrace -scan --console=plain --no-daemon -DballerinaInstalled=true
        env:
          TEST_MODE_ACTIVE: true

  macos-arm-installer-build:
    needs: [build-lang, build-distribution]
    name: Build MacOS ARM Installer
    runs-on: macos-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v2
      - name: Set up JDK 21.0.3
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '21.0.3'
      - name: Download Ballerina Distribution
        run: |
          git clone https://github.com/ballerina-platform/ballerina-distribution.git
          cd ballerina-distribution
          git checkout $PATCH_LEVEL
      - name: Download MacOS-ARM Intaller Zip
        uses: actions/download-artifact@v4
        with:
          name: MacOS-ARM Installer ZIP
      - name: Create macos-pkg
        id: run_installers_pkg
        working-directory: ballerina-distribution/installers/mac
        run: ./build-ballerina-macos-x64.sh -v ${{ needs.build-distribution.outputs.project-version }} -p ./../../../ -a arm
      - name: Archive MacOS pkg
        uses: actions/upload-artifact@v4
        with:
          name: MacOS Installer pkg
          path: ballerina-distribution/installers/mac/target/pkg/ballerina-*-macos-arm-x64.pkg

  windows-installer-build:
    needs: [build-lang, build-distribution]
    name: Build Windows Installer
    runs-on: windows-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v2
      - name: Set up JDK 21.0.3
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '21.0.3'
      - uses: actions/setup-dotnet@v1
        with:
          dotnet-version: '2.1.x'
      - name: Install GUID Generator
        run: dotnet tool install -g dotnet-guid --version 0.5.2
      - name: Download Ballerina Distribution
        run: |
          git clone https://github.com/ballerina-platform/ballerina-distribution.git
          cd ballerina-distribution
          git checkout $env:PATCH_LEVEL
      - name: Download Windows Intaller Zip
        uses: actions/download-artifact@v4
        with:
          name: Windows Installer ZIP
      - name: Create windows-msi
        id: run_installers_msi
        run: |
          $env:Path += ";C:\Program Files (x86)\WiX Toolset v3.11\bin"
          move ballerina-distribution\installers\windows .\
          ren windows w
          cd w
          .\build-ballerina-windows-x64.bat --version ${{ needs.build-distribution.outputs.project-version }} --path .\..\
      - name: Archive Windows msi
        uses: actions/upload-artifact@v4
        with:
          name: Windows Installer msi
          path: w\target\msi\ballerina-*-windows-x64.msi
      - name: Install Ballerina msi
        run: msiexec /i w\target\msi\ballerina-${{ needs.build-distribution.outputs.project-version }}-windows-x64.msi /quiet /qr
        shell: cmd
      - name: Update Installer Test Configs
        run: |
          set DISPLAY_TEXT=${{ needs.build-distribution.outputs.short-version }}
          set SWAN_LAKE_LATEST_VERSION=swan-lake-%DISPLAY_TEXT%
          set SPEC_VERSION=${{ needs.build-lang.outputs.spec-version }}
          perl -pi -e "s/^\s*swan-lake-latest-version-display-text=.*/swan-lake-latest-version-display-text=%DISPLAY_TEXT%/" ballerina-distribution/ballerina-test-automation/gradle.properties
          perl -pi -e "s/^\s*swan-lake-latest-version=.*/swan-lake-latest-version=%SWAN_LAKE_LATEST_VERSION%/" ballerina-distribution/ballerina-test-automation/gradle.properties
          perl -pi -e "s/^\s*swan-lake-latest-spec-version=.*/swan-lake-latest-spec-version=%SPEC_VERSION%/" ballerina-distribution/ballerina-test-automation/gradle.properties
        shell: cmd
      - name: Run Installer Tests
        working-directory: .\ballerina-distribution\ballerina-test-automation\installer-test
        run: |
          $env:Path += ";C:\Program Files\Ballerina\bin"
          .\..\gradlew build --stacktrace -scan --console=plain --no-daemon -DballerinaInstalled=true
        env:
          TEST_MODE_ACTIVE: true

  nballerina-build:
    needs: build-distribution
    name: Build nBallerina
    runs-on: ubuntu-latest

    steps:
      - name: Checkout Repository
        uses: actions/checkout@v2
      - name: Set up JDK 21.0.3
        uses: actions/setup-java@v3
        with:
          distribution: 'temurin'
          java-version: '21.0.3'
      - name: Install Python Packages
        run: |
          pip install requests
          pip install retry
          pip install PyGithub
          pip install cryptography
          pip install httplib2
      - name: Clone nBallerina repository
        run: git clone https://github.com/ballerina-platform/nballerina.git
      - name: Download Linux Deb Installer
        uses: actions/download-artifact@v4
        with:
          name: Linux Installer deb
      - name: Install Ballerina DEB
        run: sudo dpkg -i ballerina-*-linux-x64.deb
      - name: Run compiler tests
        working-directory: nballerina/compiler
        run: bal test --disable-groups private
      - name: Notify Build Failure
        if: ${{ failure() }}
        shell: bash
        run: python3 dependabot/notify_full_build_failure.py nballerina $PATCH_LEVEL
        env:
          CHAT_ID: ${{ secrets.NOTIFICATIONS_CHAT_ID }}
          CHAT_KEY: ${{ secrets.NOTIFICATIONS_CHAT_KEY }}
          CHAT_TOKEN: ${{ secrets.NOTIFICATIONS_CHAT_TOKEN }}
          ENV_USER_ENCRYPTION_KEY: ${{secrets.USER_ENCRYPTION_KEY}}
          BALLERINA_BOT_TOKEN: ${{ secrets.BALLERINA_BOT_TOKEN }}