Merge pull request #4422 from Rohde-Schwarz/chore/repo_config

.github/actions/read-repo-config/action.yml

@@ -0,0 +1,25 @@
+
+# (C) 2024 Jack Lloyd
+# (C) 2024 René Meusel, Rohde & Schwarz Cybersecurity
+#
+# Botan is released under the Simplified BSD License (see license.txt)
+
+name: Read Repository Configuration
+description: Reads the repository configuration file and makes it available as environment variables
+
+runs:
+  using: composite
+  steps:
+    - name: Read Repository Configuration into Environment Variables
+      run: |
+        python3 ${{ github.action_path }}/../../../src/scripts/repo_config.py all >> $GITHUB_ENV
+        echo "REPO_CONFIG_LOADED=true" >> $GITHUB_ENV
+      shell: bash
+      if: runner.os != 'Windows'
+
+    - name: Read Repository Configuration into Environment Variables
+      run: |
+        python3 ${{ github.action_path }}/../../../src/scripts/repo_config.py all >> $GITHUB_ENV
+        echo "REPO_CONFIG_LOADED=true" >> $GITHUB_ENV
+      shell: pwsh
+      if: runner.os == 'Windows'

.github/workflows/ci.yml

@@ -48,6 +48,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:
@@ -79,6 +82,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:
@@ -113,6 +119,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:
@@ -128,6 +137,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:
@@ -185,11 +197,14 @@ jobs:
         with:
           path: ./source
 
+      - name: Read Repository Configuration
+        uses: ./source/.github/actions/read-repo-config
+
       - name: Fetch BoringSSL fork for BoGo tests
         uses: actions/checkout@v4
         with:
-          repository: randombit/boringssl
-          ref: rene/runner-20241016
+          repository: ${{ env.BORINGSSL_REPO }}
+          ref: ${{ env.BORINGSSL_BRANCH }}
           path: ./boringssl
         if: matrix.target == 'coverage' || matrix.target == 'sanitizer'
 
@@ -232,6 +247,9 @@ jobs:
         with:
           path: ./source
 
+      - name: Read Repository Configuration
+        uses: ./source/.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./source/.github/actions/setup-build-agent
         with:
@@ -276,12 +294,12 @@ jobs:
 
     runs-on: ${{ matrix.host_os }}
 
-    env:
-      ANDROID_NDK: android-ndk-r26
-
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:

.github/workflows/codeql.yml

@@ -26,6 +26,9 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:

.github/workflows/nightly.yml

@@ -44,11 +44,14 @@ jobs:
         with:
           path: ./source
 
+      - name: Read Repository Configuration
+        uses: ./source/.github/actions/read-repo-config
+
       - name: Fetch BoringSSL fork for BoGo tests
         uses: actions/checkout@v4
         with:
-          repository: randombit/boringssl
-          ref: rene/runner-20241016
+          repository: ${{ env.BORINGSSL_REPO }}
+          ref: ${{ env.BORINGSSL_BRANCH }}
           path: ./boringssl
 
       - name: Setup Build Agent
@@ -109,12 +112,12 @@ jobs:
 
     runs-on: ${{ matrix.host_os }}
 
-    env:
-      ANDROID_NDK: android-ndk-r26
-
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:
@@ -132,6 +135,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:
@@ -184,6 +190,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:
@@ -201,6 +210,9 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:
@@ -219,6 +231,9 @@ jobs:
       - name: Fetch Botan Repository
         uses: actions/checkout@v4
 
+      - name: Read Repository Configuration
+        uses: ./.github/actions/read-repo-config
+
       - name: Setup Build Agent
         uses: ./.github/actions/setup-build-agent
         with:

doc/dev_ref/continuous_integration.rst

@@ -8,6 +8,38 @@ The Github Actions builds are orchestrated using a script
 ``src/scripts/ci_build.py``. This allows one to easily reproduce the CI process
 on a local machine.
 
+Repository Configuration
+------------------------
+
+Specific configuration for test dependencies and CI-related global settings
+are centralized in ``src/configs/repo_config.env``. This file is pulled into
+the CI environment using the python script ``src/scripts/repo_config.py``.
+
+If one needs direct access to the configuration variables (without relying on
+environment variables in CI), use ``src/scripts/repo_config.py`` in one of the
+following ways:
+
+1. From the command line:
+
+   .. code-block:: bash
+
+      # print all key-value pairs, like: VAR=VALUE\n...
+      python3 src/scripts/repo_config.py all
+
+      # print the value of a specific key
+      python3 src/scripts/repo_config.py get VAR
+
+      # list all available variables in repo_config.env
+      python3 src/scripts/repo_config.py list
+
+2. As a python module (assuming the script is in the PYTHONPATH):
+
+   .. code-block:: python
+
+      from repo_config import RepoConfig
+      config = RepoConfig()
+      print(config['VAR'])
+
 Github Actions
 ---------------
 

src/configs/repo_config.env

@@ -0,0 +1,36 @@
+# This file contains configurations that are relevant for the entire repository
+#
+# The variables defined in this file are made available as environment variables
+# in the CI jobs.
+#
+# If one needs to read these variables in a script, they can be accessed via
+# src/scripts/repo_config.py. For example, to read the value of BORINGSSL_REPO,
+# one can use the following command:
+#
+#     python3 src/scripts/repo_config.py get BORINGSSL_REPO
+#
+# The settings in this file _explicitly do not_ affect the build of the shipped
+# production code or library. Such settings may be found in src/build-data.
+
+# The fork of boringssl that should be used for BoGo tests
+BORINGSSL_REPO="randombit/boringssl"
+
+# The branch in our fork of boringssl that should be used for BoGo tests
+BORINGSSL_BRANCH="rene/runner-20241016"
+
+# The Android NDK to for the cross platform builds to Android
+ANDROID_NDK="android-ndk-r26"
+
+# Jitterentropy library version to be used for testing the 'jitter_rng' module
+JITTERENTROPY_VERSION="3.6.0"
+
+# The version of the Intel SDE tool to use for running the Intel SDE tests
+INTEL_SDE_VERSION="sde-external-9.38.0-2024-04-18-lin"
+
+# Limbo test suite revision to be used in run_limbo_tests.py
+LIMBO_TEST_SUITE_REVISION="f98aa03f45d108ae4e1bc5a61ec4bd0b8d137559"
+
+# The maximum size of the compiler cache in CI
+# Those variables are directly consumed by ccache and sccache respectively
+CCACHE_MAXSIZE="200M"
+SCCACHE_CACHE_SIZE="200M"

src/scripts/ci/setup_gh_actions.ps1

@@ -31,5 +31,3 @@ if($identifiers_for_64bit -contains $ARCH ) {
 } else {
     echo "VSENV_ARCH=$ARCH" >> $env:GITHUB_ENV
 }
-
-echo "SCCACHE_CACHE_SIZE=200M" >> $env:GITHUB_ENV

src/scripts/ci/setup_gh_actions.sh

@@ -18,7 +18,7 @@ SCRIPT_LOCATION=$(cd "$(dirname "$0")"; pwd)
 
 function build_and_install_jitterentropy() {
     mkdir jitterentropy-library
-    curl -L https://github.com/smuellerDD/jitterentropy-library/archive/refs/tags/v3.6.0.tar.gz | tar -xz -C .
+    curl -L "https://github.com/smuellerDD/jitterentropy-library/archive/refs/tags/v${JITTERENTROPY_VERSION}.tar.gz" | tar -xz -C .
     jel_dir="$(realpath jitterentropy-library-*)"
     cmake -B "${jel_dir}/build" -S "${jel_dir}" -DCMAKE_BUILD_TYPE=Release
     cmake --build "${jel_dir}/build"
@@ -27,6 +27,11 @@ function build_and_install_jitterentropy() {
     rm -rf "${jel_dir}"
 }
 
+if [ -z $REPO_CONFIG_LOADED ]; then
+    echo "Repository configuration not loaded" >&2
+    exit 1
+fi
+
 if type -p "apt-get"; then
     # TPM2-TSS library (to build the library against)
     tpm2_specific_packages=("libtss2-dev")
@@ -133,13 +138,11 @@ if type -p "apt-get"; then
         sudo apt-get -qq install qemu-user g++-s390x-linux-gnu
 
     elif [ "$TARGET" = "sde" ]; then
-        SDE_VER=sde-external-9.38.0-2024-04-18-lin
-        wget https://downloadmirror.intel.com/823664/${SDE_VER}.tar.xz
-        tar -xvf ${SDE_VER}.tar.xz
-        echo ${SDE_VER} >> "$GITHUB_PATH"
+        wget https://downloadmirror.intel.com/823664/${INTEL_SDE_VERSION}.tar.xz
+        tar -xvf ${INTEL_SDE_VERSION}.tar.xz
 
     elif [ "$TARGET" = "cross-android-arm32" ] || [ "$TARGET" = "cross-android-arm64" ] || [ "$TARGET" = "cross-android-arm64-amalgamation" ]; then
-        wget -nv https://dl.google.com/android/repository/"$ANDROID_NDK"-linux.zip
+        wget -nv "https://dl.google.com/android/repository/${ANDROID_NDK}-linux.zip"
         unzip -qq "$ANDROID_NDK"-linux.zip
 
     elif [ "$TARGET" = "cross-arm32-baremetal" ]; then
@@ -156,7 +159,7 @@ if type -p "apt-get"; then
 
     elif [ "$TARGET" = "limbo" ]; then
         sudo apt-get -qq install python3-dateutil
-        wget -nv https://raw.githubusercontent.com/C2SP/x509-limbo/f98aa03f45d108ae4e1bc5a61ec4bd0b8d137559/limbo.json -O "${SCRIPT_LOCATION}/../../../limbo.json"
+        wget -nv "https://raw.githubusercontent.com/C2SP/x509-limbo/${LIMBO_TEST_SUITE_REVISION}/limbo.json" -O "${SCRIPT_LOCATION}/../../../limbo.json"
 
     elif [ "$TARGET" = "coverage" ] || [ "$TARGET" = "sanitizer" ]; then
         if [ "$TARGET" = "coverage" ]; then
@@ -213,5 +216,3 @@ if type -p "ccache"; then
     cache_location="$( ccache --get-config cache_dir )"
     echo "COMPILER_CACHE_LOCATION=${cache_location}" >> "${GITHUB_ENV}"
 fi
-
-echo "CCACHE_MAXSIZE=200M" >> "${GITHUB_ENV}"

src/scripts/ci_build.py

@@ -767,6 +767,7 @@ def main(args=None):
             'src/scripts/test_python.py',
             'src/scripts/test_fuzzers.py',
             'src/scripts/test_cli.py',
+            'src/scripts/repo_config.py',
             'src/scripts/python_unittests.py',
             'src/scripts/python_unittests_unix.py',
             'src/scripts/dev_tools/run_clang_format.py',