diff --git a/.github/actions/read-repo-config/action.yml b/.github/actions/read-repo-config/action.yml new file mode 100644 index 00000000000..be61447e289 --- /dev/null +++ b/.github/actions/read-repo-config/action.yml @@ -0,0 +1,25 @@ + +# (C) 2024 Jack Lloyd +# (C) 2024 René Meusel, Rohde & Schwarz Cybersecurity +# +# Botan is released under the Simplified BSD License (see license.txt) + +name: Read Repository Configuration +description: Reads the repository configuration file and makes it available as environment variables + +runs: + using: composite + steps: + - name: Read Repository Configuration into Environment Variables + run: | + python3 ${{ github.action_path }}/../../../src/scripts/repo_config.py all >> $GITHUB_ENV + echo "REPO_CONFIG_LOADED=true" >> $GITHUB_ENV + shell: bash + if: runner.os != 'Windows' + + - name: Read Repository Configuration into Environment Variables + run: | + python3 ${{ github.action_path }}/../../../src/scripts/repo_config.py all >> $GITHUB_ENV + echo "REPO_CONFIG_LOADED=true" >> $GITHUB_ENV + shell: pwsh + if: runner.os == 'Windows' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4e801800f07..387fdea5294 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,6 +48,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: @@ -79,6 +82,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: @@ -113,6 +119,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: @@ -128,6 +137,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: @@ -185,11 +197,14 @@ jobs: with: path: ./source + - name: Read Repository Configuration + uses: ./source/.github/actions/read-repo-config + - name: Fetch BoringSSL fork for BoGo tests uses: actions/checkout@v4 with: - repository: randombit/boringssl - ref: rene/runner-20241016 + repository: ${{ env.BORINGSSL_REPO }} + ref: ${{ env.BORINGSSL_BRANCH }} path: ./boringssl if: matrix.target == 'coverage' || matrix.target == 'sanitizer' @@ -232,6 +247,9 @@ jobs: with: path: ./source + - name: Read Repository Configuration + uses: ./source/.github/actions/read-repo-config + - name: Setup Build Agent uses: ./source/.github/actions/setup-build-agent with: @@ -276,12 +294,12 @@ jobs: runs-on: ${{ matrix.host_os }} - env: - ANDROID_NDK: android-ndk-r26 - steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index e5e7454fdb7..b1c962bcec6 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -26,6 +26,9 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 84128e63472..feb3f650b49 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -44,11 +44,14 @@ jobs: with: path: ./source + - name: Read Repository Configuration + uses: ./source/.github/actions/read-repo-config + - name: Fetch BoringSSL fork for BoGo tests uses: actions/checkout@v4 with: - repository: randombit/boringssl - ref: rene/runner-20241016 + repository: ${{ env.BORINGSSL_REPO }} + ref: ${{ env.BORINGSSL_BRANCH }} path: ./boringssl - name: Setup Build Agent @@ -109,12 +112,12 @@ jobs: runs-on: ${{ matrix.host_os }} - env: - ANDROID_NDK: android-ndk-r26 - steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: @@ -132,6 +135,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: @@ -184,6 +190,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: @@ -201,6 +210,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: @@ -219,6 +231,9 @@ jobs: - name: Fetch Botan Repository uses: actions/checkout@v4 + - name: Read Repository Configuration + uses: ./.github/actions/read-repo-config + - name: Setup Build Agent uses: ./.github/actions/setup-build-agent with: diff --git a/doc/dev_ref/continuous_integration.rst b/doc/dev_ref/continuous_integration.rst index ef21cee52fa..82237b42a33 100644 --- a/doc/dev_ref/continuous_integration.rst +++ b/doc/dev_ref/continuous_integration.rst @@ -8,6 +8,38 @@ The Github Actions builds are orchestrated using a script ``src/scripts/ci_build.py``. This allows one to easily reproduce the CI process on a local machine. +Repository Configuration +------------------------ + +Specific configuration for test dependencies and CI-related global settings +are centralized in ``src/configs/repo_config.env``. This file is pulled into +the CI environment using the python script ``src/scripts/repo_config.py``. + +If one needs direct access to the configuration variables (without relying on +environment variables in CI), use ``src/scripts/repo_config.py`` in one of the +following ways: + +1. From the command line: + + .. code-block:: bash + + # print all key-value pairs, like: VAR=VALUE\n... + python3 src/scripts/repo_config.py all + + # print the value of a specific key + python3 src/scripts/repo_config.py get VAR + + # list all available variables in repo_config.env + python3 src/scripts/repo_config.py list + +2. As a python module (assuming the script is in the PYTHONPATH): + + .. code-block:: python + + from repo_config import RepoConfig + config = RepoConfig() + print(config['VAR']) + Github Actions --------------- diff --git a/src/configs/repo_config.env b/src/configs/repo_config.env new file mode 100644 index 00000000000..2c53506e944 --- /dev/null +++ b/src/configs/repo_config.env @@ -0,0 +1,36 @@ +# This file contains configurations that are relevant for the entire repository +# +# The variables defined in this file are made available as environment variables +# in the CI jobs. +# +# If one needs to read these variables in a script, they can be accessed via +# src/scripts/repo_config.py. For example, to read the value of BORINGSSL_REPO, +# one can use the following command: +# +# python3 src/scripts/repo_config.py get BORINGSSL_REPO +# +# The settings in this file _explicitly do not_ affect the build of the shipped +# production code or library. Such settings may be found in src/build-data. + +# The fork of boringssl that should be used for BoGo tests +BORINGSSL_REPO="randombit/boringssl" + +# The branch in our fork of boringssl that should be used for BoGo tests +BORINGSSL_BRANCH="rene/runner-20241016" + +# The Android NDK to for the cross platform builds to Android +ANDROID_NDK="android-ndk-r26" + +# Jitterentropy library version to be used for testing the 'jitter_rng' module +JITTERENTROPY_VERSION="3.6.0" + +# The version of the Intel SDE tool to use for running the Intel SDE tests +INTEL_SDE_VERSION="sde-external-9.38.0-2024-04-18-lin" + +# Limbo test suite revision to be used in run_limbo_tests.py +LIMBO_TEST_SUITE_REVISION="f98aa03f45d108ae4e1bc5a61ec4bd0b8d137559" + +# The maximum size of the compiler cache in CI +# Those variables are directly consumed by ccache and sccache respectively +CCACHE_MAXSIZE="200M" +SCCACHE_CACHE_SIZE="200M" diff --git a/src/scripts/ci/setup_gh_actions.ps1 b/src/scripts/ci/setup_gh_actions.ps1 index 1fd505b08f6..ad38b5b46c6 100644 --- a/src/scripts/ci/setup_gh_actions.ps1 +++ b/src/scripts/ci/setup_gh_actions.ps1 @@ -31,5 +31,3 @@ if($identifiers_for_64bit -contains $ARCH ) { } else { echo "VSENV_ARCH=$ARCH" >> $env:GITHUB_ENV } - -echo "SCCACHE_CACHE_SIZE=200M" >> $env:GITHUB_ENV diff --git a/src/scripts/ci/setup_gh_actions.sh b/src/scripts/ci/setup_gh_actions.sh index cb8993f3be9..5ee65a415d4 100755 --- a/src/scripts/ci/setup_gh_actions.sh +++ b/src/scripts/ci/setup_gh_actions.sh @@ -18,7 +18,7 @@ SCRIPT_LOCATION=$(cd "$(dirname "$0")"; pwd) function build_and_install_jitterentropy() { mkdir jitterentropy-library - curl -L https://github.com/smuellerDD/jitterentropy-library/archive/refs/tags/v3.6.0.tar.gz | tar -xz -C . + curl -L "https://github.com/smuellerDD/jitterentropy-library/archive/refs/tags/v${JITTERENTROPY_VERSION}.tar.gz" | tar -xz -C . jel_dir="$(realpath jitterentropy-library-*)" cmake -B "${jel_dir}/build" -S "${jel_dir}" -DCMAKE_BUILD_TYPE=Release cmake --build "${jel_dir}/build" @@ -27,6 +27,11 @@ function build_and_install_jitterentropy() { rm -rf "${jel_dir}" } +if [ -z $REPO_CONFIG_LOADED ]; then + echo "Repository configuration not loaded" >&2 + exit 1 +fi + if type -p "apt-get"; then # TPM2-TSS library (to build the library against) tpm2_specific_packages=("libtss2-dev") @@ -133,13 +138,11 @@ if type -p "apt-get"; then sudo apt-get -qq install qemu-user g++-s390x-linux-gnu elif [ "$TARGET" = "sde" ]; then - SDE_VER=sde-external-9.38.0-2024-04-18-lin - wget https://downloadmirror.intel.com/823664/${SDE_VER}.tar.xz - tar -xvf ${SDE_VER}.tar.xz - echo ${SDE_VER} >> "$GITHUB_PATH" + wget https://downloadmirror.intel.com/823664/${INTEL_SDE_VERSION}.tar.xz + tar -xvf ${INTEL_SDE_VERSION}.tar.xz elif [ "$TARGET" = "cross-android-arm32" ] || [ "$TARGET" = "cross-android-arm64" ] || [ "$TARGET" = "cross-android-arm64-amalgamation" ]; then - wget -nv https://dl.google.com/android/repository/"$ANDROID_NDK"-linux.zip + wget -nv "https://dl.google.com/android/repository/${ANDROID_NDK}-linux.zip" unzip -qq "$ANDROID_NDK"-linux.zip elif [ "$TARGET" = "cross-arm32-baremetal" ]; then @@ -156,7 +159,7 @@ if type -p "apt-get"; then elif [ "$TARGET" = "limbo" ]; then sudo apt-get -qq install python3-dateutil - wget -nv https://raw.githubusercontent.com/C2SP/x509-limbo/f98aa03f45d108ae4e1bc5a61ec4bd0b8d137559/limbo.json -O "${SCRIPT_LOCATION}/../../../limbo.json" + wget -nv "https://raw.githubusercontent.com/C2SP/x509-limbo/${LIMBO_TEST_SUITE_REVISION}/limbo.json" -O "${SCRIPT_LOCATION}/../../../limbo.json" elif [ "$TARGET" = "coverage" ] || [ "$TARGET" = "sanitizer" ]; then if [ "$TARGET" = "coverage" ]; then @@ -213,5 +216,3 @@ if type -p "ccache"; then cache_location="$( ccache --get-config cache_dir )" echo "COMPILER_CACHE_LOCATION=${cache_location}" >> "${GITHUB_ENV}" fi - -echo "CCACHE_MAXSIZE=200M" >> "${GITHUB_ENV}" diff --git a/src/scripts/ci_build.py b/src/scripts/ci_build.py index 5df273a9cdf..22ee5f7f332 100755 --- a/src/scripts/ci_build.py +++ b/src/scripts/ci_build.py @@ -767,6 +767,7 @@ def main(args=None): 'src/scripts/test_python.py', 'src/scripts/test_fuzzers.py', 'src/scripts/test_cli.py', + 'src/scripts/repo_config.py', 'src/scripts/python_unittests.py', 'src/scripts/python_unittests_unix.py', 'src/scripts/dev_tools/run_clang_format.py', diff --git a/src/scripts/repo_config.py b/src/scripts/repo_config.py new file mode 100644 index 00000000000..a4cfdcc13f5 --- /dev/null +++ b/src/scripts/repo_config.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +""" +(C) 2024 Jack Lloyd +(C) 2024 René Meusel - Rohde & Schwarz Cybersecurity + +Botan is released under the Simplified BSD License (see license.txt) +""" + +# Use this as a script to read the repository's configuration file and access +# its contents. For instance: +# +# $ python3 repo_config.py all +# $ python3 repo_config.py get BORINGSSL_REPO +# +# This might also be used as a module to access the configuration file from +# other python scripts. + +import argparse +import os +import re +import sys + +_DEFAULT_REPO_CONFIG_LOCATION = os.path.realpath(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'configs', 'repo_config.env')) + +class RepoConfig(dict): + """ Reads the repository's configuration file and provides access to its variables. """ + + def __init__(self, env_file: str = _DEFAULT_REPO_CONFIG_LOCATION): + self._file_path = env_file + parser = re.compile(r'^(?P\w+)\s*=\s*((?P[^\"]\S+)|\"(?P\S+)\")\s*$') + with open(self._file_path, 'r', encoding='utf-8') as f: + for line in f.readlines(): + if m := parser.match(line): + var = m.group('var') + val = m.group('val') or m.group('quoted_val') + try: + self[var] = int(val) + except ValueError: + self[var] = val + + @property + def config_file_path(self): + return self._file_path + +def main(): + def print_all(cfg: RepoConfig, *_): + print('\n'.join(f'{key}={value}' for key, value in cfg.items())) + + def list_vars(cfg: RepoConfig, *_): + print('\n'.join(key for key in cfg)) + + def get_var(cfg: RepoConfig, args): + if args.var not in cfg: + print(f'Variable "{args.var}" not found in the configuration file.', file=sys.stderr) + raise KeyError() + print(cfg[args.var]) + + def print_config_file_path(cfg, *_): + print(cfg.config_file_path) + + parser = argparse.ArgumentParser(description='Read and process a .env file.') + subparsers = parser.add_subparsers(dest='command', required=True) + + parser_all = subparsers.add_parser('all', help='Print all variables and their values.') + parser_all.set_defaults(dispatch=print_all) + + parser_all = subparsers.add_parser('list', help='Print all variable names.') + parser_all.set_defaults(dispatch=list_vars) + + parser_get = subparsers.add_parser('get', help='Get the value of a specific variable.') + parser_get.add_argument('var', type=str, help='The variable name to retrieve.') + parser_get.set_defaults(dispatch=get_var) + + parser_file = subparsers.add_parser('file', help='Print the path to the configuration file.') + parser_file.set_defaults(dispatch=print_config_file_path) + + args = parser.parse_args() + try: + args.dispatch(RepoConfig(), args) + except Exception: + return 1 + + return 0 + +if __name__ == '__main__': + sys.exit(main())