diff --git a/.bazelignore b/.bazelignore index 19dbe9577..c99fb3545 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,4 +1,6 @@ bin +doc/_build +doc/venv docker/_build -rules_openapi/tools/node_modules +private/mgmtapi/tools/node_modules tools/lint/logctxcheck/testdata/src diff --git a/.bazelrc b/.bazelrc index 610c9114f..ae43f2c0d 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,10 +1,12 @@ ### common options for all subcommands (help, query, build, ...) -common --show_timestamps +common --show_timestamps --enable_platform_specific_config + # connect to buchgr/bazel-remote cache # These flags can unfortunately not be specified for `common`, as they are not accepted by all subcommands (help, version, dump) -build --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 -query --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 -fetch --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 +# The --experimental_remote_downloader_local_fallback is used as workaround for issue with rules_oci (https://github.com/bazel-contrib/rules_oci/issues/275). +build --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true +query --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true +fetch --remote_cache=grpc://localhost:9092 --experimental_remote_downloader=grpc://localhost:9092 --experimental_remote_downloader_local_fallback=true ### options for build, test, run, clean, etc. # expose git version (etc) to bazel @@ -13,6 +15,14 @@ build --workspace_status_command=./tools/bazel-build-env build --java_runtime_version=remotejdk_11 # disable legacy_create_init for py_binary, py_test etc. This may eventually become the default. build --incompatible_default_to_explicit_init_py +# Enable resolution of cc toolchain by go toolchain +build --incompatible_enable_cc_toolchain_resolution +build --flag_alias=file_name_version=//:file_name_version + +# include one of "--define gotags=sqlite_mattn" or "--define gotags=sqlite_modernc" +# cannot be in common, because query chokes on it. +build --define gotags=sqlite_modernc,netgo +build:osx --define gotags=sqlite_modernc ### options for test test --build_tests_only --print_relative_test_log_paths --test_output=errors @@ -25,7 +35,7 @@ test:unit_all --config=unit //... test:integration --test_tag_filters=integration,-lint test:integration_all --config=integration //... -test:lint --test_tag_filters=lint --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore //... +test:lint --test_tag_filters=lint,write_src --test_summary=terse --noshow_progress --experimental_convenience_symlinks=ignore # run quietly, only display errors common:quiet --ui_event_filters=-warning,-info,-debug,-stdout,-stderr --noshow_progress diff --git a/.bazelversion b/.bazelversion index 831446cbd..19b860c18 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -5.1.0 +6.4.0 diff --git a/.buildkite/cleanup-leftovers.sh b/.buildkite/cleanup-leftovers.sh new file mode 100755 index 000000000..cc1bb05b3 --- /dev/null +++ b/.buildkite/cleanup-leftovers.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +echo "~~~ Cleaning up any leftovers" +cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))" +[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; } +echo "Remove leftover networks" +docker network prune -f +echo "Remove leftover volumes" +docker volume prune -f + +rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts test-out.tar.gz diff --git a/.buildkite/hooks/bazel-remote.yml b/.buildkite/hooks/bazel-remote.yml index 07523b1c2..b4fdae7b2 100644 --- a/.buildkite/hooks/bazel-remote.yml +++ b/.buildkite/hooks/bazel-remote.yml @@ -1,4 +1,5 @@ version: "2.4" +name: bazel_remote services: bazel-remote: container_name: bazel-remote-cache diff --git a/.buildkite/hooks/go-module-proxy.yml b/.buildkite/hooks/go-module-proxy.yml index 4c852825f..566068cca 100644 --- a/.buildkite/hooks/go-module-proxy.yml +++ b/.buildkite/hooks/go-module-proxy.yml @@ -1,5 +1,6 @@ --- version: "2.4" +name: athens services: go-module-proxy: container_name: go-module-proxy diff --git a/.buildkite/hooks/pre-artifact b/.buildkite/hooks/pre-artifact deleted file mode 100755 index 231320d4e..000000000 --- a/.buildkite/hooks/pre-artifact +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -# Attempt to do clean docker topology shutdown. -# This lets applications flush the logs to avoid cutting them off. -if [ -f "gen/scion-dc.yml" ]; then - # This hook is global, therefore we need to disable fail on error because - # not all docker-compose tests are following the pattern - # COMPOSE_PROJECT_NAME=scion, COMPOSE_FILE=gen/scion-dc.yml - # and logs in stdout. - set +e - docker-compose -f gen/scion-dc.yml -p scion stop - - docker-compose -f gen/scion-dc.yml -p scion logs --no-color > logs/scion-dc.log - - for s in $(docker-compose -f gen/scion-dc.yml -p scion ps --services); do - cat logs/scion-dc.log | grep $s| cut -f2 -d"|" > logs/${s#"scion_"}.log - done - - docker-compose -f gen/scion-dc.yml -p scion down -v - - # a subset of tests are using testgen and they do a collect log and dc stop - # on their own. Therefore the above code produces empty files that are confusing. - # Given the limitation of buildkite that the hook is global we just cleanup - # the empty files. - find . -type f -empty -delete - - set -e -fi - -# Now we build the artifact name next, for this we first need TARGET and BUILD, -# see below. -# -# For PRs the target is the pull request, otherwise it is the branch. -TARGET="$BUILDKITE_PULL_REQUEST" -if [ "$BUILDKITE_PULL_REQUEST" == "false" ]; then - TARGET="$BUILDKITE_BRANCH" -fi -TARGET="${TARGET//\//_}" -echo "\$TARGET=$TARGET" - -# For nightly builds instead of the build number print nightly and the date. -BUILD="build-${BUILDKITE_BUILD_NUMBER}" -[ -n "$NIGHTLY" ] && BUILD=nightly-"$(date +%s)" -echo "\$BUILD=$BUILD" - -ARTIFACTS="buildkite.${BUILDKITE_ORGANIZATION_SLUG}.${TARGET}.${BUILD}.${BUILDKITE_STEP_KEY:-unset}.${BUILDKITE_JOB_ID}" -mkdir -p "artifacts/$ARTIFACTS" artifacts.out - -function save { - if [ -d "$1" ]; then - echo Found artifacts: "$1" - cp -R "$1" "artifacts/$ARTIFACTS" - fi -} - -# Also store remote cache logs -cache_ctr=$(docker ps -aq -f "name=bazel-remote-cache") -if [ ! -z "$cache_ctr" ]; then - mkdir -p logs/docker - docker logs bazel-remote-cache > logs/docker/bazel-remote-cache.log -fi - -save "bazel-testlogs" -save "outputs" -save "logs" -save "traces" -save "gen" -save "gen-cache" -save "/tmp/test-artifacts" - -tar chaf "artifacts.out/$ARTIFACTS.tar.gz" -C artifacts "$ARTIFACTS" -rm -rf artifacts - -echo "Output tar= artifacts.out/$ARTIFACTS.tar.gz" diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index f43be60ae..8da7d1b2e 100755 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -11,16 +11,29 @@ printenv PATH # Install build tools on first job for this runner. .buildkite/provision-agent.sh -# Clean up left-overs from previous run -PRE_COMMAND_SETUP=true . .buildkite/hooks/pre-exit - set -euo pipefail -echo "--- Increase receive network buffer size" +echo "~~~ Increase receive network buffer size" sudo sysctl -w net.core.rmem_max=1048576 -echo "--- Setting up bazel environment" +# Export SCION_VERSION as environment variable, unless it's set from outside (e.g. as variable for this build) +# Note that this precommand hook runs even in "step zero" of the pipeline, where we `buildkite-agent upload'. +# With this, the SCION_VERSION can be interpolated by the agent throughout the pipeline yaml. +if [ -z ${SCION_VERSION+x} ]; then + echo "~~~ Export SCION_VERSION" + if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then + # Shorten the git version to omit commit information, improving cache reuse. + # The format of git-version is "--" + # This will be shortened to "-modified-ci" + export SCION_VERSION=$(tools/git-version | sed 's/-.*/-modified-ci/') + else + export SCION_VERSION=$(tools/git-version) + fi + echo SCION_VERSION=${SCION_VERSION} +fi + +echo "~~~ Setting up bazel environment" if [ -z ${BAZEL_REMOTE_S3_ACCESS_KEY_ID+x} ]; then echo "S3 env not set, not starting bazel remote proxy" exit 0 @@ -33,20 +46,17 @@ rm -f $HOME/.bazelrc # --nostamp is required for better caching (only on non-release jobs). if [ "$BUILDKITE_PIPELINE_SLUG" == "scion" ]; then echo "build --nostamp" > $HOME/.bazelrc - # Also set a fixed GIT_VERSION so that the workspace_status_command always - # returns the same value on CI to improve cache reuse. - export GIT_VERSION="ci-fixed" else echo "build --stamp" > $HOME/.bazelrc fi echo "test --test_env CI" >> $HOME/.bazelrc -echo "--- Starting bazel remote cache proxy" +echo "~~~ Starting bazel remote cache proxy" # Start bazel remote cache proxy for S3 # Note that S3 keys are injected by buildkite, see # https://buildkite.com/docs/pipelines/secrets#storing-secrets-with-the-elastic-ci-stack-for-aws -docker-compose -f .buildkite/hooks/bazel-remote.yml -p bazel_remote up -d +docker compose -f .buildkite/hooks/bazel-remote.yml up -d -echo "--- Starting go module proxy" -docker-compose -f .buildkite/hooks/go-module-proxy.yml -p athens up -d +echo "~~~ Starting go module proxy" +docker compose -f .buildkite/hooks/go-module-proxy.yml up -d diff --git a/.buildkite/hooks/pre-exit b/.buildkite/hooks/pre-exit index 1b85eab6d..ba1758cc1 100644 --- a/.buildkite/hooks/pre-exit +++ b/.buildkite/hooks/pre-exit @@ -1,8 +1,7 @@ #!/bin/bash - -if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then - echo "--- Uploading bazel-remote and go-module-proxy logs/metrics" +if [ -f ".buildkite/hooks/bazel-remote.yml" ]; then + echo "~~~ Uploading bazel-remote and go-module-proxy logs/metrics" curl http://localhost:8080/metrics > bazel-remote-cache.metrics docker logs bazel-remote-cache &> bazel-remote-cache.log @@ -11,19 +10,3 @@ if [ -f ".buildkite/hooks/bazel-remote.yml" -a -z "$PRE_COMMAND_SETUP" ]; then buildkite-agent artifact upload "bazel-remote-cache.*;go-module-proxy.*" fi - -echo "--- Cleaning up the topology" - -./scion.sh topo_clean - -echo "--- Cleaning up docker containers/networks/volumes" -cntrs="$(docker ps -aq | grep -v -f <(docker ps -q --filter "name=go-module-proxy" --filter "name=bazel-remote-cache"))" -[ -n "$cntrs" ] && { echo "Remove leftover containers..."; docker rm -f $cntrs; } - -echo "Remove leftover networks" -docker network prune -f -echo "Remove leftover volumes" -docker volume prune -f - -echo "--- Cleaning up logs and artifacts" -rm -rf bazel-testlogs logs/* traces gen gen-cache /tmp/test-artifacts diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 0744a03b5..d3dda80ee 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -3,21 +3,72 @@ env: steps: - label: "Build :bazel:" command: - - bazel build --verbose_failures --announce_rc //:all - - bazel run --verbose_failures //docker:prod //docker:test + - bazel build --verbose_failures --announce_rc //:scion //:scion-ci + - bazel build --verbose_failures //docker:prod //docker:test key: build + artifact_paths: + - "scion_${SCION_VERSION}_amd64_linux.tar.gz" # Note: SCION_VERSION interpolated by buildkite agent uploading pipeline, see pre-command hook + - "scion-ci_${SCION_VERSION}_amd64_linux.tar.gz" + plugins: + - scionproto/metahook#v0.3.0: + pre-artifact: | + gzip --to-stdout bazel-bin/scion.tar > scion_${SCION_VERSION}_amd64_linux.tar.gz + gzip --to-stdout bazel-bin/scion-ci.tar > scion-ci_${SCION_VERSION}_amd64_linux.tar.gz + post-artifact: | + cat << EOF | buildkite-agent annotate --style "info" --context "binaries" + #### Build outputs + - SCION binaries + - SCION test tools and utilities + EOF retry: &automatic-retry automatic: - exit_status: -1 # Agent was lost - exit_status: 255 # Forced agent shutdown timeout_in_minutes: 10 - wait + - label: "Package :debian: :openwrt: :rpm:" + command: | + make dist-deb BFLAGS="--file_name_version=${SCION_VERSION}" + make dist-openwrt BFLAGS="--file_name_version=${SCION_VERSION}" + make dist-rpm BFLAGS="--file_name_version=${SCION_VERSION}" + artifact_paths: + - "installables/scion_*.tar.gz" + plugins: + - scionproto/metahook#v0.3.0: + pre-artifact: | + set -x + pushd installables + tar -chaf scion_${SCION_VERSION}_deb_amd64.tar.gz *_${SCION_VERSION}_amd64.deb + tar -chaf scion_${SCION_VERSION}_deb_arm64.tar.gz *_${SCION_VERSION}_arm64.deb + tar -chaf scion_${SCION_VERSION}_deb_i386.tar.gz *_${SCION_VERSION}_i386.deb + tar -chaf scion_${SCION_VERSION}_deb_armel.tar.gz *_${SCION_VERSION}_armel.deb + tar -chaf scion_${SCION_VERSION}_openwrt_x86_64.tar.gz *_${SCION_VERSION}_x86_64.ipk + tar -chaf scion_${SCION_VERSION}_rpm_x86_64.tar.gz *_${SCION_VERSION}_x86_64.rpm + popd + ls installables + post-artifact: | + cat << EOF | buildkite-agent annotate --style "info" --context "packages" + #### Packages :debian: + - amd64 + - arm64 + - i386 + - armel + #### Packages :openwrt: + - x86_64 + #### Packages :rpm: + - x86_64 + EOF + key: dist + retry: *automatic-retry - label: "Unit Tests :bazel:" command: - bazel test --config=race --config=unit_all key: unit_tests + plugins: + - scionproto/metahook#v0.3.0: + pre-artifact: tar -chaf bazel-testlogs.tar.gz bazel-testlogs artifact_paths: - - "artifacts.out/**/*" + - bazel-testlogs.tar.gz retry: *automatic-retry timeout_in_minutes: 20 - label: "Lint :bash:" @@ -31,8 +82,8 @@ steps: - echo "--- go_deps.bzl" - mkdir -p /tmp/test-artifacts - cp go.mod go.sum go_deps.bzl /tmp/test-artifacts/ + - make go.mod - make go_deps.bzl -B - - $(bazel info output_base 2>/dev/null)/external/go_sdk/bin/go mod tidy - diff -u /tmp/test-artifacts/go.mod go.mod - diff -u /tmp/test-artifacts/go.sum go.sum - diff -u /tmp/test-artifacts/go_deps.bzl go_deps.bzl @@ -56,6 +107,7 @@ steps: timeout_in_minutes: 20 key: check_generated retry: *automatic-retry + - wait - group: "End to End" key: e2e steps: @@ -69,14 +121,20 @@ steps: - tools/await-connectivity - ./bin/scion_integration || ( echo "^^^ +++" && false ) - ./bin/end2end_integration || ( echo "^^^ +++" && false ) - plugins: &shutdown-scion-post-command + plugins: &scion-run-hooks - scionproto/metahook#v0.3.0: + pre-command: .buildkite/cleanup-leftovers.sh post-command: | - echo "--- Shutting down SCION topology" + echo "~~~ Shutting down SCION topology" ./scion.sh stop - echo "SCION topology successfully shut down" - artifact_paths: - - "artifacts.out/**/*" + pre-artifact: | + if [ -f "gen/scion-dc.yml" ]; then + tools/dc collect_logs scion logs/ + fi + tar -chaf test-out.tar.gz $(ls -d logs traces gen gen-cache) # ls -d to filter missing directories + pre-exit: .buildkite/cleanup-leftovers.sh + artifact_paths: &scion-run-artifact-paths + - test-out.tar.gz timeout_in_minutes: 15 key: e2e_integration_tests_v2 retry: *automatic-retry @@ -90,25 +148,23 @@ steps: - tools/await-connectivity - ./bin/end2end_integration || ( echo "^^^ +++" && false ) - ./tools/integration/revocation_test.sh - plugins: *shutdown-scion-post-command - artifact_paths: - - "artifacts.out/**/*" + plugins: *scion-run-hooks + artifact_paths: *scion-run-artifact-paths timeout_in_minutes: 15 key: e2e_revocation_test_v2 retry: *automatic-retry - label: "E2E: default :docker: (ping)" command: - echo "--- build" - - make build docker-images + - make build-dev docker-images - echo "--- start topology" - ./scion.sh topology -d - ./scion.sh run - tools/await-connectivity - echo "--- run tests" - ./bin/end2end_integration -d || ( echo "^^^ +++" && false ) - plugins: *shutdown-scion-post-command - artifact_paths: - - "artifacts.out/**/*" + plugins: *scion-run-hooks + artifact_paths: *scion-run-artifact-paths timeout_in_minutes: 15 key: docker_integration_e2e_default retry: *automatic-retry diff --git a/.buildkite/pipeline_lib.sh b/.buildkite/pipeline_lib.sh index 7f1de5016..bf61b3cde 100644 --- a/.buildkite/pipeline_lib.sh +++ b/.buildkite/pipeline_lib.sh @@ -44,12 +44,21 @@ gen_bazel_test_steps() { echo " command:" echo " - bazel test --test_output=streamed $test $args $cache" echo " key: \"${name////_}\"" + echo " plugins:" + echo " - scionproto/metahook#v0.3.0:" + echo " pre-command: .buildkite/cleanup-leftovers.sh" + echo " pre-artifact: tar -chaf bazel-testlogs.tar.gz bazel-testlogs" + echo " pre-exit: .buildkite/cleanup-leftovers.sh" echo " artifact_paths:" - echo " - \"artifacts.out/**/*\"" + echo " - \"bazel-testlogs.tar.gz\"" echo " timeout_in_minutes: 20" echo " retry:" + echo " manual:" + echo " permit_on_passed: true" echo " automatic:" echo " - exit_status: -1 # Agent was lost" echo " - exit_status: 255 # Forced agent shutdown" + echo " - exit_status: 3 # Test may be flaky or it just didn't pass" + echo " limit: 2" done } diff --git a/.buildkite/provision-agent.sh b/.buildkite/provision-agent.sh index 69dc76da8..d8f2a5fcc 100755 --- a/.buildkite/provision-agent.sh +++ b/.buildkite/provision-agent.sh @@ -13,4 +13,4 @@ echo "~~~ Install build tools" tools/install_bazel tools/install_deps -sha1sum tools/install_bazel tools/install_deps tools/env/pip3/deps tools/env/pip3/requirements.txt tools/env/rhel/deps > /tmp/buildkite-scionproto-runner-provision.sum +sha1sum tools/install_bazel tools/install_deps tools/env/pip3/deps tools/env/pip3/requirements.txt tools/env/rhel/deps tools/env/rhel/pkgs.txt tools/env/debian/deps tools/env/debian/pkgs.txt > /tmp/buildkite-scionproto-runner-provision.sum diff --git a/.github/ISSUE_TEMPLATE/01-proposal.md b/.github/ISSUE_TEMPLATE/01-proposal.md index 77c7b316e..05f316350 100644 --- a/.github/ISSUE_TEMPLATE/01-proposal.md +++ b/.github/ISSUE_TEMPLATE/01-proposal.md @@ -6,6 +6,5 @@ labels: i/proposal - diff --git a/.github/ISSUE_TEMPLATE/02-workitem.md b/.github/ISSUE_TEMPLATE/02-workitem.md new file mode 100644 index 000000000..8ac5d4a0f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/02-workitem.md @@ -0,0 +1,5 @@ +--- +name: Work Item +about: "Something needs doing" +labels: workitem +--- diff --git a/.github/workflows/pr-title.yml b/.github/workflows/pr-title.yml new file mode 100644 index 000000000..c875e63bb --- /dev/null +++ b/.github/workflows/pr-title.yml @@ -0,0 +1,31 @@ +name: Check the pull request title + +on: + pull_request: + types: [opened, edited, reopened, synchronize] + +jobs: + check: + runs-on: ubuntu-latest + steps: + - name : Check the PR title + env: + TITLE: ${{ github.event.pull_request.title }} + run: | + # Check that PR is of the form `: ` + + url='https://docs.scion.org/en/latest/dev/git.html#good-commit-messages' + if [[ ! "$TITLE" =~ ^[a-z0-9,/-]*:[[:space:]] ]]; then + echo '::error::The PR title should start with `: `. See '"$url" + exit 1 + fi + # Title should be lower case; initialisms and identifiers still occur occasionally and should be allowed. + # -> enforce only the first word + if [[ ! "$TITLE" =~ ^[a-z0-9,/-]*:[[:space:]][a-z] ]]; then + echo '::error::The PR title should be lower case (enforced on first letter). See '"$url" + exit 1 + fi + if [[ $TITLE =~ \.[[:space:]]*$ ]]; then + echo '::error::The PR title should not end with a ".". See '"$url" + exit 1 + fi diff --git a/.gitignore b/.gitignore index 098a44ecb..382315432 100644 --- a/.gitignore +++ b/.gitignore @@ -43,6 +43,7 @@ compile_commands.json # Environment files ######################### /venv/ +doc/venv/ # Docker working directory ########################## @@ -65,6 +66,10 @@ compile_commands.json /bin/* !/bin/.keepme +# Generated package files +########################## +/installables/ + # CTags ########################## tags @@ -103,3 +108,10 @@ target .metals .bazelbsp .bazel +*.vpr +tmp/ +logger.log + +# emacs backup files +#################### +*~ diff --git a/.golangcilint.yml b/.golangcilint.yml index 100bb83af..d585f3d63 100644 --- a/.golangcilint.yml +++ b/.golangcilint.yml @@ -3,6 +3,7 @@ linters: disable-all: true enable: # Default linters. + - forbidigo - errcheck - gosimple - govet @@ -10,7 +11,6 @@ linters: - typecheck - unused # Extended linters. - - depguard - exportloopref - rowserrcheck - sqlclosecheck @@ -20,18 +20,18 @@ linters: - misspell - goheader linters-settings: + # ... + forbidigo: + forbid: + - p: "([iI][fF][iI]d)|([iI]F[iI][dD])|([iI][fF]i[dD])" + msg: "spell interface ID as ifID / IfID" + - p: "(?i)interfaceID" # case insensitive + msg: "spell interface ID as ifID / IfID" + - p: "Trc" + msg: "spell trust root certificate as trc / TRC" lll: line-length: 100 tab-width: 4 - depguard: - list-type: blacklist - include-go-root: true - packages: "io/ioutil" - packages-with-error-message: - # specify an error message to output when a blacklisted package is used - "io/ioutil": > - "The use of 'io/ioutil' is deprecated. Check - https://pkg.go.dev/io/ioutil for alternatives" errcheck: exclude-functions: - (*github.com/spf13/cobra.Command).MarkFlagRequired @@ -73,86 +73,3 @@ issues: - path: pkg/scrypto/cms linters: [goheader] - # list of exceptions to the errcheck check - # TODO(matzf): most of these should rather be fixed! It may be ok to be a - # bit more lenient in the test code perhaps, but some of these are lurking - # bugs. - - path: "^control/beacon/policy.go$|\ - ^control/cmd/control/main.go$|\ - ^control/colibri/reservation/conf/capacities_test.go$|\ - ^control/colibri/reservation/e2e/reservation_test.go$|\ - ^control/colibri/reservation/index_test.go$|\ - ^control/colibri/reservation/reservationdbtest/reservationdbtest.go$|\ - ^control/colibri/reservation/segment/path.go$|\ - ^control/colibri/reservation/segment/reservation_test.go$|\ - ^control/colibri/reservation/sqlite/db_test.go$|\ - ^control/colibri/reservationstore/store.go$|\ - ^control/mgmtapi/api.go$|\ - ^dispatcher/cmd/dispatcher/main.go$|\ - ^dispatcher/dispatcher.go$|\ - ^dispatcher/internal/registration/bench_test.go$|\ - ^gateway/control/aggregator.go$|\ - ^gateway/control/engine.go$|\ - ^gateway/control/engine_test.go$|\ - ^gateway/control/enginecontroller_test.go$|\ - ^gateway/control/export_test.go$|\ - ^gateway/control/remotemonitor.go$|\ - ^gateway/control/router.go$|\ - ^gateway/control/session.go$|\ - ^gateway/control/sessionconfigurator.go$|\ - ^gateway/control/watcher_test.go$|\ - ^gateway/dataplane/diagnostics_test.go$|\ - ^gateway/dataplane/ipforwarder_test.go$|\ - ^gateway/dataplane/routingtable.go$|\ - ^gateway/dataplane/session_test.go$|\ - ^gateway/gateway.go$|\ - ^gateway/routing/file.go$|\ - ^gateway/xnet/xnet.go$|\ - ^pkg/experimental/colibri/reservation/types.go$|\ - ^pkg/experimental/epic/epic_test.go$|\ - ^pkg/experimental/hiddenpath/beaconwriter_test.go$|\ - ^pkg/experimental/hiddenpath/grpc/registerer_test.go$|\ - ^pkg/experimental/hiddenpath/store_test.go$|\ - ^pkg/grpc/dialer_test.go$|\ - ^pkg/log/log.go$|\ - ^pkg/log/testlog/log.go$|\ - ^pkg/private/xtest/graph/graph.go$|\ - ^pkg/private/xtest/grpc.go$|\ - ^pkg/segment/seg.go$|\ - ^pkg/segment/segs_test.go$|\ - ^pkg/slayers/extn_test.go$|\ - ^pkg/slayers/scion_test.go$|\ - ^pkg/snet/packet_test.go$|\ - ^pkg/snet/path.go$|\ - ^pkg/snet/squic/net.go$|\ - ^pkg/sock/reliable/reconnect/conn.go$|\ - ^pkg/sock/reliable/reconnect/conn_io_test.go$|\ - ^pkg/sock/reliable/reconnect/network_test.go$|\ - ^pkg/sock/reliable/reconnect/reconnecter_test.go$|\ - ^pkg/sock/reliable/reliable.go$|\ - ^private/app/appnet/infraenv.go$|\ - ^private/app/launcher/launcher.go$|\ - ^private/app/path/pathprobe/paths.go$|\ - ^private/config/sample.go$|\ - ^private/mgmtapi/cppki/api/api.go$|\ - ^private/mgmtapi/segments/api/api.go$|\ - ^private/path/combinator/combinator.go$|\ - ^private/revcache/revcachetest/revcachetest.go$|\ - ^private/service/statuspages.go$|\ - ^private/storage/trust/fspersister/db_test.go$|\ - ^private/svc/internal/ctxconn/ctxconn.go$|\ - ^private/trust/db_inspector.go$|\ - ^private/trust/verifier.go$|\ - ^private/trust/verifier_bench_test.go$|\ - ^private/worker/worker_test.go$|\ - ^router/dataplane_test.go$|\ - ^router/export_test.go$|\ - ^router/mgmtapi/api.go$|\ - ^scion-pki/trcs/combine_test.go$|\ - ^tools/end2end/main.go$|\ - ^tools/end2end_integration/main.go$|\ - ^tools/integration/binary.go$|\ - ^tools/integration/done.go$|\ - ^tools/integration/integration.go$|\ - ^tools/integration/progress/progress.go$" - linters: [errcheck] diff --git a/.markdownlint-cli2.jsonc b/.markdownlint-cli2.jsonc new file mode 100644 index 000000000..7be2f0b20 --- /dev/null +++ b/.markdownlint-cli2.jsonc @@ -0,0 +1,27 @@ +{ + "globs": ["**/*.md"], + "ignores": [ + "**/venv/**", + "**/_build/**", + "**/node_modules/**", + "licenses/data/**", + "tools/coremark/LICENSE.md" + ], + "config": { + "default": true, + "MD007": { + "indent": 4 + }, + "MD013": { + "line_length": 100, + "code_blocks": false, + "tables": false + }, + "MD024": { + "siblings_only": true + } + }, + "outputFormatters": [ + [ "markdownlint-cli2-formatter-pretty", { "appendLink": true } ] + ] +} diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 000000000..36699c0dd --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,18 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.10" + +sphinx: + configuration: doc/conf.py + fail_on_warning: true + +python: + install: + - requirements: doc/requirements.txt diff --git a/BUILD.bazel b/BUILD.bazel index f48ed70cd..0f04ad4d1 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,12 +1,13 @@ -load("@bazel_skylib//rules:common_settings.bzl", "bool_flag") +load("@bazel_skylib//rules:common_settings.bzl", "bool_flag", "string_flag") load("@bazel_gazelle//:def.bzl", "gazelle") load("@rules_pkg//:pkg.bzl", "pkg_tar") load("@io_bazel_rules_go//go:def.bzl", "nogo") load("//tools/lint:go_config.bzl", "go_lint_config") +load("//tools/lint:write_source_files.bzl", "write_source_files") load("//tools/lint/python:flake8_config.bzl", "flake8_lint_config") load("//:nogo.bzl", "nogo_deps") load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier") -load("@cgrindel_bazel_starlib//updatesrc:defs.bzl", "updatesrc_update_all") +load("@npm//private/mgmtapi/tools:@stoplight/spectral-cli/package_json.bzl", spectral_bin = "bin") # gazelle:prefix github.com/scionproto/scion # gazelle:map_kind go_library go_library //tools/lint:go.bzl @@ -16,14 +17,122 @@ load("@cgrindel_bazel_starlib//updatesrc:defs.bzl", "updatesrc_update_all") # gazelle:exclude doc/** # gazelle:exclude rules_openapi/tools/node_modules/** # gazelle:exclude tools/lint/**/testdata/src/** -gazelle(name = "gazelle") + +# We suport two sqlite implementations: modernc and mattn. Each implementation +# has a corresponding sqlite_.go "driver" in private/storage/db. Which +# driver gets compiled and linked is controled by a go build tag: sqlite_mattn +# or sqlite_modernc. Those are specified on the command line +# with "--define gotags=sqlite_mattn" or "--define gotags=sqlite_modernc" +# (see the build options in .bazelrc). +# +# Unfortunately Gazelle needs to be given these tags explicitly via the builtags +# attribute. So, to ensure consistency we have to translate our two gotags into +# build_tags. To that end, we create two config_setting flags that are +# set in response to matching the gotags value and use them to select the relevant +# tag for gazelle. (The "define_value" attribute of config_setting doesn't define +# anything. It matches a key-value pair from "--define"). +# +# This is simplistic but the complete, by-the-everchanging-bazel-book, solution +# is ludicrously complicated. Go there if and when needed. +config_setting( + name = "sqlite_mattn_netgo", + define_values = { + "gotags": "sqlite_mattn,netgo", + }, +) + +config_setting( + name = "sqlite_modernc_netgo", + define_values = { + "gotags": "sqlite_modernc,netgo", + }, +) + +config_setting( + name = "sqlite_mattn", + define_values = { + "gotags": "sqlite_mattn", + }, +) + +config_setting( + name = "sqlite_modernc", + define_values = { + "gotags": "sqlite_modernc", + }, +) + +gazelle( + name = "gazelle", + build_tags = select({ + ":sqlite_modernc_netgo": [ + "sqlite_modernc", + "sqlite_modernc_netgo", + ], + ":sqlite_modernc": [ + "sqlite_modernc", + ], + ":sqlite_mattn_netgo": [ + "sqlite_mattn", + "netgo", + ], + ":sqlite_mattn": [ + "sqlite_mattn", + ], + }), + command = "update", + extra_args = [ + "-mode", + "fix", + "-go_naming_convention", + "go_default_library", + ], +) + +gazelle( + name = "gazelle_diff", + build_tags = select({ + ":sqlite_modernc_netgo": [ + "sqlite_modernc", + "netgo", + ], + ":sqlite_modernc": [ + "sqlite_modernc", + ], + ":sqlite_mattn_netgo": [ + "sqlite_mattn", + "netgo", + ], + ":sqlite_mattn": [ + "sqlite_mattn", + ], + }), + command = "update", + extra_args = [ + "-mode", + "diff", + "-go_naming_convention", + "go_default_library", + ], +) + +gazelle( + name = "gazelle_update_repos", + command = "update-repos", + extra_args = [ + "-prune", + "-from_file=go.mod", + "-to_macro=go_deps.bzl%go_deps", + ], +) go_lint_config( name = "go_lint_config", exclude_filter = [ - "mock_", - ".pb.go", + ".connect.go", ".gen.go", + ".pb.go", + "mock_", ], visibility = [ "//visibility:public", @@ -38,6 +147,17 @@ flake8_lint_config( ], ) +# Optional version string to produce versioned file names. End deliverables, such as installable +# packages will have a name derived from that string. +# The flag is to be used when producing publishable assets (so, typically by the CI build). +# The rest of the time the assets will have an unversioned name. The version tags embedded +# in binaries and package manifests are always set. Regardless of this flag. +string_flag( + name = "file_name_version", + build_setting_default = "dev", + visibility = ["//visibility:public"], +) + # Add a build flag to enable bundling the management API documentation with the # binaries. This can be enabled by passing --//:mgmtapi_bundle_doc=true when # invoking bazel @@ -75,6 +195,7 @@ pkg_tar( name = "scion-ci", srcs = [ "//acceptance/cmd/sig_ping_acceptance", + "//acceptance/router_benchmark/brload", "//pkg/private/xtest/graphupdater", "//tools/braccept", "//tools/buildkite/cmd/buildkite_artifacts", @@ -115,11 +236,30 @@ buildifier( mode = "check", ) -# Runs all update_src targets in this Workspace. -updatesrc_update_all( - name = "update_all", - targets_to_run = [ - "//doc/command:copy_scion", - "//doc/command:copy_scion-pki", +spectral_bin.spectral_binary( + name = "spectral", +) + +# Runs all write_source_files targets in this Workspace. To update the list run +# bazel run @com_github_bazelbuild_buildtools//buildozer -- --root_dir $PWD "add additional_update_targets $( bazel query 'filter("^.*[^\d]$", kind(_write_source_file, //...)) except //:write_all_source_files' | tr '\n' ' ')" //:write_all_source_files +write_source_files( + name = "write_all_source_files", + additional_update_targets = [ + "//control/mgmtapi:write_files", + "//daemon/mgmtapi:write_files", + "//dispatcher/mgmtapi:write_files", + "//doc/command:write_files", + "//gateway/mgmtapi:write_files", + "//pkg/proto/control_plane/v1/control_planeconnect:write_files", + "//pkg/proto/daemon/v1/daemonconnect:write_files", + "//pkg/proto/discovery/v1/discoveryconnect:write_files", + "//pkg/proto/gateway/v1/gatewayconnect:write_files", + "//pkg/proto/hidden_segment/v1/hidden_segmentconnect:write_files", + "//private/ca/api:write_files", + "//private/mgmtapi/cppki/api:write_files", + "//private/mgmtapi/health/api:write_files", + "//private/mgmtapi/segments/api:write_files", + "//router/mgmtapi:write_files", + "//spec:write_files", ], ) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8d01e624d..0277a8a3e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ # Contribution Guide The Contribution Guide for the SCION project can be found -[here](https://docs.scion.org/en/latest/contribute.html). +[here](https://docs.scion.org/en/latest/dev/contribute.html). diff --git a/Makefile b/Makefile index f18b51a2a..8a43295c7 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,41 @@ -.PHONY: all antlr bazel build clean docker-images gazelle licenses mocks protobuf scion-topo test test-acceptance +.PHONY: all build build-dev dist-deb antlr clean docker-images gazelle go.mod licenses mocks protobuf scion-topo test test-integration write_all_source_files git-version -GAZELLE_MODE?=fix -GAZELLE_DIRS=. +build-dev: + rm -f bin/* + bazel build //:scion //:scion-ci + tar -kxf bazel-bin/scion.tar -C bin + tar -kxf bazel-bin/scion-ci.tar -C bin -build: bazel +build: + rm -f bin/* + bazel build //:scion + tar -kxf bazel-bin/scion.tar -C bin + +# BFLAGS is optional. It may contain additional command line flags for CI builds. Currently this is: +# "--file_name_version=$(tools/git-version)" to include the git version in the artifacts names. +dist-deb: + bazel build //dist:deb_all $(BFLAGS) + @ # These artefacts have unique names but varied locations. Link them somewhere convenient. + @ mkdir -p installables + @ cd installables ; ln -sfv ../bazel-out/*/bin/dist/*.deb . + +dist-openwrt: + bazel build //dist:openwrt_all $(BFLAGS) + @ # These artefacts have unique names but varied locations. Link them somewhere convenient. + @ mkdir -p installables + @ cd installables ; ln -sfv ../bazel-out/*/bin/dist/*.ipk . + +dist-openwrt-testing: + bazel build //dist:openwrt_testing_all $(BFLAGS) + @ # These artefacts have unique names but varied locations. Link them somewhere convenient. + @ mkdir -p installables + @ cd installables ; ln -sfv ../bazel-out/*/bin/dist/*.ipk . + +dist-rpm: + bazel build //dist:rpm_all $(BFLAGS) + @ # These artefacts have unique names but varied locations. Link them somewhere convenient. + @ mkdir -p installables + @ cd installables ; ln -sfv ../bazel-out/*/bin/dist/*.rpm . # all: performs the code-generation steps and then builds; the generated code # is git controlled, and therefore this is only necessary when changing the @@ -11,17 +43,17 @@ build: bazel # Use NOTPARALLEL to force correct order. # Note: From GNU make 4.4, this still allows building any other targets (e.g. lint) in parallel. .NOTPARALLEL: all -all: go_deps.bzl protobuf mocks gazelle licenses build antlr +all: go_deps.bzl protobuf mocks gazelle build-dev antlr write_all_source_files licenses clean: bazel clean rm -f bin/* + docker image ls --filter label=org.scion -q | xargs --no-run-if-empty docker image rm -bazel: +scrub: + bazel clean --expunge rm -f bin/* - bazel build //:scion //:scion-ci - tar -kxf bazel-bin/scion.tar -C bin - tar -kxf bazel-bin/scion-ci.tar -C bin + rm -f installables/* test: bazel test --config=unit_all @@ -29,16 +61,19 @@ test: test-integration: bazel test --config=integration_all +go.mod: + bazel run --config=quiet @go_sdk//:bin/go -- mod tidy + go_deps.bzl: go.mod - bazel run //:gazelle -- update-repos -prune -from_file=go.mod -to_macro=go_deps.bzl%go_deps + bazel run --verbose_failures --config=quiet //:gazelle_update_repos @# XXX(matzf): clean up; gazelle update-repose inconsistently inserts blank lines (see bazelbuild/bazel-gazelle#1088). @sed -e '/def go_deps/,$${/^$$/d}' -i go_deps.bzl docker-images: - @echo "Build perapp images" - bazel run //docker:prod - @echo "Build scion tester" - bazel run //docker:test + @echo "Build images" + bazel build //docker:prod //docker:test + @echo "Load images" + @bazel cquery '//docker:prod union //docker:test' --output=files 2>/dev/null | xargs -I{} docker load --input {} scion-topo: bazel build //:scion-topo @@ -50,21 +85,25 @@ protobuf: rm -f pkg/proto/*/*.pb.go cp -r bazel-bin/pkg/proto/*/go_default_library_/github.com/scionproto/scion/pkg/proto/* pkg/proto cp -r bazel-bin/pkg/proto/*/*/go_default_library_/github.com/scionproto/scion/pkg/proto/* pkg/proto - chmod 0644 pkg/proto/*/*.pb.go + chmod 0644 pkg/proto/*/*.pb.go pkg/proto/*/*/*.pb.go mocks: tools/gomocks.py -gazelle: - bazel run //:gazelle --config=quiet -- update -mode=$(GAZELLE_MODE) -go_naming_convention go_default_library $(GAZELLE_DIRS) +gazelle: go_deps.bzl + bazel run //:gazelle --verbose_failures --config=quiet + ./tools/buildrill/go_integration_test_sync licenses: tools/licenses.sh antlr: - antlr/generate.sh $(GAZELLE_MODE) + antlr/generate.sh fix + +write_all_source_files: + bazel run //:write_all_source_files -.PHONY: lint lint-bazel lint-bazel-buildifier lint-doc lint-doc-mdlint lint-go lint-go-bazel lint-go-gazelle lint-go-golangci lint-go-semgrep lint-openapi lint-openapi-spectral lint-protobuf lint-protobuf-buf +.PHONY: lint lint-bazel lint-bazel-buildifier lint-doc lint-doc-mdlint lint-doc-sphinx lint-go lint-go-bazel lint-go-gazelle lint-go-golangci lint-go-semgrep lint-openapi lint-openapi-spectral lint-protobuf lint-protobuf-buf # Enable --keep-going if all goals specified on the command line match the pattern "lint%" ifeq ($(filter-out lint%, $(MAKECMDGOALS)), ) @@ -77,43 +116,53 @@ lint-go: lint-go-gazelle lint-go-bazel lint-go-golangci lint-go-semgrep lint-go-gazelle: $(info ==> $@) - @$(MAKE) -s gazelle GAZELLE_MODE=diff + bazel run //:gazelle_diff --verbose_failures --config=quiet lint-go-bazel: $(info ==> $@) - @tools/quiet bazel test --config lint + @tools/quiet bazel test --config lint //... + +GO_BUILD_TAGS_ARG=$(shell bazel info --ui_event_filters=-stdout,-stderr --announce_rc --noshow_progress 2>&1 | grep "'build' options" | sed -n "s/^.*--define gotags=\(\S*\).*/--build-tags \1/p" ) lint-go-golangci: $(info ==> $@) @if [ -t 1 ]; then tty=true; else tty=false; fi; \ - tools/quiet docker run --tty=$$tty --rm -v golangci-lint-modcache:/go -v golangci-lint-buildcache:/root/.cache -v "${PWD}:/src" -w /src golangci/golangci-lint:v1.50.0 golangci-lint run --config=/src/.golangcilint.yml --timeout=3m --skip-dirs doc ./... + tools/quiet docker run --tty=$$tty --rm -v golangci-lint-modcache:/go -v golangci-lint-buildcache:/root/.cache -v "${PWD}:/src" -w /src golangci/golangci-lint:v1.60.3 golangci-lint run --config=/src/.golangcilint.yml --timeout=3m $(GO_BUILD_TAGS_ARG) --skip-dirs doc ./... lint-go-semgrep: $(info ==> $@) @if [ -t 1 ]; then tty=true; else tty=false; fi; \ tools/quiet docker run --tty=$$tty --rm -v "${PWD}:/src" returntocorp/semgrep@sha256:3bef9d533a44e6448c43ac38159d61fad89b4b57f63e565a8a55ca265273f5ba semgrep --config=/src/tools/lint/semgrep --error -lint-bazel: lint-bazel-buildifier +lint-bazel: lint-bazel-buildifier lint-bazel-writeall lint-bazel-buildifier: $(info ==> $@) @tools/quiet bazel run --config=quiet //:buildifier_check +lint-bazel-writeall: + $(info ==> $@) + @tools/quiet ./tools/lint/write_source_files_sync + lint-protobuf: lint-protobuf-buf lint-protobuf-buf: $(info ==> $@) - @tools/quiet bazel run --config=quiet @buf_bin//file:buf -- check lint + @tools/quiet bazel run --config=quiet @buf//:buf -- lint $(PWD) --path $(PWD)/proto lint-openapi: lint-openapi-spectral lint-openapi-spectral: $(info ==> $@) - @tools/quiet bazel run --config=quiet @rules_openapi_npm//@stoplight/spectral-cli/bin:spectral -- lint --ruleset ${PWD}/spec/.spectral.yml ${PWD}/spec/*.gen.yml + @tools/quiet bazel run --config=quiet //:spectral -- lint --ruleset ${PWD}/spec/.spectral.yml ${PWD}/spec/*.gen.yml -lint-doc: lint-doc-mdlint +lint-doc: lint-doc-mdlint lint-doc-sphinx lint-doc-mdlint: $(info ==> $@) - @FILES=$$(find -type f -iname '*.md' -not -path "./rules_openapi/tools/node_modules/*" -not -path "./.github/**/*" | grep -vf tools/md/skipped); \ - docker run --rm -v ${PWD}:/data -v ${PWD}/tools/md/mdlintstyle.rb:/style.rb $$(docker build -q tools/md) $${FILES} -s /style.rb + @if [ -t 1 ]; then tty=true; else tty=false; fi; \ + tools/quiet docker run --tty=$$tty --rm -v ${PWD}:/workdir davidanson/markdownlint-cli2:v0.12.1 + +lint-doc-sphinx: + $(info ==> $@) + @tools/quiet bazel test --config=lint //doc:sphinx_lint_test diff --git a/WORKSPACE b/WORKSPACE index c8b7dc780..1bd17a7c1 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,11 +1,8 @@ workspace( name = "com_github_scionproto_scion", - managed_directories = { - "@rules_openapi_npm": ["rules_openapi/tools/node_modules"], - }, ) -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # linter rules http_archive( @@ -29,42 +26,55 @@ lint_setup({ "flake8": "//:flake8_lint_config", }) +http_archive( + name = "aspect_bazel_lib", + sha256 = "714cf8ce95a198bab0a6a3adaffea99e929d2f01bf6d4a59a2e6d6af72b4818c", + strip_prefix = "bazel-lib-2.7.8", + url = "https://github.com/aspect-build/bazel-lib/releases/download/v2.7.8/bazel-lib-v2.7.8.tar.gz", +) + +load("@aspect_bazel_lib//lib:repositories.bzl", "aspect_bazel_lib_dependencies", "aspect_bazel_lib_register_toolchains") + +# Required bazel-lib dependencies + +aspect_bazel_lib_dependencies() + +# Register bazel-lib toolchains + +aspect_bazel_lib_register_toolchains() + # Bazel rules for Golang http_archive( name = "io_bazel_rules_go", - sha256 = "16e9fca53ed6bd4ff4ad76facc9b7b651a89db1689a2877d6fd7b82aa824e366", + patch_args = ["-p0"], + patches = ["//patches:io_bazel_rules_go/import.patch"], + sha256 = "af47f30e9cbd70ae34e49866e201b3f77069abb111183f2c0297e7e74ba6bbc0", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.34.0/rules_go-v0.34.0.zip", - "https://github.com/bazelbuild/rules_go/releases/download/v0.34.0/rules_go-v0.34.0.zip", + "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip", + "https://github.com/bazelbuild/rules_go/releases/download/v0.47.0/rules_go-v0.47.0.zip", ], ) load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") +go_rules_dependencies() + go_register_toolchains( nogo = "@//:nogo", - version = "1.19.2", + version = "1.22.7", ) # Gazelle http_archive( name = "bazel_gazelle", - patch_args = ["-p1"], - patches = [ - # PR: https://github.com/bazelbuild/bazel-gazelle/pull/1243 - "@//patches/bazel_gazelle:gazelle.patch", - ], - sha256 = "501deb3d5695ab658e82f6f6f549ba681ea3ca2a5fb7911154b5aa45596183fa", + sha256 = "d3fa66a39028e97d76f9e2db8f1b0c11c099e8e01bf363a923074784e451f809", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.26.0/bazel-gazelle-v0.26.0.tar.gz", - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.26.0/bazel-gazelle-v0.26.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz", + "https://github.com/bazelbuild/bazel-gazelle/releases/download/v0.33.0/bazel-gazelle-v0.33.0.tar.gz", ], ) -load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies", "go_repository") - -go_rules_dependencies() - +load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") load("//:tool_deps.bzl", "tool_deps") tool_deps() @@ -74,47 +84,48 @@ load("//:go_deps.bzl", "go_deps") go_deps() -## Explictly override xerrors: https://github.com/bazelbuild/bazel-gazelle/issues/1217 -go_repository( - name = "org_golang_x_xerrors", - importpath = "golang.org/x/xerrors", - sum = "h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=", - version = "v0.0.0-20200804184101-5ec99f83aff1", -) - gazelle_dependencies() -# XXX Needs to be before rules_docker # Python rules http_archive( name = "rules_python", - sha256 = "8c8fe44ef0a9afc256d1e75ad5f448bb59b81aba149b8958f02f7b3a98f5d9b4", - strip_prefix = "rules_python-0.13.0", - url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz", + sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", + strip_prefix = "rules_python-0.26.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", ) -load("@rules_python//python:repositories.bzl", "python_register_toolchains") +load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains") + +py_repositories() python_register_toolchains( name = "python3_10", python_version = "3.10", ) -load("@python3_10//:defs.bzl", "interpreter") +load("@python3_10//:defs.bzl", python_interpreter = "interpreter") load("//tools/env/pip3:deps.bzl", "python_deps") -python_deps(interpreter) +python_deps(python_interpreter) load("@com_github_scionproto_scion_python_deps//:requirements.bzl", install_python_deps = "install_deps") install_python_deps() +load("//doc:deps.bzl", "python_doc_deps") + +python_doc_deps(python_interpreter) + +load("@com_github_scionproto_scion_python_doc_deps//:requirements.bzl", install_python_doc_deps = "install_deps") + +install_python_doc_deps() + http_archive( name = "rules_pkg", - sha256 = "62eeb544ff1ef41d786e329e1536c1d541bb9bcad27ae984d57f18f314018e66", + sha256 = "8f9ee2dc10c1ae514ee599a8b42ed99fa262b757058f65ad3c384289ff70c4b8", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", - "https://github.com/bazelbuild/rules_pkg/releases/download/0.6.0/rules_pkg-0.6.0.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.9.1/rules_pkg-0.9.1.tar.gz", ], ) @@ -125,103 +136,100 @@ rules_pkg_dependencies() # Antlr rules http_archive( name = "rules_antlr", - sha256 = "234c401cfabab78f2d7f5589239d98f16f04338768a72888f660831964948ab1", - strip_prefix = "rules_antlr-0.6.0", - urls = ["https://github.com/artisoft-io/rules_antlr/archive/0.6.0.tar.gz"], + # XXX(roosd): This hash is not guaranteed to be stable by GitHub. + # See: https://github.blog/changelog/2023-01-30-git-archive-checksums-may-change + sha256 = "a9b2f98aae1fb26e9608be1e975587e6271a3287e424ced28cbc77f32190ec41", + strip_prefix = "rules_antlr-0.6.1", + urls = ["https://github.com/bacek/rules_antlr/archive/refs/tags/0.6.1.tar.gz"], ) load("@rules_antlr//antlr:repositories.bzl", "rules_antlr_dependencies") -rules_antlr_dependencies("4.9.3") +rules_antlr_dependencies("4.13.1") +# Rules for container image building http_archive( - name = "io_bazel_rules_docker", - sha256 = "85ffff62a4c22a74dbd98d05da6cf40f497344b3dbf1e1ab0a37ab2a1a6ca014", - strip_prefix = "rules_docker-0.23.0", - urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.23.0/rules_docker-v0.23.0.tar.gz"], + name = "rules_oci", + sha256 = "4a276e9566c03491649eef63f27c2816cc222f41ccdebd97d2c5159e84917c3b", + strip_prefix = "rules_oci-1.7.4", + url = "https://github.com/bazel-contrib/rules_oci/releases/download/v1.7.4/rules_oci-v1.7.4.tar.gz", ) -load("@io_bazel_rules_docker//repositories:repositories.bzl", container_repositories = "repositories") +load("@rules_oci//oci:dependencies.bzl", "rules_oci_dependencies") -container_repositories() +rules_oci_dependencies() -load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps") +load("@rules_oci//oci:repositories.bzl", "LATEST_CRANE_VERSION", "oci_register_toolchains") -container_deps() - -load("@io_bazel_rules_docker//go:image.bzl", _go_image_repos = "repositories") - -_go_image_repos() - -http_archive( - name = "rules_deb_packages", - sha256 = "674ce7b66c345aaa9ab898608618a0a0db857cbed8e8d0794ca46e375fd5ff76", - urls = ["https://github.com/petermylemans/rules_deb_packages/releases/download/v0.4.0/rules_deb_packages.tar.gz"], +oci_register_toolchains( + name = "oci", + crane_version = LATEST_CRANE_VERSION, ) -load("@rules_deb_packages//:repositories.bzl", "deb_packages_dependencies") - -deb_packages_dependencies() - -load("@rules_deb_packages//:deb_packages.bzl", "deb_packages") - -deb_packages( - name = "debian_buster_amd64", - arch = "amd64", - packages = { - "libc6": "pool/main/g/glibc/libc6_2.28-10_amd64.deb", - "libcap2": "pool/main/libc/libcap2/libcap2_2.25-2_amd64.deb", - "libcap2-bin": "pool/main/libc/libcap2/libcap2-bin_2.25-2_amd64.deb", - }, - packages_sha256 = { - "libc6": "6f703e27185f594f8633159d00180ea1df12d84f152261b6e88af75667195a79", - "libcap2": "8f93459c99e9143dfb458353336c5171276860896fd3e10060a515cd3ea3987b", - "libcap2-bin": "3c8c5b1410447356125fd8f5af36d0c28853b97c072037af4a1250421008b781", - }, - sources = [ - "http://deb.debian.org/debian buster main", - "http://deb.debian.org/debian buster-updates main", - "http://deb.debian.org/debian-security buster/updates main", - ], - timestamp = "20210812T060609Z", - urls = [ - "http://deb.debian.org/debian/$(package_path)", - "http://deb.debian.org/debian-security/$(package_path)", - "https://snapshot.debian.org/archive/debian/$(timestamp)/$(package_path)", # Needed in case of supersed archive no more available on the mirrors - "https://snapshot.debian.org/archive/debian-security/$(timestamp)/$(package_path)", # Needed in case of supersed archive no more available on the mirrors - ], -) - -load("@io_bazel_rules_docker//container:container.bzl", "container_pull") - -container_pull( - name = "static_debian10", - digest = "sha256:4433370ec2b3b97b338674b4de5ffaef8ce5a38d1c9c0cb82403304b8718cde9", - registry = "gcr.io", - repository = "distroless/static-debian10", -) +load("@rules_oci//oci:pull.bzl", "oci_pull") -container_pull( - name = "debug_debian10", +oci_pull( + name = "distroless_base_debian10", digest = "sha256:72d496b69d121960b98ac7078cbacd7678f1941844b90b5e1cac337b91309d9d", registry = "gcr.io", repository = "distroless/base-debian10", ) -container_pull( +oci_pull( name = "debian10", digest = "sha256:60cb30babcd1740309903c37d3d408407d190cf73015aeddec9086ef3f393a5d", registry = "index.docker.io", repository = "library/debian", - tag = "10", +) + +# Debian packaging +http_archive( + name = "rules_debian_packages", + sha256 = "0ae3b332f9d894e57693ce900769d2bd1b693e1f5ea1d9cdd82fa4479c93bcc8", + strip_prefix = "rules_debian_packages-0.2.0", + url = "https://github.com/bazel-contrib/rules_debian_packages/releases/download/v0.2.0/rules_debian_packages-v0.2.0.tar.gz", +) + +load("@rules_debian_packages//debian_packages:repositories.bzl", "rules_debian_packages_dependencies") + +rules_debian_packages_dependencies(python_interpreter_target = python_interpreter) + +load("@rules_debian_packages//debian_packages:defs.bzl", "debian_packages_repository") + +debian_packages_repository( + name = "tester_debian10_packages", + default_arch = "amd64", + default_distro = "debian10", + lock_file = "//docker:tester_packages.lock", +) + +load("@tester_debian10_packages//:packages.bzl", tester_debian_packages_install_deps = "install_deps") + +tester_debian_packages_install_deps() + +# RPM packaging +load("@rules_pkg//toolchains/rpm:rpmbuild_configure.bzl", "find_system_rpmbuild") + +find_system_rpmbuild( + name = "rules_pkg_rpmbuild", + verbose = False, +) + +# Buf CLI +http_archive( + name = "buf", + build_file_content = "exports_files([\"buf\"])", + sha256 = "16253b6702dd447ef941b01c9c386a2ab7c8d20bbbc86a5efa5953270f6c9010", + strip_prefix = "buf/bin", + urls = ["https://github.com/bufbuild/buf/releases/download/v1.32.2/buf-Linux-x86_64.tar.gz"], ) # protobuf/gRPC http_archive( name = "rules_proto_grpc", - sha256 = "7954abbb6898830cd10ac9714fbcacf092299fda00ed2baf781172f545120419", - strip_prefix = "rules_proto_grpc-3.1.1", - urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/archive/3.1.1.tar.gz"], + sha256 = "9ba7299c5eb6ec45b6b9a0ceb9916d0ab96789ac8218269322f0124c0c0d24e2", + strip_prefix = "rules_proto_grpc-4.5.0", + urls = ["https://github.com/rules-proto-grpc/rules_proto_grpc/releases/download/4.5.0/rules_proto_grpc-4.5.0.tar.gz"], ) load("@rules_proto_grpc//:repositories.bzl", "rules_proto_grpc_repos", "rules_proto_grpc_toolchains") @@ -236,48 +244,68 @@ rules_proto_dependencies() rules_proto_toolchains() -load("@rules_proto_grpc//python:repositories.bzl", rules_proto_grpc_python_repos = "python_repos") - -rules_proto_grpc_python_repos() +load("@rules_proto_grpc//buf:repositories.bzl", rules_proto_grpc_buf_repos = "buf_repos") -load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") - -grpc_deps() +rules_proto_grpc_buf_repos() http_archive( name = "com_github_bazelbuild_buildtools", - strip_prefix = "buildtools-master", - url = "https://github.com/bazelbuild/buildtools/archive/2.2.1.zip", -) - -http_file( - name = "buf_bin", - downloaded_file_path = "buf", - executable = True, - sha256 = "5faf15ed0a3cd4bd0919ba5fcb95334c1fd2ba32770df289d615138fa188d36a", + strip_prefix = "buildtools-6.3.3", urls = [ - "https://github.com/bufbuild/buf/releases/download/v0.20.5/buf-Linux-x86_64", + "https://github.com/bazelbuild/buildtools/archive/refs/tags/6.3.3.tar.gz", ], ) load("//tools/lint/python:deps.bzl", "python_lint_deps") -python_lint_deps(interpreter) +python_lint_deps(python_interpreter) load("@com_github_scionproto_scion_python_lint_deps//:requirements.bzl", install_python_lint_deps = "install_deps") install_python_lint_deps() -load("//rules_openapi:dependencies.bzl", "rules_openapi_dependencies") +http_archive( + name = "aspect_rules_js", + sha256 = "a723815986f3dd8b2c58d0ea76fde0ed56eed65de3212df712e631e5fc7d8790", + strip_prefix = "rules_js-2.0.0-rc6", + url = "https://github.com/aspect-build/rules_js/releases/download/v2.0.0-rc6/rules_js-v2.0.0-rc6.tar.gz", +) + +load("@aspect_rules_js//js:repositories.bzl", "rules_js_dependencies") + +rules_js_dependencies() + +load("@rules_nodejs//nodejs:repositories.bzl", "nodejs_register_toolchains") -rules_openapi_dependencies() +nodejs_register_toolchains( + name = "nodejs", + node_version = "16.19.0", # use DEFAULT_NODE_VERSION from previous version rules_nodejs; the current version links against too new glibc +) + +load("@aspect_rules_js//npm:repositories.bzl", "npm_translate_lock") + +npm_translate_lock( + name = "npm", + pnpm_lock = "@com_github_scionproto_scion//private/mgmtapi/tools:pnpm-lock.yaml", + pnpm_version = "9.4.0", + verify_node_modules_ignored = "@com_github_scionproto_scion//:.bazelignore", +) -load("//rules_openapi:install.bzl", "rules_openapi_install_yarn_dependencies") +load("@npm//:repositories.bzl", "npm_repositories") -rules_openapi_install_yarn_dependencies() +npm_repositories() -# TODO(lukedirtwalker): can that be integrated in the rules_openapi_dependencies -# call above somehow? -load("@cgrindel_bazel_starlib//:deps.bzl", "bazel_starlib_dependencies") +# Support cross building and packaging for openwrt_amd64 via the openwrt SDK +http_archive( + name = "openwrt_x86_64_SDK", + build_file = "@//dist/openwrt:BUILD.external.bazel", + patch_args = ["-p1"], + patches = ["@//dist/openwrt:endian_h.patch"], + sha256 = "df9cbce6054e6bd46fcf28e2ddd53c728ceef6cb27d1d7fc54a228f272c945b0", + strip_prefix = "openwrt-sdk-23.05.2-x86-64_gcc-12.3.0_musl.Linux-x86_64", + urls = ["https://downloads.openwrt.org/releases/23.05.2/targets/x86/64/openwrt-sdk-23.05.2-x86-64_gcc-12.3.0_musl.Linux-x86_64.tar.xz"], +) -bazel_starlib_dependencies() +register_toolchains( + "//dist/openwrt:x86_64_openwrt_toolchain", +) diff --git a/acceptance/README.md b/acceptance/README.md index cab20b6f3..49b8b435a 100644 --- a/acceptance/README.md +++ b/acceptance/README.md @@ -3,6 +3,9 @@ This directory contains a set of integration tests. Each test is defined as a bazel test target, with tags `integration` and `exclusive`. +Some integration tests use code outside this directory. For example, the +`router_multi` acceptance test cases and main executable are in `tools/braccept`. + ## Basic Commands To run all integration tests which include the acceptance tests, execute one of @@ -18,6 +21,7 @@ Run a subset of the tests by specifying a different list of targets: ```bash bazel test --config=integration //acceptance/cert_renewal:all //acceptance/trc_update/... +bazel test --config=integration //acceptance/router_multi:all --cache_test_results=no ``` The following the flags to bazel test can be helpful when running individual tests: @@ -27,8 +31,8 @@ The following the flags to bazel test can be helpful when running individual tes ## Manual Testing -Some of the tests are defined using a common framework, defined in the -bazel rules `topogen_test` and `raw_test`. +Some of the tests are defined using a common framework, implemented by the +bazel rules `topogen_test` and `raw_test` (in [raw.bzl](acceptance/common/raw.bzl)). These test cases allow more fine grained interaction. ```bash @@ -41,5 +45,13 @@ bazel run //:_run bazel run //:_teardown ``` +For example: + +```bash +bazel run //acceptance/router_multi:test_bfd_setup +bazel run //acceptance/router_multi:test_bfd_run +bazel run //acceptance/router_multi:test_bfd_teardown +``` + See [common/README](common/README.md) for more information about the internal structure of these tests. diff --git a/acceptance/app_vs_endhost_br_dispatch/BUILD.bazel b/acceptance/app_vs_endhost_br_dispatch/BUILD.bazel new file mode 100644 index 000000000..d3554771c --- /dev/null +++ b/acceptance/app_vs_endhost_br_dispatch/BUILD.bazel @@ -0,0 +1,9 @@ +load("//acceptance/common:topogen.bzl", "topogen_test") + +topogen_test( + name = "test", + src = "test.py", + args = ["--executable=end2end_integration:$(location //tools/end2end_integration)"], + data = ["//tools/end2end_integration"], + topo = "//acceptance/app_vs_endhost_br_dispatch/testdata:topology.topo", +) diff --git a/acceptance/app_vs_endhost_br_dispatch/test.py b/acceptance/app_vs_endhost_br_dispatch/test.py new file mode 100755 index 000000000..602e86f7a --- /dev/null +++ b/acceptance/app_vs_endhost_br_dispatch/test.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +# Copyright 2023 ETH Zurich + +from acceptance.common import base +from acceptance.common import scion + + +class Test(base.TestTopogen): + """ + Constructs a simple test topology with one core, two leaf ASes. + Each of them will run a different mix between BR that will replicate + the legacy endhost-port-dispatch behaviour (i.e., they will send + traffic to its own AS to the endhost default port) and + application-port-dispatch routers (i.e., they will rewrite the underlay + UDP/IP destination port with the UDP/SCION port). + + AS 1-ff00:0:1 is core. + AS 1-ff00:0:2, 1-ff00:0:3 are leaves. + + We use the shortnames AS1, AS2, etc. for the ASes above. + + AS1 contains a BR with the port rewriting configuration to the default + range. It also includes a shim dispatcher. + AS2 contains a BR with a configuration that imitates the old + behaviour, i.e., sending all traffic to default endhost port 30041. + It also includes a shim dispatcher. + AS3 contains a BR with the port rewriting configuration to the default + range. It does not include the shim dispatcher. + """ + + def setup_prepare(self): + super().setup_prepare() + + br_as_2_id = "br1-ff00_0_2-1" + + br_as_2_file = self.artifacts / "gen" / "ASff00_0_2" \ + / ("%s.toml" % br_as_2_id) + scion.update_toml({"router.dispatched_port_start": 0, + "router.dispatched_port_end": 0}, + [br_as_2_file]) + + def setup_start(self): + super().setup_start() + self.await_connectivity() + + def _run(self): + ping_test = self.get_executable("end2end_integration") + ping_test["-d", "-outDir", self.artifacts].run_fg() + + +if __name__ == "__main__": + base.main(Test) diff --git a/acceptance/app_vs_endhost_br_dispatch/testdata/BUILD.bazel b/acceptance/app_vs_endhost_br_dispatch/testdata/BUILD.bazel new file mode 100644 index 000000000..d803c4eee --- /dev/null +++ b/acceptance/app_vs_endhost_br_dispatch/testdata/BUILD.bazel @@ -0,0 +1,3 @@ +exports_files([ + "topology.topo", +]) diff --git a/acceptance/app_vs_endhost_br_dispatch/testdata/topology.topo b/acceptance/app_vs_endhost_br_dispatch/testdata/topology.topo new file mode 100644 index 000000000..68b249b07 --- /dev/null +++ b/acceptance/app_vs_endhost_br_dispatch/testdata/topology.topo @@ -0,0 +1,15 @@ +--- # Test Topology +ASes: + "1-ff00:0:1": + core: true + voting: true + authoritative: true + issuing: true + "1-ff00:0:2": + cert_issuer: 1-ff00:0:1 + "1-ff00:0:3": + cert_issuer: 1-ff00:0:1 + test_dispatcher: False +links: + - {a: "1-ff00:0:1#2", b: "1-ff00:0:2#1", linkAtoB: CHILD} + - {a: "1-ff00:0:1#3", b: "1-ff00:0:3#1", linkAtoB: CHILD} diff --git a/acceptance/cert_renewal/test.py b/acceptance/cert_renewal/test.py index 0b53b2883..2a3a9ef30 100755 --- a/acceptance/cert_renewal/test.py +++ b/acceptance/cert_renewal/test.py @@ -53,6 +53,7 @@ class Test(base.TestTopogen): """ def _run(self): + self.await_connectivity() isd_ases = scion.ASList.load("%s/gen/as_list.yml" % self.artifacts).all @@ -71,9 +72,9 @@ def _run(self): end2end.run_fg() logger.info("==> Shutting down control servers and purging caches") - for container in self.dc.list_containers("scion_sd.*"): + for container in self.dc.list_containers("sd.*"): self.dc("rm", container) - for container in self.dc.list_containers("scion_cs.*"): + for container in self.dc.list_containers("cs.*"): self.dc.stop_container(container) for cs_config in cs_configs: files = list((pathlib.Path(self.artifacts) / @@ -84,7 +85,7 @@ def _run(self): logger.info("==> Restart containers") self.setup_start() - time.sleep(5) + self.await_connectivity() logger.info("==> Check connectivity") end2end.run_fg() diff --git a/acceptance/common/README.md b/acceptance/common/README.md index 0ca1fc308..8ee5e3a99 100644 --- a/acceptance/common/README.md +++ b/acceptance/common/README.md @@ -32,9 +32,8 @@ following switches: environment variable `TEST_UNDECLARED_OUTPUTS_DIR` - `--executable :`; specifies path for an executable used in the test. This can be used to run executables that are built by bazel. -- `--container-loader #`; load the specified container images and - define the tags referenced in the test. - This can be used to run containers that are built by bazel. +- `--docker-image `; load the specified container images tars. + This can be used to load images that are built by bazel. - `--topo `; path to the .topo file for topogen tests - `--setup-params `; additional parameters for topogen. @@ -56,7 +55,7 @@ Tests can use: - `self.artifacts`: the specified directory for test outputs, created during setup. - `self.get_executable()`: returns an executable specified using the `--executable` switch. -- `self.dc`: a wrapper for `docker-compose`, instantiated during `TestTopogen.setup`. +- `self.dc`: a wrapper for `docker compose`, instantiated during `TestTopogen.setup`. The `base.main` function is the main entry point to run the tests and must be invoked in `__main__`. diff --git a/acceptance/common/base.py b/acceptance/common/base.py index a40d46dab..73e07f2fa 100644 --- a/acceptance/common/base.py +++ b/acceptance/common/base.py @@ -73,8 +73,8 @@ class TestBase(ABC): def _set_executables(self, executables): self.executables = {name: executable for (name, executable) in executables} - container_loaders = cli.SwitchAttr("container-loader", ContainerLoader, list=True, - help="Container loader, format tag#path") + docker_images = cli.SwitchAttr("docker-image", cli.ExistingFile, list=True, + help="Docker image tar files") artifacts = cli.SwitchAttr("artifacts-dir", LocalPath, @@ -117,7 +117,7 @@ def setup_prepare(self): """ docker.assert_no_networks() self._setup_artifacts() - self._setup_container_loaders() + self._setup_docker_images() # Define where coredumps will be stored. print( cmd.docker("run", "--rm", "--privileged", "alpine", "sysctl", "-w", @@ -132,16 +132,10 @@ def _setup_artifacts(self): cmd.mkdir(self.artifacts) print("artifacts dir: %s" % self.artifacts) - def _setup_container_loaders(self): - for tag, script in self.container_loaders: - o = local[script]() - idx = o.index("as ") - if idx < 0: - logger.error("extracting tag from loader script %s" % tag) - continue - bazel_tag = o[idx+len("as "):].strip() - logger.info("docker tag %s %s" % (bazel_tag, tag)) - cmd.docker("tag", bazel_tag, tag) + def _setup_docker_images(self): + for tar in self.docker_images: + o = cmd.docker("load", "--input", tar) + print(o.strip()) def get_executable(self, name: str): """Resolve the executable by name. @@ -203,7 +197,7 @@ def setup_start(self): raise Exception("Failed services.\n" + ps) def teardown(self): - # Avoid running docker-compose teardown if setup_prepare failed + # Avoid running docker compose teardown if setup_prepare failed if self._setup_prepare_failed: return out_dir = self.artifacts / "logs" @@ -213,6 +207,26 @@ def teardown(self): if re.search(r"Exit\s+[1-9]\d*", ps): raise Exception("Failed services.\n" + ps) + def await_connectivity(self, quiet_seconds=None, timeout_seconds=None): + """ + Wait for the beaconing process in a local topology to establish full connectivity, i.e. at + least one path between any two ASes. + Runs the tool/await-connectivity script. + + Returns success when full connectivity is established or an error (exception) at + timeout (default 20s). + + Remains quiet for a configurable time (default 10s). After that, + it reports the missing segments at 1s interval. + """ + cmd = self.get_executable("await-connectivity") + cmd.cwd = self.artifacts + if quiet_seconds is not None: + cmd = cmd["-q", str(quiet_seconds)] + if timeout_seconds is not None: + cmd = cmd["-t", str(timeout_seconds)] + cmd.run_fg() + def execute_tester(self, isd_as: ISD_AS, cmd: str, *args: str) -> str: """Executes a command in the designated "tester" container for the specified ISD-AS. diff --git a/acceptance/common/docker.py b/acceptance/common/docker.py index b77a7f8b4..0605214da 100644 --- a/acceptance/common/docker.py +++ b/acceptance/common/docker.py @@ -35,15 +35,12 @@ from plumbum import cmd SCION_DC_FILE = "gen/scion-dc.yml" -DC_PROJECT = "scion" SCION_TESTING_DOCKER_ASSERTIONS_OFF = 'SCION_TESTING_DOCKER_ASSERTIONS_OFF' class Compose(object): def __init__(self, - project: str = DC_PROJECT, compose_file: str = SCION_DC_FILE): - self.project = project self.compose_file = compose_file def __call__(self, *args, **kwargs) -> str: @@ -51,7 +48,7 @@ def __call__(self, *args, **kwargs) -> str: # Note: not using plumbum here due to complications with encodings in the captured output try: res = subprocess.run( - ["docker-compose", "-f", self.compose_file, "-p", self.project, *args], + ["docker", "compose", "-f", self.compose_file, *args], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") except subprocess.CalledProcessError as e: raise _CalledProcessErrorWithOutput(e) from None @@ -64,8 +61,9 @@ def collect_logs(self, out_dir: str = "logs/docker"): for svc in self("config", "--services").splitlines(): # Collect logs. dst_f = out_p / "%s.log" % svc + print(svc) with open(dst_f, "w") as log_file: - cmd.docker.run(args=("logs", svc), stdout=log_file, + cmd.docker.run(args=("logs", "scion-"+svc+"-1"), stdout=log_file, stderr=subprocess.STDOUT, retcode=None) # Collect coredupms. coredump_f = out_p / "%s.coredump" % svc @@ -200,7 +198,7 @@ def assert_no_networks(writer=None): writer.write("Docker networking assertions are OFF\n") return - allowed_nets = ['bridge', 'host', 'none'] + allowed_nets = ['bridge', 'host', 'none', 'benchmark'] unexpected_nets = [] for net in _get_networks(): if net.name not in allowed_nets: diff --git a/acceptance/common/raw.bzl b/acceptance/common/raw.bzl index 5d8734d7e..2f024ca8d 100644 --- a/acceptance/common/raw.bzl +++ b/acceptance/common/raw.bzl @@ -1,6 +1,11 @@ load("//tools/lint:py.bzl", "py_binary", "py_library", "py_test") load("@com_github_scionproto_scion_python_deps//:requirements.bzl", "requirement") +# Bug in bazel: HOME isn't set to TEST_TMPDIR. +# Bug in docker-compose v2.21 a writable HOME is required (eventhough not used). +# Poor design in Bazel, there's no sane way to obtain the path to some +# location that's not a litteral dependency. +# So, HOME must be provided by the invoker. def raw_test( name, src, @@ -8,6 +13,7 @@ def raw_test( deps = [], data = [], tags = [], + homedir = "", local = False): py_library( name = "%s_lib" % name, @@ -63,5 +69,6 @@ def raw_test( "PYTHONUNBUFFERED": "1", # Ensure that unicode output can be printed to the log/console "PYTHONIOENCODING": "utf-8", + "HOME": homedir, }, ) diff --git a/acceptance/common/scion.py b/acceptance/common/scion.py index 9ad00ae03..c330fb20c 100644 --- a/acceptance/common/scion.py +++ b/acceptance/common/scion.py @@ -14,7 +14,7 @@ import logging import json -from typing import Any, Dict, List, MutableMapping +from typing import Any, Dict, List, MutableMapping, Mapping import toml import yaml @@ -69,6 +69,29 @@ def update_json(change_dict: Dict[str, Any], files: LocalPath): json.dump(t, f, indent=2) +def load_from_json(key: str, files: LocalPath) -> Any: + """ Reads the value associated with the given key from the given json files. + + The first value found is returned. If not found, None is returned. + + Args: + key: dot separated path of the JSON key. + files: names of file or files to read. + + Returns: + The value. None if the path doesn't exist in the dictionary tree. + + Raises: + IOError / FileNotFoundError: File path is not valid + """ + for file in files: + with open(file, "r") as f: + t = json.load(f) + v = val_at_path(t, key) + if v is not None: + return v + + class ASList: """ ASList is a list of AS separated by core and non-core ASes. It can be loaded @@ -117,6 +140,18 @@ def path_to_dict(path: str, val: Any) -> Dict: return d +def val_at_path(d: Mapping[str, Any], path: str) -> Any: + """ + Walks nested dictionaries by following the given path and returns the value + associated with the leaf key. + """ + v = d + for k in path.split('.'): + v = v.get(k, None) + if not isinstance(v, Mapping): + return v + + def merge_dict(change_dict: Dict[str, Any], orig_dict: MutableMapping[str, Any]): """ Merge changes into the original dictionary. Leaf values in the change dict diff --git a/acceptance/common/topogen.bzl b/acceptance/common/topogen.bzl index 675cee50a..b400803b2 100644 --- a/acceptance/common/topogen.bzl +++ b/acceptance/common/topogen.bzl @@ -1,6 +1,11 @@ load("//tools/lint:py.bzl", "py_binary", "py_library", "py_test") load("@com_github_scionproto_scion_python_deps//:requirements.bzl", "requirement") +# Bug in bazel: HOME isn't set to TEST_TMPDIR. +# Bug in docker-compose v2.21 a writable HOME is required (eventhough not used). +# Poor design in Bazel, there's no sane way to obtain the path to some +# location that's not a litteral dependency. +# So, HOME must be provided by the invoker. def topogen_test( name, src, @@ -10,14 +15,15 @@ def topogen_test( args = [], deps = [], data = [], - tester = "//docker:tester"): + homedir = "", + tester = "//docker:tester.tarball"): """Creates a test based on a topology file. It creates a target specified by the 'name' argument that runs the entire test. Additionally, It creates _setup, _run and _teardown targets that allow to run the test in stages. - Args: + Args:cc name: name of the test src: the source code of the test topo: the topology (.topo) file to use for the test @@ -45,6 +51,7 @@ def topogen_test( common_args = [ "--executable=scion-pki:$(location //scion-pki/cmd/scion-pki)", "--executable=topogen:$(location //tools:topogen)", + "--executable=await-connectivity:$(location //tools:await_connectivity)", "--topo=$(location %s)" % topo, ] if gateway: @@ -54,13 +61,23 @@ def topogen_test( "//scion-pki/cmd/scion-pki", "//tools:topogen", "//tools:docker_ip", + "//tools:await_connectivity", topo, ] - loaders = container_loaders(tester, gateway) - for tag in loaders: - loader = loaders[tag] - common_data = common_data + ["%s" % loader] - common_args = common_args + ["--container-loader=%s#$(location %s)" % (tag, loader)] + docker_images = [ + "//docker:control.tarball", + "//docker:daemon.tarball", + "//docker:dispatcher.tarball", + "//docker:router.tarball", + ] + if tester: + docker_images += [tester] + if gateway: + docker_images += ["//docker:gateway.tarball"] + + for tar in docker_images: + common_data = common_data + [tar] + common_args = common_args + ["--docker-image=$(location %s)" % tar] py_binary( name = "%s_setup" % name, @@ -103,17 +120,6 @@ def topogen_test( "PYTHONUNBUFFERED": "1", # Ensure that unicode output can be printed to the log/console "PYTHONIOENCODING": "utf-8", + "HOME": homedir, }, ) - -def container_loaders(tester, gateway): - images = { - "control:latest": "//docker:control", - "daemon:latest": "//docker:daemon", - "dispatcher:latest": "//docker:dispatcher", - "tester:latest": tester, - "posix-router:latest": "//docker:posix_router", - } - if gateway: - images["posix-gateway:latest"] = "//docker:posix_gateway" - return images diff --git a/acceptance/hidden_paths/test.py b/acceptance/hidden_paths/test.py index c3144c9e4..305609667 100755 --- a/acceptance/hidden_paths/test.py +++ b/acceptance/hidden_paths/test.py @@ -3,13 +3,11 @@ # Copyright 2020 Anapaya Systems import http.server -import time import threading -from plumbum import cmd - from acceptance.common import base from acceptance.common import scion +from tools.topology.scion_addr import ISD_AS class Test(base.TestTopogen): @@ -45,6 +43,12 @@ class Test(base.TestTopogen): Expect no connectivity: AS3 <-> AS4 (Group ff00:0:2-3 to group ff00:0:2-4) """ + _ases = { + "2": "1-ff00:0:2", + "3": "1-ff00:0:3", + "4": "1-ff00:0:4", + "5": "1-ff00:0:5", + } http_server_port = 9099 @@ -61,16 +65,6 @@ def setup_prepare(self): "4": "172.20.0.65", "5": "172.20.0.73", } - # XXX(lukedirtwalker): The ports below are the dynamic QUIC server - # ports. Thanks to the docker setup they are setup consistently so we - # can use them. Optimally we would define a static server port inside - # the CS and use that one instead. - control_addresses = { - "2": "172.20.0.51:32768", - "3": "172.20.0.59:32768", - "4": "172.20.0.67:32768", - "5": "172.20.0.75:32768", - } # Each AS participating in hidden paths has their own hidden paths configuration file. hp_configs = { "2": "hp_groups_as2_as5.yml", @@ -98,13 +92,16 @@ def setup_prepare(self): # even though some don't need the registration service. as_dir_path = self.artifacts / "gen" / ("ASff00_0_%s" % as_number) + # The hidden_segment services are behind the same server as the control_service. + topology_file = as_dir_path / "topology.json" + control_service_addr = scion.load_from_json( + 'control_service.%s.addr' % control_id, [topology_file]) topology_update = { "hidden_segment_lookup_service.%s.addr" % control_id: - control_addresses[as_number], + control_service_addr, "hidden_segment_registration_service.%s.addr" % control_id: - control_addresses[as_number], + control_service_addr, } - topology_file = as_dir_path / "topology.json" scion.update_json(topology_update, [topology_file]) def setup_start(self): @@ -112,47 +109,39 @@ def setup_start(self): ("0.0.0.0", self.http_server_port), http.server.SimpleHTTPRequestHandler) server_thread = threading.Thread(target=configuration_server, args=[server]) server_thread.start() + self._server = server super().setup_start() - time.sleep(4) # Give applications time to download configurations - self._testers = { - "2": "tester_1-ff00_0_2", - "3": "tester_1-ff00_0_3", - "4": "tester_1-ff00_0_4", - "5": "tester_1-ff00_0_5", - } - self._ases = { - "2": "1-ff00:0:2", - "3": "1-ff00:0:3", - "4": "1-ff00:0:4", - "5": "1-ff00:0:5", - } - server.shutdown() + self.await_connectivity() + self._server.shutdown() # by now configuration must have been downloaded everywhere def _run(self): # Group 3 - self._showpaths_bidirectional("2", "3", 0) - self._showpaths_bidirectional("2", "5", 0) - self._showpaths_bidirectional("3", "5", 0) + self._showpaths_bidirectional("2", "3") + self._showpaths_bidirectional("2", "5") + self._showpaths_bidirectional("3", "5") # Group 4 - self._showpaths_bidirectional("2", "4", 0) - self._showpaths_bidirectional("2", "5", 0) - self._showpaths_bidirectional("4", "5", 0) + self._showpaths_bidirectional("2", "4") + self._showpaths_bidirectional("2", "5") + self._showpaths_bidirectional("4", "5") # Group 3 X 4 - self._showpaths_bidirectional("3", "4", 1) - - def _showpaths_bidirectional(self, source: str, destination: str, retcode: int): - self._showpaths_run(source, destination, retcode) - self._showpaths_run(destination, source, retcode) - - def _showpaths_run(self, source_as: str, destination_as: str, retcode: int): - print(cmd.docker("exec", "-t", self._testers[source_as], "scion", - "sp", self._ases[destination_as], - "--timeout", "2s", - retcode=retcode)) + try: + self._showpaths_bidirectional("3", "4") + except Exception as e: + print(e) + else: + raise AssertionError("Unexpected success; should not have paths 3 -> 4") + + def _showpaths_bidirectional(self, source: str, destination: str): + self._showpaths_run(source, destination) + self._showpaths_run(destination, source) + + def _showpaths_run(self, source_as: str, destination_as: str): + print(self.execute_tester(ISD_AS(self._ases[source_as]), + "scion", "sp", self._ases[destination_as], "--timeout", "2s")) def configuration_server(server): diff --git a/acceptance/router_benchmark/BUILD.bazel b/acceptance/router_benchmark/BUILD.bazel new file mode 100644 index 000000000..420219394 --- /dev/null +++ b/acceptance/router_benchmark/BUILD.bazel @@ -0,0 +1,61 @@ +load("//acceptance/common:raw.bzl", "raw_test") +load("//tools/lint:py.bzl", "py_binary", "py_library") +load("@com_github_scionproto_scion_python_deps//:requirements.bzl", "requirement") + +exports_files( + [ + "conf", + "test.py", + "conf/router.toml", + "conf/topology.json", + "conf/keys/master0.key", + "conf/keys/master1.key", + ], + visibility = ["//visibility:public"], +) + +args = [ + "--executable", + "brload:$(location //acceptance/router_benchmark/brload:brload)", + "--executable", + "coremark:$(location //tools/coremark:coremark)", + "--executable", + "mmbm:$(location //tools/mmbm:mmbm)", + "--docker-image=$(location //docker:router.tarball)", +] + +data = [ + ":conf", + "//docker:router.tarball", + "//acceptance/router_benchmark/brload:brload", + "//tools/coremark:coremark", + "//tools/mmbm:mmbm", +] + +raw_test( + name = "test", + src = "test.py", + args = args, + data = data, + homedir = "$(rootpath //docker:router.tarball)", + # This test uses sudo and accesses /var/run/netns. + local = True, + deps = ["benchmarklib"], +) + +py_library( + name = "benchmarklib", + srcs = ["benchmarklib.py"], +) + +# To ensure that the linter runs over this. Cannot actually be run with +# bazel run; it is meant to be executed from the command line. +py_binary( + name = "benchmark", + srcs = ["benchmark.py"], + visibility = ["//visibility:public"], + deps = [ + "benchmarklib", + requirement("plumbum"), + ], +) diff --git a/acceptance/router_benchmark/benchmark.py b/acceptance/router_benchmark/benchmark.py new file mode 100755 index 000000000..9f942b1e9 --- /dev/null +++ b/acceptance/router_benchmark/benchmark.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python3 + +# Copyright 2023 SCION Association +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import ipaddress +import json +import logging +import os +import ssl +import time +import traceback + +from benchmarklib import Intf, RouterBM +from collections import namedtuple +from plumbum import cli +from plumbum import cmd +from plumbum import local +from plumbum.cmd import docker +from plumbum.machines import LocalCommand +from random import randint +from urllib.request import urlopen + +logger = logging.getLogger(__name__) + +TEST_CASES = [ + "in", + "out", + "in_transit", + "out_transit", + "br_transit", +] + +# Convenience types to carry interface params. +IntfReq = namedtuple("IntfReq", "label, prefix_len, ip, peer_ip, exclusive") + + +def sudo(*args: [str]) -> str: + # -A, --askpass makes sure command is failing and does not wait for + # interactive password input. + return cmd.sudo("-A", *args) + + +class RouterBMTool(cli.Application, RouterBM): + """Evaluates the performance of an external router running the SCION reference implementation. + + The performance is reported in terms of packets per available machine*second (i.e. one + accumulated second of all configured CPUs available to execute the router code). + + The router can actually be anything that has compatible metrics scrapable by + prometheus. So that's presumably the reference implementation or a fork thereof. + + This test runs against a single router. The outgoing packets are not observed, the incoming + packets are fed directly by the test driver (brload). The other routers in the topology are a + fiction, they exist only in the routers configuration. + + The topology (./conf/topology.json) is the following: + + AS2 (br2) ---+== (br1a) AS1 (br1b) ---- (br4) AS4 + | + AS3 (br3) ---+ + + Only br1a is executed and observed. + + Pretend traffic is injected by brload's. See the test cases for details. + + """ + + avail_interfaces: list[str] = [] + mx_interface: str = None + to_flush: list[str] = [] + scrape_addr: str = None + + log_level = cli.SwitchAttr(["l", "loglevel"], str, default='warning', help="Logging level") + + doit = cli.Flag(["r", "run"], + help="Run the benchmark, as opposed to seeing the instructions.") + json = cli.Flag(["j", "json"], + help="Output the report in json format.") + + # Used by the RouterBM mixin: + coremark = cli.SwitchAttr(["c", "coremark"], int, default=0, + help="The coremark score of the subject machine.") + mmbm = cli.SwitchAttr(["m", "mmbm"], int, default=0, + help="The mmbm score of the subject machine.") + packet_size = cli.SwitchAttr(["s", "size"], int, default=172, + help="Test packet size (includes all headers - floored at 154).") + intf_map: dict[str, Intf] = {} + brload: LocalCommand = local["./bin/brload"] + brload_cpus: list[int] = [] + artifacts = f"{os.getcwd()}/acceptance/router_benchmark" + prom_address: str = "localhost:9090" + + def host_interface(self, excl: bool): + """Returns the next host interface that we should use for a brload links. + + If excl is true, we pick one and never pick that one again. + Else, we pick one the first time it's needed and keep it for reuse. + """ + if excl: + return self.avail_interfaces.pop() + + if self.mx_interface is None: + self.mx_interface = self.avail_interfaces.pop() + + return self.mx_interface + + def config_interface(self, req: IntfReq): + """Configure an interfaces according to brload's requirements. + + The device must not be in use for anything else. The devices are picked from a list + supplied by the user. + + We probably do not: + sudo("ip", "addr", "add", f"{req.peer_ip}/{req.prefix_len}", "dev", host_intf) + + It causes trouble: if an IP is assigned, the kernel responds with "unbound port" icmp + messages to the router traffic, which breaks the bound UDP connections that the router uses + for external interfaces. + + Args: + req: A requested router-side network interface. It comprises: + * A label by which brload identifies that interface. + * The IP address to be assigned to that interface. + * The IP address of one neighbor. + + """ + exclusive = req.exclusive == "true" + host_intf = self.host_interface(exclusive) + + # We need a means of connecting to the router's internal interface (from prometheus and + # to scrape the horsepower microbenchmark results. We pick one address of + # the router's subnet that's not otherwise used. This must NOT be "PeerIP". + # brload requires the internal interface to be "exclusive", that's our clue. + if exclusive: + net = ipaddress.ip_network(f"{req.ip}/{req.prefix_len}", strict=False) + hostAddr = next(net.hosts()) + 126 + self.scrape_addr = req.ip + sudo("ip", "addr", "add", f"{hostAddr}/{req.prefix_len}", + "broadcast", str(net.broadcast_address), "dev", host_intf) + self.to_flush.append(host_intf) + + logger.debug(f"=> Configuring interface {host_intf} for: {req}...") + + # We do multiplex most requested router interfaces onto one physical interface, so, we + # must check that we haven't already configured the physical one. + for i in self.intf_map.values(): + if i.name == host_intf: + break + else: + sudo("ip", "link", "set", host_intf, "mtu", "9000") + + # Do not assign the host addresses but create one link-local addr. + # Brload needs some src IP to send arp requests. (This requires rp_filter + # to be off on the router side, else, brload's arp requests are discarded). + sudo("ip", "addr", "add", f"169.254.{randint(0, 255)}.{randint(0, 255)}/16", + "broadcast", "169.254.255.255", + "dev", host_intf, "scope", "link") + sudo("sysctl", "-qw", f"net.ipv6.conf.{host_intf}.disable_ipv6=1") + self.to_flush.append(host_intf) + + # Fit for duty. + sudo("ip", "link", "set", host_intf, "up") + + # Ship it. Leave mac addresses alone. In this standalone test we use the real one. + self.intf_map[req.label] = Intf(host_intf, None, None) + + def fetch_horsepower(self) -> tuple[int]: + try: + url = f"https://{self.scrape_addr}/horsepower.json" + resp = urlopen(url, context=ssl._create_unverified_context()) + hp = json.loads(resp.read().decode("ascii")) + except Exception as e: + logger.warning(f"Fetching coremark and mmbm from {url}... {e}") + return + + if self.coremark == 0: + self.coremark = round(hp["coremark"]) + + if self.mmbm == 0: + self.mmbm = round(hp["mmbm"]) + + def setup(self, avail_interfaces: list[str]): + logger.info("Preparing...") + + # Check that the given interfaces are safe to use. We will wreck their config. + for intf in avail_interfaces: + output = sudo("ip", "addr", "show", "dev", intf) + if len(output.splitlines()) > 2: + logger.error(f"""\ + Interface {intf} appears to be in some kind of use. Cowardly refusing to modify it. + If you have a network manager, tell it to disable or ignore that interface. + Else, how about \"sudo ip addr flush dev {intf}\"? + """) + raise RuntimeError("Interface in use") + + # Looks safe. + self.avail_interfaces = avail_interfaces + + # Run test brload test with --show-interfaces and set up the interfaces as it says. + # We supply the label->host-side-name mapping to brload when we start it. + logger.debug("==> Configuring host interfaces...") + + output = self.brload("show-interfaces") + + lines = sorted(output.splitlines()) + for line in lines: + elems = line.split(",") + if len(elems) != 5: + continue + logger.debug(f"Requested by brload: {line}") + t = IntfReq._make(elems) + self.config_interface(t) + + # Start an instance of prometheus configured to scrape the router. + logger.debug("==> Starting prometheus...") + docker("run", + "-v", f"{self.artifacts}/conf:/etc/scion", + "-d", + "--network", "host", + "--name", "prometheus_bm", + "prom/prometheus:v2.47.2", + "--config.file", "/etc/scion/prometheus.yml") + + time.sleep(2) + + # Collect the horsepower microbenchmark numbers if we can. + # They'll be used to produce a performance index. + self.fetch_horsepower() + + logger.info("Prepared") + + def cleanup(self, retcode: int): + docker("rm", "-f", "prometheus_bm") + for intf in self.to_flush: + sudo("ip", "addr", "flush", "dev", intf) + return retcode + + def instructions(self): + output = self.brload("show-interfaces") + + exclusives = [] + multiplexed = [] + reqs = [] + intf_index = 0 + + # We sort the requests from brload because the interface that is picked for each can depends + # on the order in which we process them and we need to be consistent from run to run so + # the instructions we give the user actually work. + # (assuming brload's code doesn't change in-between). + + lines = sorted(output.splitlines()) + for line in lines: + elems = line.split(",") + if len(elems) != 5: + continue + req = IntfReq._make(elems) + reqs.append(req) + self.avail_interfaces.append(str(intf_index)) # Use numbers as placeholders + intf_index += 1 + + # TODO: Because of multiplexing, there are fewer real interfaces than labels requested + # by brload. So, not all placeholders get used (fine) and it happens that the low indices + # are the ones not used (confusing for the user). Currently we end-up with 1 and 2 + # (and no 0), which is acceptable but fortuitous. + for req in reqs: + e = req.exclusive == "true" + a = f"{req.ip}/{req.prefix_len}" + i = self.host_interface(e) + if e: + exclusives.append(f"{a} (must reach: #{i})") + else: + multiplexed.append(f"{a} (must reach: #{i})") + nl = "\n" + print(f""" +INSTRUCTIONS: + +1 - Configure your subject router according to accept/router_benchmark/conf/router.toml") + If using openwrt, an easy way to do that is to install the bmtools.ipk package. In addition, + bmtools includes two microbenchmarks: scion-coremark and scion-mmbm. Those will run + automatically and the results will be used to improve the benchmark report. + + Optional: If you did not install bmtools.ipk, install and run those microbenchmarks and make a + note of the results: (scion-coremark; scion-mmbm). + +2 - Configure the following interfaces on your router (The procedure depends on your router + UI) - All interfaces should have the mtu set to 9000: + - One physical interface with addresses: {", ".join(multiplexed)} +{nl.join([' - One physical interface with address: ' + s for s in exclusives])} + + IMPORTANT: if you're using a partitioned network (eg. multiple switches or no switches), + the "must reach" annotation matters. The '#' number is the order in which the corresponding host + interface must be given on the command line in step 7. + +3 - Connect the corresponding ports into your test switch (best if dedicated for the test). + +4 - Restart the scion-router service. + +5 - Pick the same number of physical interfaces on the system where you are running this + script. Make sure that these interface aren't used for anything else and have no assigned + IP addresses. Make a note of their names and, if using a partitioned network, associate each + with one of the numbers from step 2. + +6 - Connect the corresponding ports into your test switch. If using a partitioned network, make + sure that port is reachable by the corresponding subject router port. + +7 - Execute this script with arguments: --run , where is the list + of names you collected in step 5. If using a partitioned network, make sure to supply them + in the order indicated in step 2. + + If coremak and mmbm values are available, the report will include a performance index. + + If coremark and mmbm are not available from the test subject, you may supply them on the command + line. To that end, add the following arguments: "--coremark=", "--mmbm=", where + and are the results you optionally collected in step 1. + +8 - Be patient... + +9 - Read the report. +""") + + def main(self, *interfaces: str): + status = 1 + try: + logging.basicConfig(level=self.log_level.upper()) + if not self.doit: + self.instructions() + status = 0 + else: + self.setup(list(interfaces)) + results = self.run_bm(TEST_CASES) + # No CI_check. We have no particular expectations here. + # Output the performance in human-friendly form by default... + if self.json: + print(results.as_json()) + else: + print(results.as_report()) + status = 0 + except KeyboardInterrupt: + logger.info("Bailing out...") + except Exception: + logger.error(traceback.format_exc()) + except SystemExit: + pass + return status + + +if __name__ == "__main__": + RouterBMTool() diff --git a/acceptance/router_benchmark/benchmarklib.py b/acceptance/router_benchmark/benchmarklib.py new file mode 100644 index 000000000..4290de20b --- /dev/null +++ b/acceptance/router_benchmark/benchmarklib.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 + +# Copyright 2024 SCION Association +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Benchmarking code that is common to the CI benchmark test and the stand-alone +# benchmark program. + +import json +import logging + +from collections import namedtuple +from http.client import HTTPConnection +from plumbum import cmd +from urllib.parse import urlencode + +logger = logging.getLogger(__name__) + +# TODO(jchugly): this is still a work in progress. There are two unknowns in the model. +# M_COEF: the proportion by which memory performance contributes to throughput compared to +# arithmetic performance. NIC_CONSTANT: the fixed cost of the kernel interacting with the hardware +# to retrieve a frame. That one is hardware dependent and must be found by a third benchmark, so +# it is not theoretically a constant, but keeping it here to not forget. Until then, our performance +# index isn't really valid cross-hardware. M_COEF=400 gives roughly consistent results with the +# hardware we have. So, using that until we know more. NIC_CONSTANT seems to be around +# 1 microsecond. Using that, provisionally. + +M_COEF = 400 +NIC_CONSTANT = 1.0/1000000 + +# Intf: description of an interface configured for brload's use. Depending on context +# mac and peermac may be unused. "mac" is the MAC address configured on the side of the subject +# router. "peer_mac" is the MAC address that brload should use when simulating the peer router. +# If peer_mac is unset, we'll let brload use the true MAC address on its side, which is the default. +Intf = namedtuple("Intf", "name, mac, peer_mac") + + +class Results: + """Stores and format benchmark results. + """ + + cores: int = 0 + coremark: int = 0 + mmbm: int = 0 + packet_size: int = 0 + cases: list[dict] = [] + failed: list[dict] = [] + checked: bool = False + + def __init__(self, cores: int, coremark: int, mmbm: int, packet_size: int): + self.cores = cores + self.coremark = coremark + self.mmbm = mmbm + self.packet_size = packet_size + + def perf_index(self, rate: int) -> float: + # TODO(jiceatscion): The perf index assumes that line speed isn't the bottleneck. + # It almost never is, but ideally we'd need to run iperf3 to verify. + # mmbm is in mebiBytes/s, rate is in pkt/s + return rate * (1.0 / self.coremark + + M_COEF * self.packet_size / (self.mmbm * 1024 * 1024) + + NIC_CONSTANT) + + def add_case(self, name: str, rate: int, droppage: int, raw_rate: int): + dropRatio = round(float(droppage) / (rate + droppage), 2) + saturated = dropRatio > 0.03 + perf = 0.0 + if self.cores == 3 and self.coremark and self.mmbm: + perf = round(self.perf_index(rate), 1) + self.cases.append({"case": name, + "perf": perf, "rate": rate, "drop": dropRatio, + "bit_rate": rate * self.packet_size * 8, + "raw_pkt_rate": raw_rate, + "full": saturated}) + + def CI_check(self, expectations: dict[str, int]): + self.checked = True + for tc in self.cases: + want = expectations.get(tc["case"]) + if want is not None: + slow = tc["rate"] < want + unsaturated = not tc["full"] + if slow or unsaturated: + self.failed.append({"case": tc["case"], + "expected": want, "slow": slow, "unsaturated": unsaturated}) + + def as_json(self) -> str: + return json.dumps({ + "cores": self.cores, + "coremark": self.coremark, + "mmbm": self.mmbm, + "cases": self.cases, + "checked": self.checked, + "failed": self.failed, + }, indent=4) + + def as_report(self) -> str: + res = (f"Benchmark Results\n\ncores: {self.cores}\n" + f"coremark: {self.coremark or 'N/A'}\nmmbm: {self.mmbm or 'N/A'}\n") + for tc in self.cases: + res += (f"{tc['case']}: perf_index={tc['perf'] or 'N/A'}" + f" rate={tc['rate']} droppage={tc['drop']:.1%} saturated={tc['full']}\n") + res += "CI pass/fail: " + if not self.checked: + res += "N/A\n" + return res + res += "FAILED\n" if self.failed else "PASS\n" + if not self.failed: + return res + for failure in self.failed: + res += (f"{failure['case']} expected={failure['expected']}" + f" slow={failure['slow']} unsaturated={failure['unsaturated']}") + return res + + +class RouterBM(): + """Evaluates the performance of an external router running the SCION reference implementation. + + The performance is reported in terms of packets per available machine*second (i.e. one + accumulated second of all configured CPUs available to execute the router code). + + The router can actually be anything that has compatible metrics scrapable by + prometheus. So that's presumably the reference implementation or a fork thereof. + + This test runs against a single router. The outgoing packets are not observed, the incoming + packets are fed directly by the test driver (brload). The other routers in the topology are a + fiction, they exist only in the routers configuration. + + The topology (./conf/topology.json) is the following: + + AS2 (br2) ---+== (br1a) AS1 (br1b) ---- (br4) AS4 + | + AS3 (br3) ---+ + + Only br1a is executed and observed. + + Pretend traffic is injected by brload's. See the test cases for details. + + This class is a Mixin that borrows the following attributes from the host class: + * coremark: the coremark benchmark results. + * mmbm: the mmbm benchmark results. + * packet_size: the packet_size to use in the test cases. + * intf_map: the map "label->actual_interface" map to be passed to brload. + * brload: "localCmd" wraper for the brload executable (plumbum.machines.LocalCommand) + * brload_cpus: [int] cpus where it is acceptable to run brload ([] means any) + * artifacts: the data directory (passed to docker). + * prom_address: the address of the prometheus API a string in the form "host:port" + """ + + def exec_br_load(self, case: str, map_args: list[str], duration: int) -> str: + # For num-streams, attempt to distribute uniformly on many possible number of cores. + # 840 is a multiple of 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 15, 20, 21, 24, 28, ... + brload_args = [ + self.brload.executable, + "run", + "--artifacts", self.artifacts, + *map_args, + "--case", case, + "--duration", f"{duration}s", + "--num-streams", "840", + "--packet-size", f"{self.packet_size}", + ] + if self.brload_cpus: + brload_args = [ + "taskset", "-c", ",".join(map(str, self.brload_cpus)), + ] + brload_args + + return cmd.sudo("-A", *brload_args) + + def run_test_case(self, case: str, map_args: list[str]) -> (int, int): + logger.debug(f"==> Starting load {case}") + + # We transmit for 13 seconds and then ignore the first 3. + output = self.exec_br_load(case, map_args, 13) + end = "0" + for line in output.splitlines(): + logger.info("BrLoad output: " + line) + if line.startswith("metricsBegin"): + end = line.split()[3] # "... metricsEnd: " + + logger.debug(f"==> Collecting {case} performance metrics...") + + # The raw metrics are expressed in terms of core*seconds. We convert to machine*seconds + # which allows us to provide a projected packet/s; ...more intuitive than packets/core*s. + # We measure the rate over 10s. For best results we only look at the last 10 seconds. + # "end" reports a time when the transmission was still going on at maximum rate. + sampleTime = int(end) + prom_query = urlencode({ + 'time': f'{sampleTime}', + 'query': ( + 'sum by (instance, job) (' + f' rate(router_output_pkts_total{{job="BR", type="{case}"}}[10s])' + ')' + '/ on (instance, job) group_left()' + 'sum by (instance, job) (' + ' 1 - (rate(process_runnable_seconds_total[10s])' + ' / go_sched_maxprocs_threads)' + ')' + ) + }) + conn = HTTPConnection(self.prom_address) + conn.request("GET", f"/api/v1/query?{prom_query}") + resp = conn.getresponse() + if resp.status != 200: + raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}") + + # There's only one router, so whichever metric we get is the right one. + pld = json.loads(resp.read().decode("utf-8")) + processed = 0 + results = pld["data"]["result"] + for result in results: + ts, val = result["value"] + processed = int(float(val)) + break + + # Collect the raw packet rate too. Just so we can discover if the cpu-availability + # correction is bad. + prom_query = urlencode({ + 'time': f'{sampleTime}', + 'query': ( + 'sum by (instance, job) (' + f' rate(router_output_pkts_total{{job="BR", type="{case}"}}[10s])' + ')' + ) + }) + conn = HTTPConnection(self.prom_address) + conn.request("GET", f"/api/v1/query?{prom_query}") + resp = conn.getresponse() + if resp.status != 200: + raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}") + + # There's only one router, so whichever metric we get is the right one. + pld = json.loads(resp.read().decode("utf-8")) + raw = 0 + results = pld["data"]["result"] + for result in results: + ts, val = result["value"] + raw = int(float(val)) + break + + # Collect dropped packets metrics, so we can verify that the router was well saturated. + # If not, the metrics aren't very useful. + prom_query = urlencode({ + 'time': f'{sampleTime}', + 'query': ( + 'sum by (instance, job) (' + ' rate(router_dropped_pkts_total{job="BR", reason=~"busy_.*"}[10s])' + ')' + '/ on (instance, job) group_left()' + 'sum by (instance, job) (' + ' 1 - (rate(process_runnable_seconds_total[10s])' + ' / go_sched_maxprocs_threads)' + ')' + ) + }) + conn = HTTPConnection(self.prom_address) + conn.request("GET", f"/api/v1/query?{prom_query}") + resp = conn.getresponse() + if resp.status != 200: + raise RuntimeError(f"Unexpected response: {resp.status} {resp.reason}") + + # There's only one router, so whichever metric we get is the right one. + pld = json.loads(resp.read().decode("utf-8")) + dropped = 0 + results = pld["data"]["result"] + for result in results: + ts, val = result["value"] + dropped = int(float(val)) + break + + return processed, dropped, raw + + # Fetch and log the number of cores used by Go. This may inform performance + # modeling later. + def core_count(self) -> int: + logger.debug("==> Collecting number of cores...") + prom_query = urlencode({ + 'query': 'go_sched_maxprocs_threads{job="BR"}' + }) + + conn = HTTPConnection(self.prom_address) + conn.request("GET", f"/api/v1/query?{prom_query}") + resp = conn.getresponse() + if resp.status != 200: + raise RuntimeError(f"FAILED: Unexpected response: {resp.status} {resp.reason}") + + pld = json.loads(resp.read().decode("utf-8")) + results = pld["data"]["result"] + if not results: + raise RuntimeError("FAILED: Got no results when querying the core count") + if len(results) > 1: + raise RuntimeError(f"FAILED: Found more than one subject router in results: {results}") + + result = results[0] + _, val = result["value"] + return int(val) + + def run_bm(self, test_cases: [str]) -> Results: + logger.info("Benchmarking...") + + # Build the interface mapping arg (here, we do not override the brload side mac address) + map_args = [] + for label, intf in self.intf_map.items(): + if intf.peer_mac is not None: + map_args.extend(["--interface", f"{label}={intf.name},{intf.peer_mac}"]) + else: + map_args.extend(["--interface", f"{label}={intf.name}"]) + + # Run one test (30% size) as warm-up to trigger any frequency scaling, else the first test + # can get much lower performance. + logger.debug("Warmup") + self.exec_br_load(test_cases[0], map_args, 5) + + # Fetch the core count once. It doesn't change while the router is running. + # We cannot get this until the router has been up for a few seconds. If you shorten + # the warmup for some reason, make sure to add a delay. + cores = self.core_count() + + # At long last, run the tests. + results = Results(cores, self.coremark, self.mmbm, self.packet_size) + for test_case in test_cases: + logger.info(f"Case: {test_case}") + rate, droppage, raw = self.run_test_case(test_case, map_args) + results.add_case(test_case, rate or 1, droppage, raw) + + return results + logger.info("Benchmarked") diff --git a/acceptance/router_benchmark/brload/BUILD.bazel b/acceptance/router_benchmark/brload/BUILD.bazel new file mode 100644 index 000000000..f42158f1f --- /dev/null +++ b/acceptance/router_benchmark/brload/BUILD.bazel @@ -0,0 +1,31 @@ +load("//tools/lint:go.bzl", "go_library") +load("//:scion.bzl", "scion_go_binary") + +go_library( + name = "go_default_library", + srcs = [ + "main.go", + "mmsg.go", + ], + importpath = "github.com/scionproto/scion/acceptance/router_benchmark/brload", + visibility = ["//visibility:private"], + deps = [ + "//acceptance/router_benchmark/cases:go_default_library", + "//pkg/log:go_default_library", + "//pkg/private/serrors:go_default_library", + "//pkg/scrypto:go_default_library", + "//pkg/slayers:go_default_library", + "//private/keyconf:go_default_library", + "@com_github_google_gopacket//:go_default_library", + "@com_github_google_gopacket//afpacket:go_default_library", + "@com_github_google_gopacket//layers:go_default_library", + "@com_github_spf13_cobra//:go_default_library", + "@org_golang_x_sys//unix:go_default_library", + ], +) + +scion_go_binary( + name = "brload", + embed = [":go_default_library"], + visibility = ["//visibility:public"], +) diff --git a/acceptance/router_benchmark/brload/main.go b/acceptance/router_benchmark/brload/main.go new file mode 100644 index 000000000..22eafa432 --- /dev/null +++ b/acceptance/router_benchmark/brload/main.go @@ -0,0 +1,343 @@ +// Copyright 2023 SCION Association +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "hash" + "os" + "path/filepath" + "reflect" + "time" + + "github.com/google/gopacket" + "github.com/google/gopacket/afpacket" + "github.com/google/gopacket/layers" + "github.com/spf13/cobra" + + "github.com/scionproto/scion/acceptance/router_benchmark/cases" + "github.com/scionproto/scion/pkg/log" + "github.com/scionproto/scion/pkg/private/serrors" + "github.com/scionproto/scion/pkg/scrypto" + "github.com/scionproto/scion/pkg/slayers" + "github.com/scionproto/scion/private/keyconf" +) + +type Case func(packetSize int, mac hash.Hash) (string, string, []byte, []byte) + +type caseChoice string + +func (c *caseChoice) String() string { + return string(*c) +} + +func (c *caseChoice) Set(v string) error { + _, ok := allCases[v] + if !ok { + return errors.New("No such case") + } + *c = caseChoice(v) + return nil +} + +func (c *caseChoice) Type() string { + return "string enum" +} + +func (c *caseChoice) Allowed() string { + return fmt.Sprintf("One of: %v", reflect.ValueOf(allCases).MapKeys()) +} + +var ( + allCases = map[string]Case{ + "in": cases.In, + "out": cases.Out, + "in_transit": cases.InTransit, + "out_transit": cases.OutTransit, + "br_transit": cases.BrTransit, + } + logConsole string + dir string + testDuration time.Duration + packetSize int + numStreams uint16 + caseToRun caseChoice + interfaces []string +) + +func main() { + rootCmd := &cobra.Command{ + Use: "brload", + Short: "Generates traffic into a specific router of a specific topology", + } + intfCmd := &cobra.Command{ + Use: "show-interfaces", + Short: "Provides a terse list of the interfaces that this test requires", + Run: func(cmd *cobra.Command, args []string) { + os.Exit(showInterfaces(cmd)) + }, + } + runCmd := &cobra.Command{ + Use: "run", + Short: "Executes the test", + Run: func(cmd *cobra.Command, args []string) { + os.Exit(run(cmd)) + }, + } + runCmd.Flags().DurationVar(&testDuration, "duration", time.Second*15, "Test duration") + runCmd.Flags().IntVar(&packetSize, "packet-size", 172, "Total size of each packet sent") + runCmd.Flags().Uint16Var(&numStreams, "num-streams", 4, + "Number of independent streams (flowID) to use") + runCmd.Flags().StringVar(&logConsole, "log.console", "error", + "Console logging level: debug|info|error|etc.") + runCmd.Flags().StringVar(&dir, "artifacts", "", "Artifacts directory") + runCmd.Flags().Var(&caseToRun, "case", "Case to run. "+caseToRun.Allowed()) + runCmd.Flags().StringArrayVar(&interfaces, "interface", []string{}, + `label=[,] where is the host device that matches + the