diff --git a/.github/buildomat/build-and-test.sh b/.github/buildomat/build-and-test.sh index 64efc9eb3b..70babec2b4 100755 --- a/.github/buildomat/build-and-test.sh +++ b/.github/buildomat/build-and-test.sh @@ -4,8 +4,14 @@ set -o errexit set -o pipefail set -o xtrace -# Color the output for easier readability. -export CARGO_TERM_COLOR=always +# +# Set up our PATH for the test suite. +# + +# shellcheck source=/dev/null +source ./env.sh +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh target_os=$1 @@ -30,10 +36,7 @@ OUTPUT_DIR='/work' echo "tests will store non-ephemeral output in $OUTPUT_DIR" >&2 mkdir -p "$OUTPUT_DIR" -# -# Set up our PATH for the test suite. -# -source ./env.sh + banner prerequisites ptime -m bash ./tools/install_builder_prerequisites.sh -y diff --git a/.github/buildomat/ci-env.sh b/.github/buildomat/ci-env.sh new file mode 100644 index 0000000000..3971eeec3a --- /dev/null +++ b/.github/buildomat/ci-env.sh @@ -0,0 +1,6 @@ +# Setup shared across Buildomat CI builds. +# +# This file contains environment variables shared across Buildomat CI jobs. + +# Color the output for easier readability. +export CARGO_TERM_COLOR=always diff --git a/.github/buildomat/jobs/a4x2-deploy.sh b/.github/buildomat/jobs/a4x2-deploy.sh index 53153beafb..ba8d967266 100755 --- a/.github/buildomat/jobs/a4x2-deploy.sh +++ b/.github/buildomat/jobs/a4x2-deploy.sh @@ -21,6 +21,9 @@ set -o errexit set -o pipefail set -o xtrace +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh + pfexec mkdir -p /out pfexec chown "$UID" /out diff --git a/.github/buildomat/jobs/a4x2-prepare.sh b/.github/buildomat/jobs/a4x2-prepare.sh index daadec27a2..ae10da2ecb 100755 --- a/.github/buildomat/jobs/a4x2-prepare.sh +++ b/.github/buildomat/jobs/a4x2-prepare.sh @@ -22,7 +22,10 @@ #: ] #: enable = false +# shellcheck source=/dev/null source ./env.sh +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh set -o errexit set -o pipefail @@ -91,4 +94,3 @@ for x in ce cr1 cr2 g0 g1 g2 g3 tools omicron-common; do tar -czf cargo-bay-$x.tgz cargo-bay/$x mv cargo-bay-$x.tgz /out/ done - diff --git a/.github/buildomat/jobs/check-features.sh b/.github/buildomat/jobs/check-features.sh index 4ba97ec02f..03dbee4cfa 100644 --- a/.github/buildomat/jobs/check-features.sh +++ b/.github/buildomat/jobs/check-features.sh @@ -14,6 +14,9 @@ set -o errexit set -o pipefail set -o xtrace +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh + cargo --version rustc --version diff --git a/.github/buildomat/jobs/clippy.sh b/.github/buildomat/jobs/clippy.sh index 4040691b72..4cea9b1b88 100755 --- a/.github/buildomat/jobs/clippy.sh +++ b/.github/buildomat/jobs/clippy.sh @@ -16,13 +16,17 @@ set -o errexit set -o pipefail set -o xtrace -cargo --version -rustc --version - # # Set up our PATH for use with this workspace. # + +# shellcheck source=/dev/null source ./env.sh +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh + +cargo --version +rustc --version banner prerequisites ptime -m bash ./tools/install_builder_prerequisites.sh -y diff --git a/.github/buildomat/jobs/deploy.sh b/.github/buildomat/jobs/deploy.sh index 8820378e1c..018b532107 100755 --- a/.github/buildomat/jobs/deploy.sh +++ b/.github/buildomat/jobs/deploy.sh @@ -13,7 +13,8 @@ #: "%/pool/ext/*/crypt/debug/global/oxide-sled-agent:default.log.*", #: "%/pool/ext/*/crypt/debug/oxz_*/oxide-*.log.*", #: "%/pool/ext/*/crypt/debug/oxz_*/system-illumos-*.log.*", -#: "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*" +#: "!/pool/ext/*/crypt/debug/oxz_propolis-server_*/*.log.*", +#: "/tmp/kstat/*.kstat" #: ] #: skip_clone = true #: @@ -32,6 +33,10 @@ _exit_trap() { local status=$? set +o errexit + if [[ "x$OPTE_COMMIT" != "x" ]]; then + pfexec cp /tmp/opteadm /opt/oxide/opte/bin/opteadm + fi + # # Stop cron in all zones (to stop logadm log rotation) # @@ -65,12 +70,16 @@ _exit_trap() { PORTS=$(pfexec /opt/oxide/opte/bin/opteadm list-ports | tail +2 | awk '{ print $1; }') for p in $PORTS; do + pfexec /opt/oxide/opte/bin/opteadm dump-uft -p $p LAYERS=$(pfexec /opt/oxide/opte/bin/opteadm list-layers -p $p | tail +2 | awk '{ print $1; }') for l in $LAYERS; do pfexec /opt/oxide/opte/bin/opteadm dump-layer -p $p $l done done + mkdir -p /tmp/kstat + pfexec kstat -p xde: > /tmp/kstat/xde.kstat + pfexec zfs list pfexec zpool list pfexec fmdump -eVp @@ -88,9 +97,30 @@ _exit_trap() { for z in $(zoneadm list -n | grep oxz_ntp); do banner "${z/oxz_/}" - pfexec zlogin "$z" chronyc tracking - pfexec zlogin "$z" chronyc sources + pfexec zlogin "$z" chronyc -n tracking + pfexec zlogin "$z" chronyc -n sources -a pfexec zlogin "$z" cat /etc/inet/chrony.conf + pfexec zlogin "$z" ping -sn oxide.computer 56 1 + pfexec zlogin "$z" ping -sn 1.1.1.1 56 1 + pfexec zlogin "$z" /usr/sbin/dig 0.pool.ntp.org @1.1.1.1 + pfexec zlogin "$z" getent hosts time.cloudfare.com + + # Attempt to get chrony to do some time sync from the CLI with + # messages being written to the terminal and with debugging + # enabled if the chrony package was built with that option. + # Since chronyd on the CLI needs to use the ports that the + # service will be using, stop it first (with -s to wait for it + # to exit). + pfexec /usr/sbin/svcadm -z "$z" disable -s oxide/ntp + # Run in dry-run one-shot mode (-Q) + pfexec zlogin "$z" /usr/sbin/chronyd -t 10 -ddQ + # Run in one-shot mode (-q) -- attempt to set the clock + pfexec zlogin "$z" /usr/sbin/chronyd -t 10 -ddq + # Run in one-shot mode (-q) but override the configuration + # to talk to an explicit external service. This command line is + # similar to that used by the pre-flight NTP checks. + pfexec zlogin "$z" /usr/sbin/chronyd -t 10 -ddq \ + "'pool time.cloudflare.com iburst maxdelay 0.1'" done pfexec zlogin sidecar_softnpu cat /var/log/softnpu.log @@ -104,6 +134,20 @@ z_swadm () { pfexec zlogin oxz_switch /opt/oxide/dendrite/bin/swadm $@ } +# only set this if you want to override the version of opte/xde installed by the +# install_opte.sh script +OPTE_COMMIT="f3002b356da7d0e4ca15beb66a5566a92919baaa" +if [[ "x$OPTE_COMMIT" != "x" ]]; then + curl -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/module/$OPTE_COMMIT/xde + pfexec rem_drv xde || true + pfexec mv xde /kernel/drv/amd64/xde + pfexec add_drv xde || true + curl -sSfOL https://buildomat.eng.oxide.computer/public/file/oxidecomputer/opte/release/$OPTE_COMMIT/opteadm + chmod +x opteadm + cp opteadm /tmp/opteadm + pfexec mv opteadm /opt/oxide/opte/bin/opteadm +fi + # # XXX work around 14537 (UFS should not allow directories to be unlinked) which # is probably not yet fixed in xde branch? Once the xde branch merges from @@ -150,6 +194,9 @@ cd /opt/oxide/work ptime -m tar xvzf /input/package/work/package.tar.gz +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh + # Ask buildomat for the range of extra addresses that we're allowed to use, and # break them up into the ranges we need. diff --git a/.github/buildomat/jobs/omicron-common.sh b/.github/buildomat/jobs/omicron-common.sh index 0e248f9d3d..676c18f52a 100755 --- a/.github/buildomat/jobs/omicron-common.sh +++ b/.github/buildomat/jobs/omicron-common.sh @@ -14,6 +14,9 @@ set -o errexit set -o pipefail set -o xtrace +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh + cargo --version rustc --version diff --git a/.github/buildomat/jobs/package.sh b/.github/buildomat/jobs/package.sh index d9632f39e6..7a2cc2c369 100755 --- a/.github/buildomat/jobs/package.sh +++ b/.github/buildomat/jobs/package.sh @@ -13,6 +13,9 @@ set -o errexit set -o pipefail set -o xtrace +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh + cargo --version rustc --version @@ -48,6 +51,7 @@ mkdir tests # deployment phases of buildomat. files=( + .github/buildomat/ci-env.sh out/target/test out/npuzone/* package-manifest.toml diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index 221637bcaa..fbedf2ea7a 100755 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -47,6 +47,9 @@ set -o errexit set -o pipefail set -o xtrace +# shellcheck source=/dev/null +source .github/buildomat/ci-env.sh + cargo --version rustc --version diff --git a/.github/workflows/check-opte-ver.yml b/.github/workflows/check-opte-ver.yml index 777a649181..61eac8d73e 100644 --- a/.github/workflows/check-opte-ver.yml +++ b/.github/workflows/check-opte-ver.yml @@ -9,7 +9,7 @@ jobs: check-opte-ver: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Install jq diff --git a/.github/workflows/check-workspace-deps.yml b/.github/workflows/check-workspace-deps.yml index 135a1af58b..aca75aa9bc 100644 --- a/.github/workflows/check-workspace-deps.yml +++ b/.github/workflows/check-workspace-deps.yml @@ -10,7 +10,7 @@ jobs: check-workspace-deps: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Check Workspace Dependencies diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 2ac54771c9..25975873f3 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -16,14 +16,14 @@ jobs: env: RUSTFLAGS: -D warnings steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # v1 with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@9bef7e9c3d7c7aa986ef19933b0722880ae377e0 # v2 + uses: taiki-e/install-action@42f4ec8e42bf7fe4dadd39bfc534566095a8edff # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8bfb499726..33f60a68f9 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -14,7 +14,7 @@ jobs: check-style: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - name: Report cargo version @@ -37,10 +37,10 @@ jobs: - name: Disable packages.microsoft.com repo if: ${{ startsWith(matrix.os, 'ubuntu') }} run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 + - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version @@ -67,10 +67,10 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 + - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version @@ -95,10 +95,10 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 + - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version @@ -127,10 +127,10 @@ jobs: # This repo is unstable and unnecessary: https://github.com/microsoft/linux-package-repositories/issues/34 - name: Disable packages.microsoft.com repo run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - - uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3 + - uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5 if: ${{ github.ref != 'refs/heads/main' }} - name: Report cargo version run: cargo --version diff --git a/.github/workflows/update-dendrite.yml b/.github/workflows/update-dendrite.yml index 722be2c26a..93977b6f07 100644 --- a/.github/workflows/update-dendrite.yml +++ b/.github/workflows/update-dendrite.yml @@ -29,7 +29,7 @@ jobs: steps: # Checkout both the target and integration branches - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: token: ${{ inputs.reflector_access_token }} fetch-depth: 0 diff --git a/.github/workflows/update-maghemite.yml b/.github/workflows/update-maghemite.yml index 949373e275..24a4f73560 100644 --- a/.github/workflows/update-maghemite.yml +++ b/.github/workflows/update-maghemite.yml @@ -29,7 +29,7 @@ jobs: steps: # Checkout both the target and integration branches - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: token: ${{ inputs.reflector_access_token }} fetch-depth: 0 diff --git a/.github/workflows/validate-openapi-spec.yml b/.github/workflows/validate-openapi-spec.yml index c3bd899e40..b6203923a5 100644 --- a/.github/workflows/validate-openapi-spec.yml +++ b/.github/workflows/validate-openapi-spec.yml @@ -10,7 +10,7 @@ jobs: format: runs-on: ubuntu-22.04 steps: - - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 + - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 with: ref: ${{ github.event.pull_request.head.sha }} # see omicron#4461 - uses: actions/setup-node@0a44ba7841725637a19e28fa30b79a866c81b0a6 # v4.0.4 diff --git a/Cargo.lock b/Cargo.lock index 6adf89acf3..92585cf306 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -663,7 +663,7 @@ dependencies = [ [[package]] name = "bhyve_api" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927#11371b0f3743f8df5b047dc0edc2699f4bdf3927" dependencies = [ "bhyve_api_sys", "libc", @@ -673,7 +673,7 @@ dependencies = [ [[package]] name = "bhyve_api_sys" version = "0.0.0" -source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927" +source = "git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927#11371b0f3743f8df5b047dc0edc2699f4bdf3927" dependencies = [ "libc", "strum", @@ -876,8 +876,7 @@ dependencies = [ name = "bootstrap-agent-api" version = "0.1.0" dependencies = [ - "dropshot 0.12.0", - "nexus-client", + "dropshot", "omicron-common", "omicron-uuid-kinds", "omicron-workspace-hack", @@ -897,7 +896,7 @@ dependencies = [ "oxnet", "progenitor", "regress 0.9.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -1109,9 +1108,9 @@ dependencies = [ [[package]] name = "cfg-expr" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345c78335be0624ed29012dc10c49102196c6882c12dde65d9f35b02da2aada8" +checksum = "d0890061c4d3223e7267f3bad2ec40b997d64faac1c2815a4a9d95018e2b9e9c" dependencies = [ "smallvec 1.13.2", "target-lexicon", @@ -1141,7 +1140,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "dropshot 0.12.0", + "dropshot", "futures", "libc", "omicron-common", @@ -1243,9 +1242,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.18" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" +checksum = "b97f376d85a664d5837dbae44bf546e6477a679ff6610010f17276f686d867e8" dependencies = [ "clap_builder", "clap_derive", @@ -1253,9 +1252,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.18" +version = "4.5.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" +checksum = "19bc80abd44e4bed93ca373a0704ccbd1b710dc5749406201bb018272808dc54" dependencies = [ "anstream", "anstyle", @@ -1287,7 +1286,7 @@ name = "clickhouse-admin-api" version = "0.1.0" dependencies = [ "clickhouse-admin-types", - "dropshot 0.12.0", + "dropshot", "omicron-common", "omicron-uuid-kinds", "omicron-workspace-hack", @@ -1295,6 +1294,36 @@ dependencies = [ "serde", ] +[[package]] +name = "clickhouse-admin-keeper-client" +version = "0.1.0" +dependencies = [ + "chrono", + "clickhouse-admin-types", + "omicron-uuid-kinds", + "omicron-workspace-hack", + "progenitor", + "reqwest 0.12.8", + "schemars", + "serde", + "slog", +] + +[[package]] +name = "clickhouse-admin-server-client" +version = "0.1.0" +dependencies = [ + "chrono", + "clickhouse-admin-types", + "omicron-uuid-kinds", + "omicron-workspace-hack", + "progenitor", + "reqwest 0.12.8", + "schemars", + "serde", + "slog", +] + [[package]] name = "clickhouse-admin-types" version = "0.1.0" @@ -1351,7 +1380,7 @@ name = "cockroach-admin-api" version = "0.1.0" dependencies = [ "cockroach-admin-types", - "dropshot 0.12.0", + "dropshot", "omicron-common", "omicron-uuid-kinds", "omicron-workspace-hack", @@ -1367,7 +1396,7 @@ dependencies = [ "omicron-uuid-kinds", "omicron-workspace-hack", "progenitor", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "slog", @@ -1576,7 +1605,7 @@ name = "crdb-seed" version = "0.1.0" dependencies = [ "anyhow", - "dropshot 0.12.0", + "dropshot", "omicron-test-utils", "omicron-workspace-hack", "slog", @@ -1661,23 +1690,6 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" -[[package]] -name = "crossterm" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df" -dependencies = [ - "bitflags 2.6.0", - "crossterm_winapi", - "libc", - "mio 0.8.11", - "parking_lot 0.12.2", - "serde", - "signal-hook", - "signal-hook-mio", - "winapi", -] - [[package]] name = "crossterm" version = "0.28.1" @@ -1690,6 +1702,7 @@ dependencies = [ "mio 1.0.2", "parking_lot 0.12.2", "rustix", + "serde", "signal-hook", "signal-hook-mio", "winapi", @@ -1714,7 +1727,7 @@ dependencies = [ "crucible-workspace-hack", "percent-encoding", "progenitor", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -1741,7 +1754,7 @@ dependencies = [ "anyhow", "atty", "crucible-workspace-hack", - "dropshot 0.12.0", + "dropshot", "nix 0.29.0", "rusqlite", "rustls-pemfile 1.0.4", @@ -1773,7 +1786,7 @@ dependencies = [ "crucible-workspace-hack", "percent-encoding", "progenitor", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -1879,7 +1892,7 @@ dependencies = [ "digest", "fiat-crypto", "rand_core", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "subtle", "zeroize", ] @@ -1988,12 +2001,12 @@ dependencies = [ [[package]] name = "ddm-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?branch=hyper-v1-no-merge#b13b5b240f3967de753fd589b1036745d2770b52" +source = "git+https://github.com/oxidecomputer/maghemite?rev=056283eb02b6887fbf27f66a215662520f7c159c#056283eb02b6887fbf27f66a215662520f7c159c" dependencies = [ "oxnet", "percent-encoding", "progenitor", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "slog", @@ -2123,7 +2136,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "syn 2.0.79", ] @@ -2302,13 +2315,14 @@ dependencies = [ "clap", "dns-server-api", "dns-service-client", - "dropshot 0.12.0", + "dropshot", "expectorate", "hickory-client", "hickory-proto", "hickory-resolver", "hickory-server", "http 1.1.0", + "internal-dns-types", "omicron-test-utils", "omicron-workspace-hack", "openapi-lint", @@ -2335,7 +2349,8 @@ name = "dns-server-api" version = "0.1.0" dependencies = [ "chrono", - "dropshot 0.12.0", + "dropshot", + "internal-dns-types", "omicron-workspace-hack", "schemars", "serde", @@ -2345,13 +2360,13 @@ dependencies = [ name = "dns-service-client" version = "0.1.0" dependencies = [ - "anyhow", "chrono", "expectorate", "http 1.1.0", + "internal-dns-types", "omicron-workspace-hack", "progenitor", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "slog", @@ -2398,7 +2413,7 @@ dependencies = [ "quote", "rand", "regress 0.9.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "rustfmt-wrapper", "schemars", "serde", @@ -2408,52 +2423,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "dropshot" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a391eeedf8a75a188eb670327c704b7ab10eb2bb890e2ec0880dd21d609fb6e8" -dependencies = [ - "async-stream", - "async-trait", - "base64 0.22.1", - "bytes", - "camino", - "chrono", - "debug-ignore", - "dropshot_endpoint 0.10.1", - "form_urlencoded", - "futures", - "hostname 0.4.0", - "http 0.2.12", - "hyper 0.14.30", - "indexmap 2.5.0", - "multer", - "openapiv3", - "paste", - "percent-encoding", - "rustls 0.22.4", - "rustls-pemfile 2.1.3", - "schemars", - "scopeguard", - "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", - "sha1", - "slog", - "slog-async", - "slog-bunyan", - "slog-json", - "slog-term", - "tokio", - "tokio-rustls 0.25.0", - "toml 0.8.19", - "uuid", - "version_check", - "waitgroup", -] - [[package]] name = "dropshot" version = "0.12.0" @@ -2467,7 +2436,7 @@ dependencies = [ "camino", "chrono", "debug-ignore", - "dropshot_endpoint 0.12.0", + "dropshot_endpoint", "form_urlencoded", "futures", "hostname 0.4.0", @@ -2475,13 +2444,13 @@ dependencies = [ "http-body-util", "hyper 1.4.1", "hyper-util", - "indexmap 2.5.0", + "indexmap 2.6.0", "multer", "openapiv3", "paste", "percent-encoding", "rustls 0.22.4", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "schemars", "scopeguard", "serde", @@ -2503,19 +2472,6 @@ dependencies = [ "waitgroup", ] -[[package]] -name = "dropshot_endpoint" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9058c9c7e4a6b378cd12e71dc155bb15d0d4f8e1e6039ce2cf0a7c0c81043e33" -dependencies = [ - "proc-macro2", - "quote", - "serde", - "serde_tokenstream", - "syn 2.0.79", -] - [[package]] name = "dropshot_endpoint" version = "0.12.0" @@ -2683,7 +2639,7 @@ dependencies = [ "omicron-workspace-hack", "oxide-client", "rand", - "reqwest 0.12.7", + "reqwest 0.12.8", "russh", "russh-keys", "serde", @@ -2850,14 +2806,14 @@ checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" dependencies = [ "bit-set", "regex-automata", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] name = "fastrand" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" +checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" [[package]] name = "fatfs" @@ -3056,9 +3012,9 @@ checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" [[package]] name = "futures" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" dependencies = [ "futures-channel", "futures-core", @@ -3071,9 +3027,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", "futures-sink", @@ -3081,15 +3037,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] name = "futures-executor" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", @@ -3098,9 +3054,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" [[package]] name = "futures-lite" @@ -3117,9 +3073,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", @@ -3128,15 +3084,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" [[package]] name = "futures-task" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" [[package]] name = "futures-timer" @@ -3146,9 +3102,9 @@ checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" [[package]] name = "futures-util" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ "futures-channel", "futures-core", @@ -3175,7 +3131,7 @@ dependencies = [ name = "gateway-api" version = "0.1.0" dependencies = [ - "dropshot 0.12.0", + "dropshot", "gateway-types", "omicron-common", "omicron-uuid-kinds", @@ -3196,7 +3152,7 @@ dependencies = [ "gateway-messages", "omicron-common", "omicron-workspace-hack", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "slog", @@ -3218,7 +3174,7 @@ dependencies = [ "omicron-workspace-hack", "progenitor", "rand", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -3280,7 +3236,7 @@ name = "gateway-test-utils" version = "0.1.0" dependencies = [ "camino", - "dropshot 0.12.0", + "dropshot", "gateway-messages", "gateway-types", "omicron-gateway", @@ -3385,7 +3341,7 @@ dependencies = [ "bstr", "log", "regex-automata", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -3424,9 +3380,9 @@ dependencies = [ [[package]] name = "guppy" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3bff2f6a9d515cf6453282af93363f93bdf570792a6f4f619756e46696d773fa" +checksum = "bf47f1dcacf93614a4181e308b8341f5bf5d4f7ae2f67cc5a078df8e7023ea63" dependencies = [ "ahash", "camino", @@ -3435,7 +3391,7 @@ dependencies = [ "debug-ignore", "fixedbitset", "guppy-workspace-hack", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.13.0", "nested", "once_cell", @@ -3467,7 +3423,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3486,7 +3442,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.1.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", @@ -3546,6 +3502,12 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hashbrown" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e087f84d4f86bf4b218b927129862374b72199ae7d8657835f1e89000eea4fb" + [[package]] name = "hashlink" version = "0.9.1" @@ -3587,7 +3549,7 @@ checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f" dependencies = [ "atomic-polyfill", "hash32 0.2.1", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "spin 0.9.8", "stable_deref_trait", ] @@ -3996,7 +3958,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.7", "tokio", "tower-service", "tracing", @@ -4198,7 +4160,7 @@ dependencies = [ [[package]] name = "illumos-sys-hdrs" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=f3002b356da7d0e4ca15beb66a5566a92919baaa#f3002b356da7d0e4ca15beb66a5566a92919baaa" [[package]] name = "illumos-utils" @@ -4212,7 +4174,9 @@ dependencies = [ "camino-tempfile", "cfg-if", "crucible-smf", + "dropshot", "futures", + "http 1.1.0", "ipnetwork", "libc", "macaddr", @@ -4229,6 +4193,7 @@ dependencies = [ "serde", "serde_json", "slog", + "slog-error-chain", "smf", "thiserror", "tokio", @@ -4268,12 +4233,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68b900aa2f7301e21c36462b170ee99994de34dff39a4a6a528e80e7376d07e5" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.0", "serde", ] @@ -4352,7 +4317,7 @@ dependencies = [ "omicron-workspace-hack", "partial-io", "proptest", - "reqwest 0.12.7", + "reqwest 0.12.8", "sha2", "sled-hardware", "sled-hardware-types", @@ -4376,7 +4341,7 @@ name = "installinator-api" version = "0.1.0" dependencies = [ "anyhow", - "dropshot 0.12.0", + "dropshot", "hyper 1.4.1", "installinator-common", "omicron-common", @@ -4396,7 +4361,7 @@ dependencies = [ "omicron-workspace-hack", "progenitor", "regress 0.9.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -4436,25 +4401,41 @@ dependencies = [ ] [[package]] -name = "internal-dns" +name = "internal-dns-cli" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "dropshot", + "hickory-resolver", + "internal-dns-resolver", + "internal-dns-types", + "omicron-common", + "omicron-workspace-hack", + "slog", + "tokio", +] + +[[package]] +name = "internal-dns-resolver" version = "0.1.0" dependencies = [ "anyhow", "assert_matches", - "chrono", "dns-server", "dns-service-client", - "dropshot 0.12.0", + "dropshot", "expectorate", "futures", "hickory-resolver", - "hyper 1.4.1", + "internal-dns-types", "omicron-common", "omicron-test-utils", "omicron-uuid-kinds", "omicron-workspace-hack", "progenitor", - "reqwest 0.12.7", + "qorb", + "reqwest 0.12.8", "serde", "serde_json", "sled", @@ -4462,22 +4443,21 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "uuid", ] [[package]] -name = "internal-dns-cli" +name = "internal-dns-types" version = "0.1.0" dependencies = [ "anyhow", - "clap", - "dropshot 0.12.0", - "hickory-resolver", - "internal-dns", + "chrono", + "expectorate", "omicron-common", + "omicron-uuid-kinds", "omicron-workspace-hack", - "slog", - "tokio", + "schemars", + "serde", + "serde_json", ] [[package]] @@ -4515,9 +4495,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.10.0" +version = "2.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "187674a687eed5fe42285b40c6291f9a01517d415fad1c3cbc6a9f778af7fcd4" +checksum = "ddc24109865250148c2e0f3d25d4f0f479571723792d3802153c60922a4fb708" [[package]] name = "ipnetwork" @@ -4639,7 +4619,7 @@ dependencies = [ [[package]] name = "kstat-macro" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=f3002b356da7d0e4ca15beb66a5566a92919baaa#f3002b356da7d0e4ca15beb66a5566a92919baaa" dependencies = [ "quote", "syn 2.0.79", @@ -4753,7 +4733,7 @@ dependencies = [ "propolis-server-config", "rand", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "ron 0.7.1", "serde", "slog", @@ -4947,9 +4927,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "live-tests-macros" @@ -5136,13 +5116,13 @@ dependencies = [ [[package]] name = "mg-admin-client" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/maghemite?branch=hyper-v1-no-merge#b13b5b240f3967de753fd589b1036745d2770b52" +source = "git+https://github.com/oxidecomputer/maghemite?rev=056283eb02b6887fbf27f66a215662520f7c159c#056283eb02b6887fbf27f66a215662520f7c159c" dependencies = [ "anyhow", "chrono", "percent-encoding", "progenitor", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -5332,9 +5312,9 @@ dependencies = [ [[package]] name = "newtype-uuid" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526cb7c660872e401beaf3297f95f548ce3b4b4bdd8121b7c0713771d7c4a6e" +checksum = "4f4933943834e236c864a48aefdc2da43885dbd5eb77bff3ab20f31e0c3146f5" dependencies = [ "schemars", "serde", @@ -5360,7 +5340,7 @@ dependencies = [ "base64 0.22.1", "chrono", "cookie", - "dropshot 0.12.0", + "dropshot", "futures", "headers", "http 1.1.0", @@ -5403,7 +5383,7 @@ dependencies = [ "oxnet", "progenitor", "regress 0.9.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -5417,7 +5397,7 @@ version = "0.1.0" dependencies = [ "anyhow", "camino", - "dropshot 0.12.0", + "dropshot", "expectorate", "libc", "omicron-common", @@ -5505,17 +5485,19 @@ dependencies = [ "camino", "camino-tempfile", "chrono", + "clickhouse-admin-types", "const_format", "db-macros", "diesel", "diesel-dtrace", - "dropshot 0.12.0", + "dropshot", "expectorate", "futures", "gateway-client", "hyper-rustls 0.26.0", "illumos-utils", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "ipnetwork", "itertools 0.13.0", "macaddr", @@ -5591,7 +5573,7 @@ name = "nexus-external-api" version = "0.1.0" dependencies = [ "anyhow", - "dropshot 0.12.0", + "dropshot", "http 1.1.0", "hyper 1.4.1", "ipnetwork", @@ -5608,7 +5590,7 @@ dependencies = [ name = "nexus-internal-api" version = "0.1.0" dependencies = [ - "dropshot 0.12.0", + "dropshot", "nexus-types", "omicron-common", "omicron-uuid-kinds", @@ -5625,6 +5607,9 @@ dependencies = [ "anyhow", "base64 0.22.1", "chrono", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", + "clickhouse-admin-types", "expectorate", "futures", "gateway-client", @@ -5637,7 +5622,7 @@ dependencies = [ "omicron-uuid-kinds", "omicron-workspace-hack", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde_json", "sled-agent-client", "slog", @@ -5695,7 +5680,7 @@ dependencies = [ "omicron-common", "omicron-workspace-hack", "oxnet", - "reqwest 0.12.7", + "reqwest 0.12.8", "sled-agent-client", "slog", "uuid", @@ -5707,13 +5692,17 @@ version = "0.1.0" dependencies = [ "anyhow", "async-bb8-diesel", + "camino", "chrono", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", + "clickhouse-admin-types", "cockroach-admin-client", "diesel", - "dns-service-client", "futures", "httptest", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "ipnet", "newtype-uuid", "nexus-config", @@ -5735,7 +5724,7 @@ dependencies = [ "omicron-workspace-hack", "oxnet", "pq-sys", - "reqwest 0.12.7", + "reqwest 0.12.8", "sled-agent-client", "slog", "slog-error-chain", @@ -5754,8 +5743,8 @@ dependencies = [ "debug-ignore", "expectorate", "gateway-client", - "indexmap 2.5.0", - "internal-dns", + "indexmap 2.6.0", + "internal-dns-resolver", "ipnet", "maplit", "nexus-config", @@ -5869,7 +5858,7 @@ dependencies = [ "crucible-agent-client", "dns-server", "dns-service-client", - "dropshot 0.12.0", + "dropshot", "futures", "gateway-messages", "gateway-test-utils", @@ -5879,7 +5868,8 @@ dependencies = [ "http-body-util", "hyper 1.4.1", "illumos-utils", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "nexus-client", "nexus-config", "nexus-db-queries", @@ -5929,12 +5919,12 @@ dependencies = [ "cookie", "derive-where", "derive_more", - "dns-service-client", - "dropshot 0.12.0", + "dropshot", "futures", "gateway-client", "http 1.1.0", "humantime", + "internal-dns-types", "ipnetwork", "newtype-uuid", "newtype_derive", @@ -6270,7 +6260,7 @@ dependencies = [ "clickhouse-admin-api", "clickhouse-admin-types", "clickward", - "dropshot 0.12.0", + "dropshot", "expectorate", "http 1.1.0", "illumos-utils", @@ -6309,7 +6299,7 @@ dependencies = [ "cockroach-admin-api", "cockroach-admin-types", "csv", - "dropshot 0.12.0", + "dropshot", "expectorate", "http 1.1.0", "illumos-utils", @@ -6351,7 +6341,7 @@ dependencies = [ "camino", "camino-tempfile", "chrono", - "dropshot 0.12.0", + "dropshot", "expectorate", "futures", "hex", @@ -6369,7 +6359,7 @@ dependencies = [ "proptest", "rand", "regress 0.9.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "semver 1.0.23", "serde", @@ -6396,7 +6386,7 @@ dependencies = [ "omicron-common", "omicron-workspace-hack", "progenitor-client", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "sled-hardware-types", "slog", @@ -6411,7 +6401,7 @@ version = "0.1.0" dependencies = [ "anyhow", "clap", - "dropshot 0.12.0", + "dropshot", "expectorate", "futures", "libc", @@ -6451,7 +6441,7 @@ dependencies = [ "camino", "chrono", "clap", - "dropshot 0.12.0", + "dropshot", "expectorate", "futures", "gateway-api", @@ -6495,9 +6485,10 @@ version = "0.1.0" dependencies = [ "anyhow", "assert_matches", - "dropshot 0.12.0", + "dropshot", "futures", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "live-tests-macros", "nexus-client", "nexus-config", @@ -6513,7 +6504,7 @@ dependencies = [ "omicron-test-utils", "omicron-workspace-hack", "pq-sys", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "slog", "slog-error-chain", @@ -6530,11 +6521,14 @@ dependencies = [ "camino", "cargo_metadata", "clap", + "expectorate", "newtype_derive", + "omicron-test-utils", "omicron-workspace-hack", "parse-display", "petgraph", "serde", + "subprocess", "toml 0.8.19", ] @@ -6554,6 +6548,8 @@ dependencies = [ "cancel-safe-futures", "chrono", "clap", + "clickhouse-admin-keeper-client", + "clickhouse-admin-server-client", "cockroach-admin-client", "criterion", "crucible-agent-client", @@ -6564,7 +6560,7 @@ dependencies = [ "dns-server", "dns-service-client", "dpd-client", - "dropshot 0.12.0", + "dropshot", "expectorate", "fatfs", "futures", @@ -6581,7 +6577,8 @@ dependencies = [ "hyper 1.4.1", "hyper-rustls 0.26.0", "illumos-utils", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "ipnetwork", "itertools 0.13.0", "macaddr", @@ -6635,14 +6632,15 @@ dependencies = [ "pretty_assertions", "progenitor-client", "propolis-client 0.1.0 (git+https://github.com/oxidecomputer/propolis?rev=11371b0f3743f8df5b047dc0edc2699f4bdf3927)", + "qorb", "rand", "rcgen", "ref-cast", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "ring 0.17.8", "rustls 0.22.4", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "samael", "schemars", "semver 1.0.23", @@ -6686,11 +6684,11 @@ dependencies = [ "camino-tempfile", "chrono", "clap", - "crossterm 0.28.1", + "crossterm", "crucible-agent-client", "csv", "diesel", - "dropshot 0.12.0", + "dropshot", "dyn-clone", "expectorate", "futures", @@ -6700,7 +6698,8 @@ dependencies = [ "http 1.1.0", "humantime", "indicatif", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "ipnetwork", "itertools 0.13.0", "multimap", @@ -6758,7 +6757,7 @@ dependencies = [ "omicron-zone-package", "petgraph", "rayon", - "reqwest 0.12.7", + "reqwest 0.12.8", "ring 0.17.8", "semver 1.0.23", "serde", @@ -6810,7 +6809,7 @@ dependencies = [ "omicron-workspace-hack", "omicron-zone-package", "once_cell", - "reqwest 0.12.7", + "reqwest 0.12.8", "semver 1.0.23", "serde", "sha2", @@ -6855,7 +6854,7 @@ dependencies = [ "dns-server", "dns-service-client", "dpd-client", - "dropshot 0.12.0", + "dropshot", "expectorate", "flate2", "flume", @@ -6869,7 +6868,8 @@ dependencies = [ "hyper-staticfile", "illumos-utils", "installinator-common", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "ipnetwork", "itertools 0.13.0", "key-manager", @@ -6897,7 +6897,7 @@ dependencies = [ "propolis_api_types", "rand", "rcgen", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "semver 1.0.23", "serde", @@ -6944,7 +6944,7 @@ dependencies = [ "camino", "camino-tempfile", "chrono", - "dropshot 0.12.0", + "dropshot", "expectorate", "filetime", "futures", @@ -6960,7 +6960,7 @@ dependencies = [ "pem", "rcgen", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "ring 0.17.8", "rustls 0.22.4", "slog", @@ -7040,7 +7040,7 @@ dependencies = [ "hyper 1.4.1", "hyper-rustls 0.27.3", "hyper-util", - "indexmap 2.5.0", + "indexmap 2.6.0", "indicatif", "inout", "itertools 0.10.5", @@ -7071,8 +7071,8 @@ dependencies = [ "quote", "regex", "regex-automata", - "regex-syntax 0.8.4", - "reqwest 0.12.7", + "regex-syntax 0.8.5", + "reqwest 0.12.8", "ring 0.17.8", "rsa", "rustix", @@ -7133,7 +7133,7 @@ dependencies = [ "futures", "futures-util", "hex", - "reqwest 0.12.7", + "reqwest 0.12.8", "ring 0.16.20", "semver 1.0.23", "serde", @@ -7150,9 +7150,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.19.0" +version = "1.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" [[package]] name = "oorandom" @@ -7172,7 +7172,7 @@ version = "0.4.0" source = "git+https://github.com/oxidecomputer/openapi-lint?branch=main#ef442ee4343e97b6d9c217d3e7533962fe7d7236" dependencies = [ "heck 0.4.1", - "indexmap 2.5.0", + "indexmap 2.6.0", "lazy_static", "openapiv3", "regex", @@ -7190,7 +7190,7 @@ dependencies = [ "clickhouse-admin-api", "cockroach-admin-api", "dns-server-api", - "dropshot 0.12.0", + "dropshot", "fs-err", "gateway-api", "indent_write", @@ -7225,7 +7225,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc02deea53ffe807708244e5914f6b099ad7015a207ee24317c22112e17d9c5c" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_json", ] @@ -7277,7 +7277,7 @@ dependencies = [ [[package]] name = "opte" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=f3002b356da7d0e4ca15beb66a5566a92919baaa#f3002b356da7d0e4ca15beb66a5566a92919baaa" dependencies = [ "cfg-if", "dyn-clone", @@ -7294,7 +7294,7 @@ dependencies = [ [[package]] name = "opte-api" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=f3002b356da7d0e4ca15beb66a5566a92919baaa#f3002b356da7d0e4ca15beb66a5566a92919baaa" dependencies = [ "illumos-sys-hdrs", "ipnetwork", @@ -7306,7 +7306,7 @@ dependencies = [ [[package]] name = "opte-ioctl" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=f3002b356da7d0e4ca15beb66a5566a92919baaa#f3002b356da7d0e4ca15beb66a5566a92919baaa" dependencies = [ "libc", "libnet 0.1.0 (git+https://github.com/oxidecomputer/netadm-sys)", @@ -7363,7 +7363,7 @@ dependencies = [ "progenitor", "rand", "regress 0.9.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "thiserror", @@ -7374,7 +7374,7 @@ dependencies = [ [[package]] name = "oxide-vpc" version = "0.1.0" -source = "git+https://github.com/oxidecomputer/opte?rev=76878de67229ea113d70503c441eab47ac5dc653#76878de67229ea113d70503c441eab47ac5dc653" +source = "git+https://github.com/oxidecomputer/opte?rev=f3002b356da7d0e4ca15beb66a5566a92919baaa#f3002b356da7d0e4ca15beb66a5566a92919baaa" dependencies = [ "cfg-if", "illumos-sys-hdrs", @@ -7383,6 +7383,7 @@ dependencies = [ "serde", "smoltcp 0.11.0", "tabwriter", + "uuid", "zerocopy 0.7.34", ] @@ -7409,7 +7410,7 @@ name = "oximeter-api" version = "0.1.0" dependencies = [ "chrono", - "dropshot 0.12.0", + "dropshot", "omicron-common", "omicron-workspace-hack", "schemars", @@ -7426,7 +7427,7 @@ dependencies = [ "omicron-common", "omicron-workspace-hack", "progenitor", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "slog", "uuid", @@ -7441,11 +7442,12 @@ dependencies = [ "camino", "chrono", "clap", - "dropshot 0.12.0", + "dropshot", "expectorate", "futures", "httpmock", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "nexus-client", "nexus-types", "omicron-common", @@ -7459,7 +7461,7 @@ dependencies = [ "oximeter-db", "qorb", "rand", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -7490,15 +7492,15 @@ dependencies = [ "clap", "clickward", "criterion", - "crossterm 0.28.1", + "crossterm", "debug-ignore", "display-error-chain", - "dropshot 0.12.0", + "dropshot", "expectorate", "futures", "gethostname", "highway", - "indexmap 2.5.0", + "indexmap 2.6.0", "itertools 0.13.0", "libc", "num", @@ -7512,7 +7514,7 @@ dependencies = [ "qorb", "reedline", "regex", - "reqwest 0.12.7", + "reqwest 0.12.8", "rustyline", "schemars", "serde", @@ -7539,7 +7541,7 @@ version = "0.1.0" dependencies = [ "cfg-if", "chrono", - "dropshot 0.12.0", + "dropshot", "futures", "http 1.1.0", "hyper 1.4.1", @@ -7575,8 +7577,9 @@ dependencies = [ "anyhow", "chrono", "clap", - "dropshot 0.12.0", - "internal-dns", + "dropshot", + "internal-dns-resolver", + "internal-dns-types", "nexus-client", "omicron-common", "omicron-test-utils", @@ -7837,7 +7840,7 @@ checksum = "287d8d3ebdce117b8539f59411e4ed9ec226e0a4153c7f55495c6070d68e6f72" dependencies = [ "parse-display-derive", "regex", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -7849,7 +7852,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "structmeta 0.3.0", "syn 2.0.79", ] @@ -8050,7 +8053,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", ] @@ -8421,11 +8424,11 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d37c51ca738a55da99dc0c4a34860fd675453b8b36209178c2249bb13651284" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" dependencies = [ - "toml_edit 0.21.1", + "toml_edit 0.22.22", ] [[package]] @@ -8454,9 +8457,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.86" +version = "1.0.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +checksum = "b3e4daa0dcf6feba26f985457cdf104d4b4256fc5a09547140f3631bb076b19a" dependencies = [ "unicode-ident", ] @@ -8481,7 +8484,7 @@ dependencies = [ "bytes", "futures-core", "percent-encoding", - "reqwest 0.12.7", + "reqwest 0.12.8", "serde", "serde_json", "serde_urlencoded", @@ -8495,7 +8498,7 @@ checksum = "d85934a440963a69f9f04f48507ff6e7aa2952a5b2d8f96cc37fa3dd5c270f66" dependencies = [ "heck 0.5.0", "http 1.1.0", - "indexmap 2.5.0", + "indexmap 2.6.0", "openapiv3", "proc-macro2", "quote", @@ -8537,7 +8540,7 @@ dependencies = [ "futures", "progenitor", "rand", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -8558,7 +8561,7 @@ dependencies = [ "futures", "progenitor", "rand", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -8578,13 +8581,13 @@ dependencies = [ "atty", "base64 0.21.7", "clap", - "dropshot 0.12.0", + "dropshot", "futures", "hyper 1.4.1", "progenitor", "propolis_types", "rand", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -8647,7 +8650,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xorshift", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", "rusty-fork", "tempfile", "unarray", @@ -8671,14 +8674,14 @@ dependencies = [ [[package]] name = "qorb" -version = "0.0.1" -source = "git+https://github.com/oxidecomputer/qorb?branch=master#163a77838a3cfe8f7741d32e443f76d995b89df3" +version = "0.0.2" +source = "git+https://github.com/oxidecomputer/qorb?branch=master#de6f7784790c813931042dcc98c84413ecf11826" dependencies = [ "anyhow", "async-trait", "debug-ignore", "derive-where", - "dropshot 0.10.1", + "dropshot", "futures", "hickory-resolver", "rand", @@ -8852,14 +8855,14 @@ dependencies = [ [[package]] name = "ratatui" -version = "0.28.0" +version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ba6a365afbe5615999275bea2446b970b10a41102500e27ce7678d50d978303" +checksum = "fdef7f9be5c0122f890d58bdf4d964349ba6a6161f705907526d891efabba57d" dependencies = [ "bitflags 2.6.0", "cassowary", "compact_str", - "crossterm 0.28.1", + "crossterm", "instability", "itertools 0.13.0", "lru", @@ -8911,16 +8914,16 @@ dependencies = [ "assert_matches", "camino", "camino-tempfile", + "chrono", "clap", - "dns-service-client", - "dropshot 0.12.0", + "dropshot", "expectorate", "humantime", - "indexmap 2.5.0", + "indexmap 2.6.0", + "internal-dns-types", "nexus-client", "nexus-db-queries", "nexus-inventory", - "nexus-reconfigurator-execution", "nexus-reconfigurator-planning", "nexus-reconfigurator-preparation", "nexus-sled-agent-shared", @@ -8943,6 +8946,7 @@ dependencies = [ "swrite", "tabled", "tokio", + "typed-rng", "uuid", ] @@ -8977,12 +8981,12 @@ dependencies = [ [[package]] name = "reedline" -version = "0.33.0" +version = "0.35.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f8c676a3f3814a23c6a0fc9dff6b6c35b2e04df8134aae6f3929cc34de21a53" +checksum = "c5289de810296f8f2ff58d35544d92ae98d0a631453388bc3e608086be0fa596" dependencies = [ "chrono", - "crossterm 0.27.0", + "crossterm", "fd-lock", "itertools 0.12.1", "nu-ansi-term", @@ -9017,25 +9021,25 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.6" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +checksum = "38200e5ee88914975b69f657f0801b6f6dccafd44fd9326302a4aaeecfacb1d8" dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "368758f23274712b504848e9d5a6f010445cc8b87a7cdb4d7cbee666c1288da3" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.4", + "regex-syntax 0.8.5", ] [[package]] @@ -9046,9 +9050,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" [[package]] name = "regress" @@ -9116,9 +9120,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.7" +version = "0.12.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8f4955649ef5c38cc7f9e8aa41761d48fb9677197daea9984dc54f56aad5e63" +checksum = "f713147fbe92361e52392c73b8c9e48c04c6625bce969ef54dc901e58e042a7b" dependencies = [ "base64 0.22.1", "bytes", @@ -9146,7 +9150,7 @@ dependencies = [ "pin-project-lite", "quinn", "rustls 0.23.10", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -9275,30 +9279,30 @@ dependencies = [ [[package]] name = "rstest" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b423f0e62bdd61734b67cd21ff50871dfaeb9cc74f869dcd6af974fbcb19936" +checksum = "0a2c585be59b6b5dd66a9d2084aa1d8bd52fbdb806eafdeffb52791147862035" dependencies = [ "futures", "futures-timer", "rstest_macros", - "rustc_version 0.4.0", + "rustc_version 0.4.1", ] [[package]] name = "rstest_macros" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e1711e7d14f74b12a58411c542185ef7fb7f2e7f8ee6e2940a883628522b42" +checksum = "825ea780781b15345a146be27eaefb05085e337e869bff01b4306a4fd4a9ad5a" dependencies = [ "cfg-if", "glob", - "proc-macro-crate 3.1.0", + "proc-macro-crate 3.2.0", "proc-macro2", "quote", "regex", "relative-path", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "syn 2.0.79", "unicode-ident", ] @@ -9471,9 +9475,9 @@ dependencies = [ [[package]] name = "rustc_version" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ "semver 1.0.23", ] @@ -9493,9 +9497,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "8acb788b847c24f28525660c4d7758620a7210875711f79e7f663cc152726811" dependencies = [ "bitflags 2.6.0", "errno", @@ -9552,7 +9556,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1fb85efa936c42c6d5fc28d2629bb51e4b2f4b8a5211e297d599cc5a093792" dependencies = [ "openssl-probe", - "rustls-pemfile 2.1.3", + "rustls-pemfile 2.2.0", "rustls-pki-types", "schannel", "security-framework", @@ -9569,19 +9573,18 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "2.1.3" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64 0.22.1", "rustls-pki-types", ] [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "0e696e35370c65c9c541198af4543ccd580cf17fc25d8e05c5a242b202488c55" [[package]] name = "rustls-webpki" @@ -10023,15 +10026,15 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cecfa94848272156ea67b2b1a53f20fc7bc638c4a46d2f8abde08f05f4b857" +checksum = "8e28bdad6db2b8340e449f7108f020b3b092e8583a9e3fb82713e1d4e71fe817" dependencies = [ "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_derive", "serde_json", @@ -10041,9 +10044,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.9.0" +version = "3.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8fee4991ef4f274617a51ad4af30519438dacb2f56ac773b08a1922ff743350" +checksum = "9d846214a9854ef724f3da161b426242d8de7c1fc7de2f89bb1efcb154dca79d" dependencies = [ "darling", "proc-macro2", @@ -10057,7 +10060,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -10182,9 +10185,9 @@ dependencies = [ [[package]] name = "similar-asserts" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e041bb827d1bfca18f213411d51b665309f1afb37a04a5d1464530e13779fc0f" +checksum = "cfe85670573cd6f0fa97940f26e7e6601213c3b0555246c24234131f88c5709e" dependencies = [ "console", "similar", @@ -10226,7 +10229,7 @@ name = "sled-agent-api" version = "0.1.0" dependencies = [ "camino", - "dropshot 0.12.0", + "dropshot", "nexus-sled-agent-shared", "omicron-common", "omicron-uuid-kinds", @@ -10252,7 +10255,7 @@ dependencies = [ "oxnet", "progenitor", "regress 0.9.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -10597,7 +10600,7 @@ dependencies = [ "anyhow", "async-trait", "clap", - "dropshot 0.12.0", + "dropshot", "futures", "gateway-messages", "gateway-types", @@ -11097,9 +11100,9 @@ checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" [[package]] name = "target-spec" -version = "3.2.1" +version = "3.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419ccf3482090c626619fa2574290aaa00b696f9ab73af08fbf48260565431bf" +checksum = "4c5743abbf7bc7d5296ae61368b50cd218ac09432281cb5d48b97308d5c27909" dependencies = [ "cfg-expr", "guppy-workspace-hack", @@ -11109,14 +11112,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.10.1" +version = "3.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" +checksum = "f0f2c9fc62d0beef6951ccffd757e241266a2c833136efbe35af6cd2567dca5b" dependencies = [ "cfg-if", "fastrand", + "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11141,12 +11145,12 @@ dependencies = [ [[package]] name = "terminal_size" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21bebf2b7c9e0a515f6e0f8c51dc0f8e4696391e6f1ff30379559f8365fb0df7" +checksum = "4f599bd7ca042cfdf8f4512b277c02ba102247820f9d9d4a9f521f496751a6ef" dependencies = [ "rustix", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -11387,9 +11391,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.39.3" +version = "1.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babc99b9923bfa4804bd74722ff02c0381021eafa4db9949217e3be8e84fff5" +checksum = "e2b070231665d27ad9ec9b8df639893f46727666c6767db40317fbe920a5d998" dependencies = [ "backtrace", "bytes", @@ -11570,31 +11574,20 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", "winnow 0.5.40", ] -[[package]] -name = "toml_edit" -version = "0.21.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1" -dependencies = [ - "indexmap 2.5.0", - "toml_datetime", - "winnow 0.5.40", -] - [[package]] name = "toml_edit" version = "0.22.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" dependencies = [ - "indexmap 2.5.0", + "indexmap 2.6.0", "serde", "serde_spanned", "toml_datetime", @@ -11724,9 +11717,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "trybuild" -version = "1.0.99" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "207aa50d36c4be8d8c6ea829478be44a372c6a77669937bb39c698e52f1491e8" +checksum = "8923cde76a6329058a86f04d033f0945a2c6df8b94093512e4ab188b3e3a8950" dependencies = [ "glob", "serde", @@ -12050,7 +12043,7 @@ dependencies = [ "clap", "debug-ignore", "display-error-chain", - "dropshot 0.12.0", + "dropshot", "futures", "hex", "hubtools", @@ -12085,7 +12078,7 @@ dependencies = [ "either", "futures", "indent_write", - "indexmap 2.5.0", + "indexmap 2.6.0", "indicatif", "indoc 2.0.5", "libsw", @@ -12237,7 +12230,7 @@ dependencies = [ "cfg-if", "git2", "regex", - "rustc_version 0.4.0", + "rustc_version 0.4.1", "rustversion", "time", ] @@ -12466,12 +12459,12 @@ dependencies = [ "camino", "ciborium", "clap", - "crossterm 0.28.1", + "crossterm", "expectorate", "futures", "hex", "humantime", - "indexmap 2.5.0", + "indexmap 2.6.0", "indicatif", "itertools 0.13.0", "maplit", @@ -12482,7 +12475,7 @@ dependencies = [ "owo-colors", "proptest", "ratatui", - "reqwest 0.12.7", + "reqwest 0.12.8", "rpassword", "serde", "serde_json", @@ -12513,7 +12506,7 @@ version = "0.1.0" dependencies = [ "anyhow", "dpd-client", - "dropshot 0.12.0", + "dropshot", "gateway-client", "maplit", "omicron-common", @@ -12541,7 +12534,7 @@ dependencies = [ "camino", "ciborium", "clap", - "crossterm 0.28.1", + "crossterm", "omicron-workspace-hack", "reedline", "serde", @@ -12569,7 +12562,7 @@ dependencies = [ "debug-ignore", "display-error-chain", "dpd-client", - "dropshot 0.12.0", + "dropshot", "either", "expectorate", "flate2", @@ -12590,7 +12583,8 @@ dependencies = [ "installinator-api", "installinator-client", "installinator-common", - "internal-dns", + "internal-dns-resolver", + "internal-dns-types", "itertools 0.13.0", "maplit", "omicron-certificates", @@ -12605,7 +12599,7 @@ dependencies = [ "openapiv3", "oxnet", "rand", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -12637,7 +12631,7 @@ name = "wicketd-api" version = "0.1.0" dependencies = [ "bootstrap-agent-client", - "dropshot 0.12.0", + "dropshot", "gateway-client", "omicron-common", "omicron-passwords", @@ -12661,7 +12655,7 @@ dependencies = [ "omicron-workspace-hack", "progenitor", "regress 0.9.1", - "reqwest 0.12.7", + "reqwest 0.12.8", "schemars", "serde", "serde_json", @@ -12997,7 +12991,7 @@ dependencies = [ "flate2", "futures", "omicron-workspace-hack", - "reqwest 0.12.7", + "reqwest 0.12.8", "sha2", "slog", "slog-async", @@ -13136,7 +13130,7 @@ dependencies = [ "anyhow", "camino", "clap", - "dropshot 0.12.0", + "dropshot", "illumos-utils", "omicron-common", "omicron-sled-agent", diff --git a/Cargo.toml b/Cargo.toml index 5411e233b2..afa970aa36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,8 @@ members = [ "clickhouse-admin", "clickhouse-admin/api", "clients/bootstrap-agent-client", + "clients/clickhouse-admin-keeper-client", + "clients/clickhouse-admin-server-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", "clients/dns-service-client", @@ -49,8 +51,9 @@ members = [ "installinator-api", "installinator-common", "installinator", - "internal-dns-cli", - "internal-dns", + "internal-dns/cli", + "internal-dns/resolver", + "internal-dns/types", "ipcc", "key-manager", "live-tests", @@ -126,6 +129,8 @@ default-members = [ "clickhouse-admin/api", "clickhouse-admin/types", "clients/bootstrap-agent-client", + "clients/clickhouse-admin-keeper-client", + "clients/clickhouse-admin-server-client", "clients/cockroach-admin-client", "clients/ddm-admin-client", "clients/dns-service-client", @@ -171,8 +176,9 @@ default-members = [ "installinator-api", "installinator-common", "installinator", - "internal-dns-cli", - "internal-dns", + "internal-dns/cli", + "internal-dns/resolver", + "internal-dns/types", "ipcc", "key-manager", "live-tests", @@ -271,11 +277,6 @@ redundant_field_names = "warn" # idiomatically declaring a static array of atomics uses `const Atomic`). We # warn on this to catch the former, and expect any uses of the latter to allow # this locally. -# -# Note: any const value with a type containing a `bytes::Bytes` hits this lint, -# and you should `#![allow]` it for now. This is most likely to be seen with -# `http::header::{HeaderName, HeaderValue}`. This is a Clippy bug which will be -# fixed in the Rust 1.80 toolchain (rust-lang/rust-clippy#12691). declare_interior_mutable_const = "warn" # Also warn on casts, preferring explicit conversions instead. # @@ -314,6 +315,8 @@ chrono = { version = "0.4", features = [ "serde" ] } ciborium = "0.2.2" clap = { version = "4.5", features = ["cargo", "derive", "env", "wrap_help"] } clickhouse-admin-api = { path = "clickhouse-admin/api" } +clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-client" } +clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" } clickhouse-admin-types = { path = "clickhouse-admin/types" } clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "ceec762e6a87d2a22bf56792a3025e145caa095e" } cockroach-admin-api = { path = "cockroach-admin/api" } @@ -356,7 +359,7 @@ float-ord = "0.3.2" flume = "0.11.0" foreign-types = "0.3.2" fs-err = "2.11.0" -futures = "0.3.30" +futures = "0.3.31" gateway-api = { path = "gateway-api" } gateway-client = { path = "clients/gateway-client" } # If you're updating the pinned revision of these MGS dependencies, you should @@ -373,7 +376,7 @@ gateway-test-utils = { path = "gateway-test-utils" } gateway-types = { path = "gateway-types" } gethostname = "0.5.0" glob = "0.3.1" -guppy = "0.17.7" +guppy = "0.17.8" headers = "0.4.0" heck = "0.5" hex = "0.4.3" @@ -396,14 +399,15 @@ hyper-rustls = "0.26.0" hyper-staticfile = "0.10.0" illumos-utils = { path = "illumos-utils" } indent_write = "2.2.0" -indexmap = "2.5.0" +indexmap = "2.6.0" indicatif = { version = "0.17.8", features = ["rayon"] } indoc = "2.0.5" installinator = { path = "installinator" } installinator-api = { path = "installinator-api" } installinator-client = { path = "clients/installinator-client" } installinator-common = { path = "installinator-common" } -internal-dns = { path = "internal-dns" } +internal-dns-resolver = { path = "internal-dns/resolver" } +internal-dns-types = { path = "internal-dns/types" } ipcc = { path = "ipcc" } ipnet = "2.9" itertools = "0.13.0" @@ -422,10 +426,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] } maplit = "1.0.2" mockall = "0.13" newtype_derive = "0.1.6" -# mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "9e0fe45ca3862176dc31ad8cc83f605f8a7e1a42" } -# ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "9e0fe45ca3862176dc31ad8cc83f605f8a7e1a42" } -mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", branch = "hyper-v1-no-merge" } -ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", branch = "hyper-v1-no-merge" } +mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "056283eb02b6887fbf27f66a215662520f7c159c" } +ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "056283eb02b6887fbf27f66a215662520f7c159c" } multimap = "0.10.0" nexus-auth = { path = "nexus/auth" } nexus-client = { path = "clients/nexus-client" } @@ -468,17 +470,17 @@ omicron-test-utils = { path = "test-utils" } omicron-workspace-hack = "0.1.0" omicron-zone-package = "0.11.1" oxide-client = { path = "clients/oxide-client" } -oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "76878de67229ea113d70503c441eab47ac5dc653", features = [ "api", "std" ] } +oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "f3002b356da7d0e4ca15beb66a5566a92919baaa", features = [ "api", "std" ] } oxlog = { path = "dev-tools/oxlog" } oxnet = { git = "https://github.com/oxidecomputer/oxnet" } -once_cell = "1.19.0" +once_cell = "1.20.2" openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" } openapi-manager-types = { path = "dev-tools/openapi-manager/types" } openapiv3 = "2.0.0" # must match samael's crate! openssl = "0.10" openssl-sys = "0.9" -opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "76878de67229ea113d70503c441eab47ac5dc653" } +opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "f3002b356da7d0e4ca15beb66a5566a92919baaa" } oso = "0.27" owo-colors = "4.1.0" oximeter = { path = "oximeter/oximeter" } @@ -522,20 +524,20 @@ rand = "0.8.5" rand_core = "0.6.4" rand_distr = "0.4.3" rand_seeder = "0.3.0" -ratatui = "0.28.0" +ratatui = "0.28.1" rayon = "1.10" rcgen = "0.12.1" -reedline = "0.33.0" +reedline = "0.35.0" ref-cast = "1.0" -regex = "1.10.6" +regex = "1.11.0" regress = "0.9.1" reqwest = { version = "0.12", default-features = false } ring = "0.17.8" rpassword = "7.3.1" -rstest = "0.22.0" +rstest = "0.23.0" rustfmt-wrapper = "0.2" rustls = "0.22.2" -rustls-pemfile = "2.1.3" +rustls-pemfile = "2.2.0" rustyline = "14.0.0" samael = { version = "0.0.17", features = ["xmlsec"] } schemars = "0.8.21" @@ -547,7 +549,7 @@ serde_json = "1.0.128" serde_path_to_error = "0.1.16" serde_tokenstream = "0.2" serde_urlencoded = "0.7.1" -serde_with = "3.9.0" +serde_with = "3.11.0" sha2 = "0.10.8" sha3 = "0.10.8" shell-words = "1.1.0" @@ -555,7 +557,7 @@ signal-hook = "0.3" signal-hook-tokio = { version = "0.3", features = [ "futures-v0_3" ] } sigpipe = "0.1.3" similar = { version = "2.6.0", features = ["bytes"] } -similar-asserts = "1.5.0" +similar-asserts = "1.6.0" # Don't change sled's version on accident; sled's on-disk format is not yet # stable and requires manual migrations. In the limit this won't matter because # the upgrade system will replace the DNS server zones entirely, but while we @@ -601,7 +603,7 @@ textwrap = "0.16.1" test-strategy = "0.3.1" thiserror = "1.0" tofino = { git = "https://github.com/oxidecomputer/tofino", branch = "main" } -tokio = "1.39.3" +tokio = "1.40.0" tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] } tokio-stream = "0.1.16" tokio-tungstenite = "0.23.1" @@ -609,7 +611,7 @@ tokio-util = { version = "0.7.12", features = ["io", "io-util"] } toml = "0.8.19" toml_edit = "0.22.22" tough = { version = "0.17.1", features = [ "http" ] } -trybuild = "1.0.99" +trybuild = "1.0.100" tufaceous = { path = "tufaceous" } tufaceous-lib = { path = "tufaceous-lib" } tui-tree-widget = "0.22.0" @@ -636,7 +638,7 @@ zone = { version = "0.3", default-features = false, features = ["async"] } # the kinds). However, uses of omicron-uuid-kinds _within omicron_ will have # std and the other features enabled because they'll refer to it via # omicron-uuid-kinds.workspace = true. -newtype-uuid = { version = "1.1.0", default-features = false } +newtype-uuid = { version = "1.1.2", default-features = false } omicron-uuid-kinds = { path = "uuid-kinds", features = ["serde", "schemars08", "uuid-v4"] } # NOTE: The test profile inherits from the dev profile, so settings under diff --git a/clickhouse-admin/api/src/lib.rs b/clickhouse-admin/api/src/lib.rs index 7e66c9ca06..19cd2b3e8e 100644 --- a/clickhouse-admin/api/src/lib.rs +++ b/clickhouse-admin/api/src/lib.rs @@ -2,55 +2,40 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; use clickhouse_admin_types::{ - KeeperConf, KeeperSettings, Lgif, RaftConfig, ServerSettings, + ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig, + KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, + ServerConfigurableSettings, }; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, }; -use omicron_common::api::external::Generation; -use schemars::JsonSchema; -use serde::Deserialize; - -#[derive(Debug, Deserialize, JsonSchema)] -pub struct ServerConfigurableSettings { - /// A unique identifier for the configuration generation. - pub generation: Generation, - /// Configurable settings for a ClickHouse replica server node. - pub settings: ServerSettings, -} - -#[derive(Debug, Deserialize, JsonSchema)] -pub struct KeeperConfigurableSettings { - /// A unique identifier for the configuration generation. - pub generation: Generation, - /// Configurable settings for a ClickHouse keeper node. - pub settings: KeeperSettings, -} +/// API interface for our clickhouse-admin-keeper server +/// +/// We separate the admin interface for the keeper and server APIs because they +/// are completely disjoint. We only run a clickhouse keeper *or* clickhouse +/// server in a given zone, and therefore each admin api is only useful in one +/// of the zones. Using separate APIs and clients prevents us from having to +/// mark a given endpoint `unimplemented` in the case of it not being usable +/// with one of the zone types. +/// +/// Nonetheless, the interfaces themselves are small and serve a similar +/// purpose. Therfore we combine them into the same crate. #[dropshot::api_description] -pub trait ClickhouseAdminApi { +pub trait ClickhouseAdminKeeperApi { type Context; - /// Generate a ClickHouse configuration file for a server node on a specified - /// directory. - #[endpoint { - method = PUT, - path = "/server/config", - }] - async fn generate_server_config( - rqctx: RequestContext, - body: TypedBody, - ) -> Result, HttpError>; - /// Generate a ClickHouse configuration file for a keeper node on a specified - /// directory. + /// directory and enable the SMF service if not currently enabled. + /// + /// Note that we cannot start the keeper service until there is an initial + /// configuration set via this endpoint. #[endpoint { method = PUT, - path = "/keeper/config", + path = "/config", }] - async fn generate_keeper_config( + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError>; @@ -84,4 +69,40 @@ pub trait ClickhouseAdminApi { async fn keeper_conf( rqctx: RequestContext, ) -> Result, HttpError>; + + /// Retrieve cluster membership information from a keeper node. + #[endpoint { + method = GET, + path = "/keeper/cluster-membership", + }] + async fn keeper_cluster_membership( + rqctx: RequestContext, + ) -> Result, HttpError>; +} + +/// API interface for our clickhouse-admin-server server +/// +/// We separate the admin interface for the keeper and server APIs because they +/// are completely disjoint. We only run a clickhouse keeper *or* clickhouse +/// server in a given zone, and therefore each admin api is only useful in one +/// of the zones. Using separate APIs and clients prevents us from having to +/// mark a given endpoint `unimplemented` in the case of it not being usable +/// with one of the zone types. +/// +/// Nonetheless, the interfaces themselves are small and serve a similar +/// purpose. Therfore we combine them into the same crate. +#[dropshot::api_description] +pub trait ClickhouseAdminServerApi { + type Context; + + /// Generate a ClickHouse configuration file for a server node on a specified + /// directory and enable the SMF service. + #[endpoint { + method = PUT, + path = "/config" + }] + async fn generate_config( + rqctx: RequestContext, + body: TypedBody, + ) -> Result, HttpError>; } diff --git a/clickhouse-admin/src/bin/clickhouse-admin.rs b/clickhouse-admin/src/bin/clickhouse-admin-keeper.rs similarity index 90% rename from clickhouse-admin/src/bin/clickhouse-admin.rs rename to clickhouse-admin/src/bin/clickhouse-admin-keeper.rs index 3391a3459a..4ec998920b 100644 --- a/clickhouse-admin/src/bin/clickhouse-admin.rs +++ b/clickhouse-admin/src/bin/clickhouse-admin-keeper.rs @@ -2,7 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Executable program to run the Omicron ClickHouse admin interface +//! Executable program to run the Omicron ClickHouse admin interface for +//! clickhouse keepers. use anyhow::anyhow; use camino::Utf8PathBuf; @@ -14,8 +15,8 @@ use std::net::{SocketAddr, SocketAddrV6}; #[derive(Debug, Parser)] #[clap( - name = "clickhouse-admin", - about = "Omicron ClickHouse cluster admin server" + name = "clickhouse-admin-keeper", + about = "Omicron ClickHouse cluster admin server for keepers" )] enum Args { /// Start the ClickHouse admin server @@ -57,7 +58,7 @@ async fn main_impl() -> Result<(), CmdError> { let clickhouse_cli = ClickhouseCli::new(binary_path, listen_address); - let server = omicron_clickhouse_admin::start_server( + let server = omicron_clickhouse_admin::start_keeper_admin_server( clickward, clickhouse_cli, config, diff --git a/clickhouse-admin/src/bin/clickhouse-admin-server.rs b/clickhouse-admin/src/bin/clickhouse-admin-server.rs new file mode 100644 index 0000000000..1258f75805 --- /dev/null +++ b/clickhouse-admin/src/bin/clickhouse-admin-server.rs @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Executable program to run the Omicron ClickHouse admin interface for +//! clickhouse servers. + +use anyhow::anyhow; +use camino::Utf8PathBuf; +use clap::Parser; +use omicron_clickhouse_admin::{ClickhouseCli, Clickward, Config}; +use omicron_common::cmd::fatal; +use omicron_common::cmd::CmdError; +use std::net::{SocketAddr, SocketAddrV6}; + +#[derive(Debug, Parser)] +#[clap( + name = "clickhouse-admin-server", + about = "Omicron ClickHouse cluster admin server for replica servers" +)] +enum Args { + /// Start the ClickHouse admin server + Run { + /// Address on which this server should run + #[clap(long, short = 'a', action)] + http_address: SocketAddrV6, + + /// Path to the server configuration file + #[clap(long, short, action)] + config: Utf8PathBuf, + + /// Address on which the clickhouse server or keeper is listening on + #[clap(long, short = 'l', action)] + listen_address: SocketAddrV6, + + /// Path to the clickhouse binary + #[clap(long, short, action)] + binary_path: Utf8PathBuf, + }, +} + +#[tokio::main] +async fn main() { + if let Err(err) = main_impl().await { + fatal(err); + } +} + +async fn main_impl() -> Result<(), CmdError> { + let args = Args::parse(); + + match args { + Args::Run { http_address, config, listen_address, binary_path } => { + let mut config = Config::from_file(&config) + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + config.dropshot.bind_address = SocketAddr::V6(http_address); + let clickward = Clickward::new(); + let clickhouse_cli = + ClickhouseCli::new(binary_path, listen_address); + + let server = omicron_clickhouse_admin::start_server_admin_server( + clickward, + clickhouse_cli, + config, + ) + .await + .map_err(|err| CmdError::Failure(anyhow!(err)))?; + server.await.map_err(|err| { + CmdError::Failure(anyhow!( + "server failed after starting: {err}" + )) + }) + } + } +} diff --git a/clickhouse-admin/src/clickhouse_cli.rs b/clickhouse-admin/src/clickhouse_cli.rs index a84e2b3404..32afdc4ef8 100644 --- a/clickhouse-admin/src/clickhouse_cli.rs +++ b/clickhouse-admin/src/clickhouse_cli.rs @@ -4,11 +4,14 @@ use anyhow::Result; use camino::Utf8PathBuf; -use clickhouse_admin_types::{KeeperConf, Lgif, RaftConfig}; +use clickhouse_admin_types::{ + ClickhouseKeeperClusterMembership, KeeperConf, KeeperId, Lgif, RaftConfig, +}; use dropshot::HttpError; use illumos_utils::{output_to_exec_error, ExecutionError}; use slog::Logger; use slog_error_chain::{InlineErrorChain, SlogInlineError}; +use std::collections::BTreeSet; use std::ffi::OsStr; use std::io; use std::net::SocketAddrV6; @@ -102,6 +105,22 @@ impl ClickhouseCli { .await } + pub async fn keeper_cluster_membership( + &self, + ) -> Result { + let lgif_output = self.lgif().await?; + let conf_output = self.keeper_conf().await?; + let raft_output = self.raft_config().await?; + let raft_config: BTreeSet = + raft_output.keeper_servers.iter().map(|s| s.server_id).collect(); + + Ok(ClickhouseKeeperClusterMembership { + queried_keeper: conf_output.server_id, + leader_committed_log_index: lgif_output.leader_committed_log_idx, + raft_config, + }) + } + async fn keeper_client_non_interactive( &self, query: &str, diff --git a/clickhouse-admin/src/clickward.rs b/clickhouse-admin/src/clickward.rs index 5202b9b090..ca5d3df3de 100644 --- a/clickhouse-admin/src/clickward.rs +++ b/clickhouse-admin/src/clickward.rs @@ -2,8 +2,9 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; -use clickhouse_admin_types::{KeeperSettings, ServerSettings}; +use clickhouse_admin_types::{ + KeeperConfig, KeeperSettings, ReplicaConfig, ServerSettings, +}; use dropshot::HttpError; use slog_error_chain::{InlineErrorChain, SlogInlineError}; diff --git a/clickhouse-admin/src/http_entrypoints.rs b/clickhouse-admin/src/http_entrypoints.rs index b1571f26c7..49138b9cc3 100644 --- a/clickhouse-admin/src/http_entrypoints.rs +++ b/clickhouse-admin/src/http_entrypoints.rs @@ -4,47 +4,68 @@ use crate::context::ServerContext; use clickhouse_admin_api::*; -use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; -use clickhouse_admin_types::{KeeperConf, Lgif, RaftConfig}; +use clickhouse_admin_types::{ + ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfig, + KeeperConfigurableSettings, Lgif, RaftConfig, ReplicaConfig, + ServerConfigurableSettings, +}; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, RequestContext, TypedBody, }; +use illumos_utils::svcadm::Svcadm; use std::sync::Arc; type ClickhouseApiDescription = dropshot::ApiDescription>; -pub fn api() -> ClickhouseApiDescription { - clickhouse_admin_api_mod::api_description::() +pub fn clickhouse_admin_server_api() -> ClickhouseApiDescription { + clickhouse_admin_server_api_mod::api_description::() .expect("registered entrypoints") } -enum ClickhouseAdminImpl {} +pub fn clickhouse_admin_keeper_api() -> ClickhouseApiDescription { + clickhouse_admin_keeper_api_mod::api_description::() + .expect("registered entrypoints") +} -impl ClickhouseAdminApi for ClickhouseAdminImpl { +enum ClickhouseAdminServerImpl {} + +impl ClickhouseAdminServerApi for ClickhouseAdminServerImpl { type Context = Arc; - async fn generate_server_config( + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError> { let ctx = rqctx.context(); let replica_server = body.into_inner(); - // TODO(https://github.com/oxidecomputer/omicron/issues/5999): Do something - // with the generation number `replica_server.generation` let output = ctx.clickward().generate_server_config(replica_server.settings)?; + + // Once we have generated the client we can safely enable the clickhouse_server service + let fmri = "svc:/oxide/clickhouse_server:default".to_string(); + Svcadm::enable_service(fmri)?; + Ok(HttpResponseCreated(output)) } +} - async fn generate_keeper_config( +enum ClickhouseAdminKeeperImpl {} + +impl ClickhouseAdminKeeperApi for ClickhouseAdminKeeperImpl { + type Context = Arc; + + async fn generate_config( rqctx: RequestContext, body: TypedBody, ) -> Result, HttpError> { let ctx = rqctx.context(); let keeper = body.into_inner(); - // TODO(https://github.com/oxidecomputer/omicron/issues/5999): Do something - // with the generation number `keeper.generation` let output = ctx.clickward().generate_keeper_config(keeper.settings)?; + + // Once we have generated the client we can safely enable the clickhouse_keeper service + let fmri = "svc:/oxide/clickhouse_keeper:default".to_string(); + Svcadm::enable_service(fmri)?; + Ok(HttpResponseCreated(output)) } @@ -71,4 +92,13 @@ impl ClickhouseAdminApi for ClickhouseAdminImpl { let output = ctx.clickhouse_cli().keeper_conf().await?; Ok(HttpResponseOk(output)) } + + async fn keeper_cluster_membership( + rqctx: RequestContext, + ) -> Result, HttpError> + { + let ctx = rqctx.context(); + let output = ctx.clickhouse_cli().keeper_cluster_membership().await?; + Ok(HttpResponseOk(output)) + } } diff --git a/clickhouse-admin/src/lib.rs b/clickhouse-admin/src/lib.rs index 511a32dd50..1697d24adc 100644 --- a/clickhouse-admin/src/lib.rs +++ b/clickhouse-admin/src/lib.rs @@ -33,8 +33,9 @@ pub enum StartError { pub type Server = dropshot::HttpServer>; -/// Start the dropshot server -pub async fn start_server( +/// Start the dropshot server for `clickhouse-admin-server` which +/// manages clickhouse replica servers. +pub async fn start_server_admin_server( clickward: Clickward, clickhouse_cli: ClickhouseCli, server_config: Config, @@ -42,7 +43,7 @@ pub async fn start_server( let (drain, registration) = slog_dtrace::with_drain( server_config .log - .to_logger("clickhouse-admin") + .to_logger("clickhouse-admin-server") .map_err(StartError::InitializeLogger)?, ); let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv)); @@ -65,7 +66,49 @@ pub async fn start_server( ); let http_server_starter = dropshot::HttpServerStarter::new( &server_config.dropshot, - http_entrypoints::api(), + http_entrypoints::clickhouse_admin_server_api(), + Arc::new(context), + &log.new(slog::o!("component" => "dropshot")), + ) + .map_err(StartError::InitializeHttpServer)?; + + Ok(http_server_starter.start()) +} + +/// Start the dropshot server for `clickhouse-admin-server` which +/// manages clickhouse replica servers. +pub async fn start_keeper_admin_server( + clickward: Clickward, + clickhouse_cli: ClickhouseCli, + server_config: Config, +) -> Result { + let (drain, registration) = slog_dtrace::with_drain( + server_config + .log + .to_logger("clickhouse-admin-keeper") + .map_err(StartError::InitializeLogger)?, + ); + let log = slog::Logger::root(drain.fuse(), slog::o!(FileKv)); + match registration { + ProbeRegistration::Success => { + debug!(log, "registered DTrace probes"); + } + ProbeRegistration::Failed(err) => { + let err = StartError::RegisterDtraceProbes(err); + error!(log, "failed to register DTrace probes"; &err); + return Err(err); + } + } + + let context = ServerContext::new( + clickward, + clickhouse_cli + .with_log(log.new(slog::o!("component" => "ClickhouseCli"))), + log.new(slog::o!("component" => "ServerContext")), + ); + let http_server_starter = dropshot::HttpServerStarter::new( + &server_config.dropshot, + http_entrypoints::clickhouse_admin_keeper_api(), Arc::new(context), &log.new(slog::o!("component" => "dropshot")), ) diff --git a/clickhouse-admin/tests/integration_test.rs b/clickhouse-admin/tests/integration_test.rs index 79164b043f..eb26bec668 100644 --- a/clickhouse-admin/tests/integration_test.rs +++ b/clickhouse-admin/tests/integration_test.rs @@ -4,8 +4,10 @@ use anyhow::Context; use camino::Utf8PathBuf; -use clickhouse_admin_types::config::ClickhouseHost; -use clickhouse_admin_types::{KeeperServerInfo, KeeperServerType, RaftConfig}; +use clickhouse_admin_types::{ + ClickhouseHost, ClickhouseKeeperClusterMembership, KeeperId, + KeeperServerInfo, KeeperServerType, RaftConfig, +}; use clickward::{BasePorts, Deployment, DeploymentConfig}; use dropshot::test_util::log_prefix_for_test; use omicron_clickhouse_admin::ClickhouseCli; @@ -197,3 +199,83 @@ async fn test_keeper_conf_parsing() -> anyhow::Result<()> { logctx.cleanup_successful(); Ok(()) } + +#[tokio::test] +async fn test_keeper_cluster_membership() -> anyhow::Result<()> { + let logctx = test_setup_log("test_keeper_cluster_membership"); + let log = logctx.log.clone(); + + let (parent_dir, prefix) = log_prefix_for_test(logctx.test_name()); + let path = parent_dir.join(format!("{prefix}-oximeter-clickward-test")); + std::fs::create_dir(&path)?; + + // We spin up several replicated clusters and must use a + // separate set of ports in case the tests run concurrently. + let base_ports = BasePorts { + keeper: 30500, + raft: 30600, + clickhouse_tcp: 30700, + clickhouse_http: 30800, + clickhouse_interserver_http: 30900, + }; + + let config = DeploymentConfig { + path: path.clone(), + base_ports, + cluster_name: "oximeter_cluster".to_string(), + }; + + let mut deployment = Deployment::new(config); + + let num_keepers = 3; + let num_replicas = 1; + deployment + .generate_config(num_keepers, num_replicas) + .context("failed to generate config")?; + deployment.deploy().context("failed to deploy")?; + + wait_for_keepers( + &log, + &deployment, + (1..=num_keepers).map(clickward::KeeperId).collect(), + ) + .await?; + + let clickhouse_cli = ClickhouseCli::new( + Utf8PathBuf::from_str("clickhouse").unwrap(), + SocketAddrV6::new(Ipv6Addr::LOCALHOST, 30501, 0, 0), + ) + .with_log(log.clone()); + + let keeper_cluster_membership = + clickhouse_cli.keeper_cluster_membership().await.unwrap(); + + let mut raft_config = BTreeSet::new(); + + for i in 1..=num_keepers { + raft_config.insert(clickhouse_admin_types::KeeperId(i)); + } + + let expected_keeper_cluster_membership = + ClickhouseKeeperClusterMembership { + queried_keeper: KeeperId(1), + // This number is always different so we won't be testing it + leader_committed_log_index: 0, + raft_config, + }; + + assert_eq!( + keeper_cluster_membership.queried_keeper, + expected_keeper_cluster_membership.queried_keeper + ); + assert_eq!( + keeper_cluster_membership.raft_config, + expected_keeper_cluster_membership.raft_config + ); + + info!(&log, "Cleaning up test"); + deployment.teardown()?; + std::fs::remove_dir_all(path)?; + logctx.cleanup_successful(); + Ok(()) +} diff --git a/clickhouse-admin/types/src/config.rs b/clickhouse-admin/types/src/config.rs index fca74146f9..20aa3d71f8 100644 --- a/clickhouse-admin/types/src/config.rs +++ b/clickhouse-admin/types/src/config.rs @@ -2,7 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::{KeeperId, ServerId, OXIMETER_CLUSTER}; +use crate::{path_schema, KeeperId, ServerId, OXIMETER_CLUSTER}; use anyhow::{bail, Error}; use camino::Utf8PathBuf; use omicron_common::address::{ @@ -10,23 +10,11 @@ use omicron_common::address::{ CLICKHOUSE_KEEPER_RAFT_PORT, CLICKHOUSE_KEEPER_TCP_PORT, CLICKHOUSE_TCP_PORT, }; -use schemars::{ - gen::SchemaGenerator, - schema::{Schema, SchemaObject}, - JsonSchema, -}; +use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::net::{Ipv4Addr, Ipv6Addr}; use std::{fmt::Display, str::FromStr}; -// Used for schemars to be able to be used with camino: -// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 -pub fn path_schema(gen: &mut SchemaGenerator) -> Schema { - let mut schema: SchemaObject = ::json_schema(gen).into(); - schema.format = Some("Utf8PathBuf".to_owned()); - schema.into() -} - /// Configuration for a ClickHouse replica server #[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] pub struct ReplicaConfig { @@ -137,6 +125,13 @@ impl ReplicaConfig { + + system + query_log
+ Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 7 DAY + 10000 + + {temp_files_path} {user_files_path} default diff --git a/clickhouse-admin/types/src/lib.rs b/clickhouse-admin/types/src/lib.rs index a58f7b7cc4..021b9db356 100644 --- a/clickhouse-admin/types/src/lib.rs +++ b/clickhouse-admin/types/src/lib.rs @@ -7,7 +7,12 @@ use atomicwrites::AtomicFile; use camino::Utf8PathBuf; use derive_more::{Add, AddAssign, Display, From}; use itertools::Itertools; -use schemars::JsonSchema; +use omicron_common::api::external::Generation; +use schemars::{ + gen::SchemaGenerator, + schema::{Schema, SchemaObject}, + JsonSchema, +}; use serde::{Deserialize, Serialize}; use slog::{info, Logger}; use std::collections::BTreeSet; @@ -16,8 +21,20 @@ use std::io::{ErrorKind, Write}; use std::net::Ipv6Addr; use std::str::FromStr; -pub mod config; -use config::*; +mod config; +pub use config::{ + ClickhouseHost, KeeperConfig, KeeperConfigsForReplica, KeeperNodeConfig, + LogConfig, LogLevel, Macros, NodeType, RaftServerConfig, + RaftServerSettings, RaftServers, ReplicaConfig, ServerNodeConfig, +}; + +// Used for schemars to be able to be used with camino: +// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 +pub fn path_schema(gen: &mut SchemaGenerator) -> Schema { + let mut schema: SchemaObject = ::json_schema(gen).into(); + schema.format = Some("Utf8PathBuf".to_owned()); + schema.into() +} pub const OXIMETER_CLUSTER: &str = "oximeter_cluster"; @@ -59,6 +76,26 @@ pub struct KeeperId(pub u64); )] pub struct ServerId(pub u64); +/// The top most type for configuring clickhouse-servers via +/// clickhouse-admin-server-api +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct ServerConfigurableSettings { + /// A unique identifier for the configuration generation. + pub generation: Generation, + /// Configurable settings for a ClickHouse replica server node. + pub settings: ServerSettings, +} + +/// The top most type for configuring clickhouse-servers via +/// clickhouse-admin-keeper-api +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct KeeperConfigurableSettings { + /// A unique identifier for the configuration generation. + pub generation: Generation, + /// Configurable settings for a ClickHouse keeper node. + pub settings: KeeperSettings, +} + /// Configurable settings for a ClickHouse replica server node. #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] #[serde(rename_all = "snake_case")] @@ -900,6 +937,34 @@ impl KeeperConf { } } +/// The configuration of the clickhouse keeper raft cluster returned from a +/// single keeper node +/// +/// Each keeper is asked for its known raft configuration via `clickhouse-admin` +/// dropshot servers running in `ClickhouseKeeper` zones. state. We include the +/// leader committed log index known to the current keeper node (whether or not +/// it is the leader) to determine which configuration is newest. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub struct ClickhouseKeeperClusterMembership { + /// Keeper ID of the keeper being queried + pub queried_keeper: KeeperId, + /// Index of the last committed log entry from the leader's perspective + pub leader_committed_log_index: u64, + /// Keeper IDs of all keepers in the cluster + pub raft_config: BTreeSet, +} + #[cfg(test)] mod tests { use camino::Utf8PathBuf; diff --git a/clickhouse-admin/types/testutils/replica-server-config.xml b/clickhouse-admin/types/testutils/replica-server-config.xml index d918b0f2a6..cd79cf4a68 100644 --- a/clickhouse-admin/types/testutils/replica-server-config.xml +++ b/clickhouse-admin/types/testutils/replica-server-config.xml @@ -42,6 +42,13 @@ + + system + query_log
+ Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 7 DAY + 10000 + + ./data/tmp ./data/user_files default diff --git a/clients/clickhouse-admin-keeper-client/Cargo.toml b/clients/clickhouse-admin-keeper-client/Cargo.toml new file mode 100644 index 0000000000..1b8839ae64 --- /dev/null +++ b/clients/clickhouse-admin-keeper-client/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "clickhouse-admin-keeper-client" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono.workspace = true +clickhouse-admin-types.workspace = true +omicron-uuid-kinds.workspace = true +progenitor.workspace = true +reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } +schemars.workspace = true +serde.workspace = true +slog.workspace = true +omicron-workspace-hack.workspace = true + +[lints] +workspace = true diff --git a/clients/clickhouse-admin-keeper-client/src/lib.rs b/clients/clickhouse-admin-keeper-client/src/lib.rs new file mode 100644 index 0000000000..502b8ccbdd --- /dev/null +++ b/clients/clickhouse-admin-keeper-client/src/lib.rs @@ -0,0 +1,28 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Interface for making API requests to a clickhouse-admin-keeper server +//! running in an omicron zone. + +progenitor::generate_api!( + spec = "../../openapi/clickhouse-admin-keeper.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + derives = [schemars::JsonSchema], + replace = { + TypedUuidForOmicronZoneKind = omicron_uuid_kinds::OmicronZoneUuid, + KeeperConfigurableSettings = clickhouse_admin_types::KeeperConfigurableSettings, + ClickhouseKeeperClusterMembership = clickhouse_admin_types::ClickhouseKeeperClusterMembership, + KeeperId = clickhouse_admin_types::KeeperId + } +); diff --git a/clients/clickhouse-admin-server-client/Cargo.toml b/clients/clickhouse-admin-server-client/Cargo.toml new file mode 100644 index 0000000000..61142f8a7b --- /dev/null +++ b/clients/clickhouse-admin-server-client/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "clickhouse-admin-server-client" +version = "0.1.0" +edition = "2021" + +[dependencies] +chrono.workspace = true +clickhouse-admin-types.workspace = true +omicron-uuid-kinds.workspace = true +progenitor.workspace = true +reqwest = { workspace = true, features = [ "json", "rustls-tls", "stream" ] } +schemars.workspace = true +serde.workspace = true +slog.workspace = true +omicron-workspace-hack.workspace = true + +[lints] +workspace = true diff --git a/clients/clickhouse-admin-server-client/src/lib.rs b/clients/clickhouse-admin-server-client/src/lib.rs new file mode 100644 index 0000000000..3092160d65 --- /dev/null +++ b/clients/clickhouse-admin-server-client/src/lib.rs @@ -0,0 +1,26 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Interface for making API requests to a clickhouse-admin-server server +//! running in an omicron zone. + +progenitor::generate_api!( + spec = "../../openapi/clickhouse-admin-server.json", + inner_type = slog::Logger, + pre_hook = (|log: &slog::Logger, request: &reqwest::Request| { + slog::debug!(log, "client request"; + "method" => %request.method(), + "uri" => %request.url(), + "body" => ?&request.body(), + ); + }), + post_hook = (|log: &slog::Logger, result: &Result<_, _>| { + slog::debug!(log, "client response"; "result" => ?result); + }), + derives = [schemars::JsonSchema], + replace = { + TypedUuidForOmicronZoneKind = omicron_uuid_kinds::OmicronZoneUuid, + ServerConfigurableSettings = clickhouse_admin_types::ServerConfigurableSettings, + } +); diff --git a/clients/dns-service-client/Cargo.toml b/clients/dns-service-client/Cargo.toml index cdaef701bd..d6fde92315 100644 --- a/clients/dns-service-client/Cargo.toml +++ b/clients/dns-service-client/Cargo.toml @@ -8,10 +8,10 @@ license = "MPL-2.0" workspace = true [dependencies] -anyhow.workspace = true chrono.workspace = true expectorate.workspace = true http.workspace = true +internal-dns-types.workspace = true progenitor.workspace = true reqwest = { workspace = true, features = ["json", "rustls-tls", "stream"] } schemars.workspace = true diff --git a/clients/dns-service-client/src/lib.rs b/clients/dns-service-client/src/lib.rs index 316c4787b0..0f3360ab10 100644 --- a/clients/dns-service-client/src/lib.rs +++ b/clients/dns-service-client/src/lib.rs @@ -2,13 +2,6 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -mod diff; - -use crate::Error as DnsConfigError; -use anyhow::ensure; -pub use diff::DnsDiff; -use std::collections::HashMap; - progenitor::generate_api!( spec = "../../openapi/dns-server.json", inner_type = slog::Logger, @@ -23,23 +16,32 @@ progenitor::generate_api!( post_hook = (|log: &slog::Logger, result: &Result<_, _>| { slog::debug!(log, "client response"; "result" => ?result); }), + replace = { + DnsConfig = internal_dns_types::config::DnsConfig, + DnsConfigParams = internal_dns_types::config::DnsConfigParams, + DnsConfigZone = internal_dns_types::config::DnsConfigZone, + DnsRecord = internal_dns_types::config::DnsRecord, + Srv = internal_dns_types::config::Srv, + } ); +pub type DnsError = crate::Error; + pub const ERROR_CODE_UPDATE_IN_PROGRESS: &'static str = "UpdateInProgress"; pub const ERROR_CODE_BAD_UPDATE_GENERATION: &'static str = "BadUpdateGeneration"; /// Returns whether an error from this client should be retried -pub fn is_retryable(error: &DnsConfigError) -> bool { +pub fn is_retryable(error: &DnsError) -> bool { let response_value = match error { - DnsConfigError::CommunicationError(_) => return true, - DnsConfigError::InvalidRequest(_) - | DnsConfigError::InvalidResponsePayload(_, _) - | DnsConfigError::UnexpectedResponse(_) - | DnsConfigError::InvalidUpgrade(_) - | DnsConfigError::ResponseBodyError(_) - | DnsConfigError::PreHookError(_) => return false, - DnsConfigError::ErrorResponse(response_value) => response_value, + DnsError::CommunicationError(_) => return true, + DnsError::InvalidRequest(_) + | DnsError::InvalidResponsePayload(_, _) + | DnsError::UnexpectedResponse(_) + | DnsError::InvalidUpgrade(_) + | DnsError::ResponseBodyError(_) + | DnsError::PreHookError(_) => return false, + DnsError::ErrorResponse(response_value) => response_value, }; let status_code = response_value.status(); @@ -89,62 +91,3 @@ pub fn is_retryable(error: &DnsConfigError) -> bool { false } - -type DnsRecords = HashMap>; - -impl types::DnsConfigParams { - /// Given a high-level DNS configuration, return a reference to its sole - /// DNS zone. - /// - /// # Errors - /// - /// Returns an error if there are 0 or more than one zones in this - /// configuration. - pub fn sole_zone(&self) -> Result<&types::DnsConfigZone, anyhow::Error> { - ensure!( - self.zones.len() == 1, - "expected exactly one DNS zone, but found {}", - self.zones.len() - ); - Ok(&self.zones[0]) - } -} - -impl Ord for types::DnsRecord { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - use types::DnsRecord; - match (self, other) { - // Same kinds: compare the items in them - (DnsRecord::A(addr1), DnsRecord::A(addr2)) => addr1.cmp(addr2), - (DnsRecord::Aaaa(addr1), DnsRecord::Aaaa(addr2)) => { - addr1.cmp(addr2) - } - (DnsRecord::Srv(srv1), DnsRecord::Srv(srv2)) => srv1 - .target - .cmp(&srv2.target) - .then_with(|| srv1.port.cmp(&srv2.port)), - - // Different kinds: define an arbitrary order among the kinds. - // We could use std::mem::discriminant() here but it'd be nice if - // this were stable over time. - // We define (arbitrarily): A < Aaaa < Srv - (DnsRecord::A(_), DnsRecord::Aaaa(_) | DnsRecord::Srv(_)) => { - std::cmp::Ordering::Less - } - (DnsRecord::Aaaa(_), DnsRecord::Srv(_)) => std::cmp::Ordering::Less, - - // Anything else will result in "Greater". But let's be explicit. - (DnsRecord::Aaaa(_), DnsRecord::A(_)) - | (DnsRecord::Srv(_), DnsRecord::A(_)) - | (DnsRecord::Srv(_), DnsRecord::Aaaa(_)) => { - std::cmp::Ordering::Greater - } - } - } -} - -impl PartialOrd for types::DnsRecord { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} diff --git a/clients/nexus-client/src/lib.rs b/clients/nexus-client/src/lib.rs index 97f6373e29..f28a8e97bb 100644 --- a/clients/nexus-client/src/lib.rs +++ b/clients/nexus-client/src/lib.rs @@ -31,6 +31,9 @@ progenitor::generate_api!( Blueprint = nexus_types::deployment::Blueprint, Certificate = omicron_common::api::internal::nexus::Certificate, DatasetKind = omicron_common::api::internal::shared::DatasetKind, + DnsConfigParams = nexus_types::internal_api::params::DnsConfigParams, + DnsConfigZone = nexus_types::internal_api::params::DnsConfigZone, + DnsRecord = nexus_types::internal_api::params::DnsRecord, Generation = omicron_common::api::external::Generation, ImportExportPolicy = omicron_common::api::external::ImportExportPolicy, MacAddr = omicron_common::api::external::MacAddr, @@ -41,6 +44,7 @@ progenitor::generate_api!( OmicronPhysicalDiskConfig = nexus_types::disk::OmicronPhysicalDiskConfig, OmicronPhysicalDisksConfig = nexus_types::disk::OmicronPhysicalDisksConfig, RecoverySiloConfig = nexus_sled_agent_shared::recovery_silo::RecoverySiloConfig, + Srv = nexus_types::internal_api::params::Srv, TypedUuidForCollectionKind = omicron_uuid_kinds::CollectionUuid, TypedUuidForDemoSagaKind = omicron_uuid_kinds::DemoSagaUuid, TypedUuidForDownstairsKind = omicron_uuid_kinds::TypedUuid, diff --git a/clients/sled-agent-client/src/lib.rs b/clients/sled-agent-client/src/lib.rs index 257719b300..dfa89f4cc6 100644 --- a/clients/sled-agent-client/src/lib.rs +++ b/clients/sled-agent-client/src/lib.rs @@ -46,6 +46,7 @@ progenitor::generate_api!( DatasetKind = omicron_common::api::internal::shared::DatasetKind, DiskIdentity = omicron_common::disk::DiskIdentity, DiskVariant = omicron_common::disk::DiskVariant, + ExternalIpGatewayMap = omicron_common::api::internal::shared::ExternalIpGatewayMap, Generation = omicron_common::api::external::Generation, ImportExportPolicy = omicron_common::api::external::ImportExportPolicy, Inventory = nexus_sled_agent_shared::inventory::Inventory, diff --git a/common/src/api/external/mod.rs b/common/src/api/external/mod.rs index a34f5b71ac..1ddb5be864 100644 --- a/common/src/api/external/mod.rs +++ b/common/src/api/external/mod.rs @@ -971,6 +971,9 @@ pub enum ResourceType { IpPool, IpPoolResource, InstanceNetworkInterface, + InternetGateway, + InternetGatewayIpPool, + InternetGatewayIpAddress, PhysicalDisk, Rack, Service, @@ -1215,6 +1218,20 @@ pub struct InstanceAutoRestartStatus { #[serde(rename = "auto_restart_enabled")] pub enabled: bool, + /// The auto-restart policy configured for this instance, or `None` if no + /// explicit policy is configured. + /// + /// If this is not present, then this instance uses the default auto-restart + /// policy, which may or may not allow it to be restarted. The + /// `auto_restart_enabled` field indicates whether the instance will be + /// automatically restarted. + // + // Rename this field, as the struct is `#[serde(flatten)]`ed into the + // `Instance` type, and we would like the field to be prefixed with + // `auto_restart`. + #[serde(rename = "auto_restart_policy")] + pub policy: Option, + /// The time at which the auto-restart cooldown period for this instance /// completes, permitting it to be automatically restarted again. If the /// instance enters the `Failed` state, it will not be restarted until after @@ -1233,7 +1250,9 @@ pub struct InstanceAutoRestartStatus { /// A policy determining when an instance should be automatically restarted by /// the control plane. -#[derive(Copy, Clone, Debug, Deserialize, Serialize, JsonSchema)] +#[derive( + Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, Eq, PartialEq, +)] #[serde(rename_all = "snake_case")] pub enum InstanceAutoRestartPolicy { /// The instance should not be automatically restarted by the control plane @@ -1551,12 +1570,34 @@ pub struct RouterRoute { pub vpc_router_id: Uuid, /// Describes the kind of router. Set at creation. `read-only` pub kind: RouterRouteKind, - /// The location that matched packets should be forwarded to. + /// The location that matched packets should be forwarded to pub target: RouteTarget, - /// Selects which traffic this routing rule will apply to. + /// Selects which traffic this routing rule will apply to pub destination: RouteDestination, } +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InternetGatewayIpPool { + /// Common identifying metadata + #[serde(flatten)] + pub identity: IdentityMetadata, + /// The ID of the internet gateway to which the IP pool entry belongs + pub internet_gateway_id: Uuid, + /// The ID of the referenced IP pool + pub ip_pool_id: Uuid, +} + +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InternetGatewayIp { + /// Common identifying metadata + #[serde(flatten)] + pub identity: IdentityMetadata, + /// The ID of the internet gateway to which the IP belongs + pub internet_gateway_id: Uuid, + /// The IP address + pub address: IpAddr, +} + /// A single rule in a VPC firewall #[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct VpcFirewallRule { @@ -2566,8 +2607,8 @@ pub struct SwitchPortRouteConfig { /// over an 802.1Q tagged L2 segment. pub vlan_id: Option, - /// Local preference indicating priority within and across protocols. - pub local_pref: Option, + /// RIB Priority indicating priority within and across protocols. + pub rib_priority: Option, } /* diff --git a/common/src/api/internal/shared.rs b/common/src/api/internal/shared.rs index 236ee30f42..7776958254 100644 --- a/common/src/api/internal/shared.rs +++ b/common/src/api/internal/shared.rs @@ -306,9 +306,9 @@ pub struct RouteConfig { /// The VLAN id associated with this route. #[serde(default)] pub vlan_id: Option, - /// The local preference associated with this route. + /// The RIB priority (i.e. Admin Distance) associated with this route. #[serde(default)] - pub local_pref: Option, + pub rib_priority: Option, } #[derive( @@ -777,7 +777,7 @@ pub struct DhcpConfig { #[serde(tag = "type", rename_all = "snake_case", content = "value")] pub enum RouterTarget { Drop, - InternetGateway, + InternetGateway(Option), Ip(IpAddr), VpcSubnet(IpNet), } @@ -837,6 +837,13 @@ pub struct ResolvedVpcRouteSet { pub routes: HashSet, } +/// Per-NIC mappings from external IP addresses to the Internet Gateways +/// which can choose them as a source. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)] +pub struct ExternalIpGatewayMap { + pub mappings: HashMap>>, +} + /// Describes the purpose of the dataset. #[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Hash, EnumCount)] pub enum DatasetKind { diff --git a/common/src/policy.rs b/common/src/policy.rs index 5f086e7a5b..8d0ec8793f 100644 --- a/common/src/policy.rs +++ b/common/src/policy.rs @@ -13,12 +13,24 @@ pub const BOUNDARY_NTP_REDUNDANCY: usize = 2; /// Reconfigurator (to know whether to add new Nexus zones) pub const NEXUS_REDUNDANCY: usize = 3; +// The amount of redundancy for Oximeter services. +/// +/// This is used by both RSS (to distribute the initial set of services) and the +/// Reconfigurator (to know whether to add new Oximeter zones) +pub const OXIMETER_REDUNDANCY: usize = 1; + /// The amount of redundancy for CockroachDb services. /// /// This is used by both RSS (to distribute the initial set of services) and the /// Reconfigurator (to know whether to add new crdb zones) pub const COCKROACHDB_REDUNDANCY: usize = 5; +/// The amount of redundancy for Crucible Pantry services. +/// +/// This is used by both RSS (to distribute the initial set of services) and the +/// Reconfigurator (to know whether to add new pantry zones) +pub const CRUCIBLE_PANTRY_REDUNDANCY: usize = 3; + /// The amount of redundancy for internal DNS servers. /// /// Must be less than or equal to RESERVED_INTERNAL_DNS_REDUNDANCY. @@ -34,6 +46,10 @@ pub const INTERNAL_DNS_REDUNDANCY: usize = 3; /// value. pub const RESERVED_INTERNAL_DNS_REDUNDANCY: usize = 5; +/// The amount of redundancy for single-node ClickHouse servers +/// (*not* replicated aka multi-node clusters). +pub const SINGLE_NODE_CLICKHOUSE_REDUNDANCY: usize = 1; + /// The amount of redundancy for clickhouse servers /// /// Clickhouse servers contain lazily replicated data diff --git a/dev-tools/ls-apis/Cargo.toml b/dev-tools/ls-apis/Cargo.toml index f66f3e4ee2..9a5f3197ea 100644 --- a/dev-tools/ls-apis/Cargo.toml +++ b/dev-tools/ls-apis/Cargo.toml @@ -18,3 +18,8 @@ petgraph.workspace = true serde.workspace = true toml.workspace = true omicron-workspace-hack.workspace = true + +[dev-dependencies] +expectorate.workspace = true +omicron-test-utils.workspace = true +subprocess.workspace = true diff --git a/dev-tools/ls-apis/api-manifest.toml b/dev-tools/ls-apis/api-manifest.toml index 65dc28d7b2..f91aba74aa 100644 --- a/dev-tools/ls-apis/api-manifest.toml +++ b/dev-tools/ls-apis/api-manifest.toml @@ -138,6 +138,24 @@ client_package_name = "bootstrap-agent-client" label = "Bootstrap Agent" server_package_name = "bootstrap-agent-api" +[[apis]] +client_package_name = "clickhouse-admin-keeper-client" +label = "Clickhouse Cluster Admin for Keepers" +server_package_name = "clickhouse-admin-api" +notes = """ +This is the server running inside multi-node Clickhouse keeper zones that's \ +responsible for local configuration and monitoring. +""" + +[[apis]] +client_package_name = "clickhouse-admin-server-client" +label = "Clickhouse Cluster Admin for Servers" +server_package_name = "clickhouse-admin-api" +notes = """ +This is the server running inside multi-node Clickhouse server zones that's \ +responsible for local configuration and monitoring. +""" + [[apis]] client_package_name = "cockroach-admin-client" label = "CockroachDB Cluster Admin" @@ -387,18 +405,6 @@ mg-admin-client, which isn't true. It'd be nice to remove this. Most clients put those conversions into the client rather than omicron_common. """ -[[dependency_filter_rules]] -ancestor = "internal-dns" -client = "dns-service-client" -evaluation = "bogus" -note = """ -internal-dns depends on dns-service-client to use its types. They're only used -when configuring DNS, which is only done in a couple of components. But many -other components use internal-dns solely to read DNS. This dependency makes it -look like everything uses the DNS server API, but that's not true. We should -consider splitting this crate in two to eliminate this false positive. -""" - [[dependency_filter_rules]] ancestor = "nexus-types" client = "gateway-client" @@ -412,7 +418,8 @@ ancestor = "nexus-types" client = "dns-service-client" evaluation = "bogus" note = """ -nexus-types depends on dns-service-client for defining some types. +Past versions of nexus-types that are still referenced in the dependency tree +depended on dns-service-client for defining some types. """ [[dependency_filter_rules]] diff --git a/dev-tools/ls-apis/src/system_apis.rs b/dev-tools/ls-apis/src/system_apis.rs index 6d624d4e57..60b9b9246a 100644 --- a/dev-tools/ls-apis/src/system_apis.rs +++ b/dev-tools/ls-apis/src/system_apis.rs @@ -67,9 +67,17 @@ impl SystemApis { // Load Cargo metadata and validate it against the manifest. let (workspaces, warnings) = Workspaces::load(&api_metadata)?; if !warnings.is_empty() { + // We treat these warnings as fatal here. for e in warnings { - eprintln!("warning: {:#}", e); + eprintln!("error: {:#}", e); } + + bail!( + "found inconsistency between API manifest ({}) and \ + information found from the Cargo dependency tree \ + (see above)", + &args.api_manifest_path + ); } // Create an index of server package names, mapping each one to the API diff --git a/dev-tools/ls-apis/src/workspaces.rs b/dev-tools/ls-apis/src/workspaces.rs index ef1ba0ee79..ace565e011 100644 --- a/dev-tools/ls-apis/src/workspaces.rs +++ b/dev-tools/ls-apis/src/workspaces.rs @@ -28,7 +28,7 @@ impl Workspaces { /// The data found is validated against `api_metadata`. /// /// On success, returns `(workspaces, warnings)`, where `warnings` is a list - /// of potential inconsistencies between API metadata and Cargo metadata. + /// of inconsistencies between API metadata and Cargo metadata. pub fn load( api_metadata: &AllApiMetadata, ) -> Result<(Workspaces, Vec)> { @@ -114,8 +114,8 @@ impl Workspaces { client_pkgnames_unused.remove(client_pkgname); } else { warnings.push(anyhow!( - "workspace {}: found client package missing from API \ - manifest: {}", + "workspace {}: found Progenitor-based client package \ + missing from API manifest: {}", workspace.name(), client_pkgname )); diff --git a/dev-tools/ls-apis/tests/api_dependencies.out b/dev-tools/ls-apis/tests/api_dependencies.out new file mode 100644 index 0000000000..a902484a75 --- /dev/null +++ b/dev-tools/ls-apis/tests/api_dependencies.out @@ -0,0 +1,80 @@ +Bootstrap Agent (client: bootstrap-agent-client) + consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: wicketd (omicron/wicketd) + +Clickhouse Cluster Admin for Keepers (client: clickhouse-admin-keeper-client) + consumed by: omicron-nexus (omicron/nexus) + +Clickhouse Cluster Admin for Servers (client: clickhouse-admin-server-client) + consumed by: omicron-nexus (omicron/nexus) + +CockroachDB Cluster Admin (client: cockroach-admin-client) + consumed by: omicron-nexus (omicron/nexus) + +Crucible Agent (client: crucible-agent-client) + consumed by: omicron-nexus (omicron/nexus) + +Crucible Control (for testing only) (client: crucible-control-client) + +Crucible Pantry (client: crucible-pantry-client) + consumed by: omicron-nexus (omicron/nexus) + +Maghemite DDM Admin (client: ddm-admin-client) + consumed by: installinator (omicron/installinator) + consumed by: mgd (maghemite/mgd) + consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: wicketd (omicron/wicketd) + +DNS Server (client: dns-service-client) + consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-sled-agent (omicron/sled-agent) + +Dendrite DPD (client: dpd-client) + consumed by: ddmd (maghemite/ddmd) + consumed by: mgd (maghemite/mgd) + consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: tfportd (dendrite/tfportd) + consumed by: wicketd (omicron/wicketd) + +Downstairs Controller (debugging only) (client: dsc-client) + +Management Gateway Service (client: gateway-client) + consumed by: dpd (dendrite/dpd) + consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: wicketd (omicron/wicketd) + +Wicketd Installinator (client: installinator-client) + consumed by: installinator (omicron/installinator) + +Maghemite MG Admin (client: mg-admin-client) + consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-sled-agent (omicron/sled-agent) + +Nexus Internal API (client: nexus-client) + consumed by: dpd (dendrite/dpd) + consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-sled-agent (omicron/sled-agent) + consumed by: oximeter-collector (omicron/oximeter/collector) + consumed by: propolis-server (propolis/bin/propolis-server) + +External API (client: oxide-client) + +Oximeter (client: oximeter-client) + consumed by: omicron-nexus (omicron/nexus) + +Propolis (client: propolis-client) + consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-sled-agent (omicron/sled-agent) + +Crucible Repair (client: repair-client) + consumed by: crucible-downstairs (crucible/downstairs) + +Sled Agent (client: sled-agent-client) + consumed by: dpd (dendrite/dpd) + consumed by: omicron-nexus (omicron/nexus) + consumed by: omicron-sled-agent (omicron/sled-agent) + +Wicketd (client: wicketd-client) + diff --git a/dev-tools/ls-apis/tests/test_dependencies.rs b/dev-tools/ls-apis/tests/test_dependencies.rs new file mode 100644 index 0000000000..ffb0edc566 --- /dev/null +++ b/dev-tools/ls-apis/tests/test_dependencies.rs @@ -0,0 +1,29 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! The tests here are intended help us catch cases where we're introducing new +//! API dependencies, and particular circular API dependencies. It's okay if +//! API dependencies change, but it's important to make sure we're not +//! introducing new barriers to online upgrade. +//! +//! This isn't (supposed to be) a test for the `ls-apis` tool itself. + +use omicron_test_utils::dev::test_cmds::assert_exit_code; +use omicron_test_utils::dev::test_cmds::path_to_executable; +use omicron_test_utils::dev::test_cmds::run_command; +use omicron_test_utils::dev::test_cmds::EXIT_SUCCESS; + +/// name of the "ls-apis" executable +const CMD_LS_APIS: &str = env!("CARGO_BIN_EXE_ls-apis"); + +#[test] +fn test_api_dependencies() { + let cmd_path = path_to_executable(CMD_LS_APIS); + let exec = subprocess::Exec::cmd(cmd_path).arg("apis"); + let (exit_status, stdout_text, stderr_text) = run_command(exec); + assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text); + + println!("stderr:\n------\n{}\n-----", stderr_text); + expectorate::assert_contents("tests/api_dependencies.out", &stdout_text); +} diff --git a/dev-tools/omdb/Cargo.toml b/dev-tools/omdb/Cargo.toml index 3e942523b4..78bceb8cc0 100644 --- a/dev-tools/omdb/Cargo.toml +++ b/dev-tools/omdb/Cargo.toml @@ -27,7 +27,8 @@ gateway-client.workspace = true gateway-messages.workspace = true gateway-test-utils.workspace = true humantime.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true itertools.workspace = true nexus-client.workspace = true nexus-config.workspace = true diff --git a/dev-tools/omdb/src/bin/omdb/db.rs b/dev-tools/omdb/src/bin/omdb/db.rs index be068f0912..29aa0f9376 100644 --- a/dev-tools/omdb/src/bin/omdb/db.rs +++ b/dev-tools/omdb/src/bin/omdb/db.rs @@ -45,6 +45,7 @@ use gateway_client::types::SpType; use indicatif::ProgressBar; use indicatif::ProgressDrawTarget; use indicatif::ProgressStyle; +use internal_dns_types::names::ServiceName; use ipnetwork::IpNetwork; use nexus_config::PostgresConfigWithUrl; use nexus_db_model::Dataset; @@ -221,10 +222,7 @@ impl DbUrlOptions { ); eprintln!("note: (override with --db-url or OMDB_DB_URL)"); let addrs = omdb - .dns_lookup_all( - log.clone(), - internal_dns::ServiceName::Cockroach, - ) + .dns_lookup_all(log.clone(), ServiceName::Cockroach) .await?; format!( @@ -5182,31 +5180,23 @@ fn inv_collection_print_sleds(collection: &Collection) { println!(" reservation: {reservation:?}, quota: {quota:?}"); } - if let Some(zones) = collection.omicron_zones.get(&sled.sled_id) { - println!( - " zones collected from {} at {}", - zones.source, zones.time_collected, - ); - println!( - " zones generation: {} (count: {})", - zones.zones.generation, - zones.zones.zones.len() - ); + println!( + " zones generation: {} (count: {})", + sled.omicron_zones.generation, + sled.omicron_zones.zones.len(), + ); - if zones.zones.zones.is_empty() { - continue; - } + if sled.omicron_zones.zones.is_empty() { + continue; + } - println!(" ZONES FOUND"); - for z in &zones.zones.zones { - println!( - " zone {} (type {})", - z.id, - z.zone_type.kind().report_str() - ); - } - } else { - println!(" warning: no zone information found"); + println!(" ZONES FOUND"); + for z in &sled.omicron_zones.zones { + println!( + " zone {} (type {})", + z.id, + z.zone_type.kind().report_str() + ); } } } diff --git a/dev-tools/omdb/src/bin/omdb/main.rs b/dev-tools/omdb/src/bin/omdb/main.rs index f1a13310ca..f5c5d3f907 100644 --- a/dev-tools/omdb/src/bin/omdb/main.rs +++ b/dev-tools/omdb/src/bin/omdb/main.rs @@ -41,6 +41,7 @@ use clap::ColorChoice; use clap::Parser; use clap::Subcommand; use futures::StreamExt; +use internal_dns_types::names::ServiceName; use omicron_common::address::Ipv6Subnet; use std::net::SocketAddr; use std::net::SocketAddrV6; @@ -151,7 +152,7 @@ impl Omdb { async fn dns_lookup_all( &self, log: slog::Logger, - service_name: internal_dns::ServiceName, + service_name: ServiceName, ) -> Result, anyhow::Error> { let resolver = self.dns_resolver(log).await?; resolver @@ -165,7 +166,7 @@ impl Omdb { async fn dns_lookup_one( &self, log: slog::Logger, - service_name: internal_dns::ServiceName, + service_name: ServiceName, ) -> Result { let addrs = self.dns_lookup_all(log, service_name).await?; ensure!( @@ -222,10 +223,10 @@ impl Omdb { async fn dns_resolver( &self, log: slog::Logger, - ) -> Result { + ) -> Result { match &self.dns_server { Some(dns_server) => { - internal_dns::resolver::Resolver::new_from_addrs( + internal_dns_resolver::Resolver::new_from_addrs( log, &[*dns_server], ) @@ -258,7 +259,7 @@ impl Omdb { "note: (if this is not right, use --dns-server \ to specify an alternate DNS server)", ); - internal_dns::resolver::Resolver::new_from_subnet(log, subnet) + internal_dns_resolver::Resolver::new_from_subnet(log, subnet) .with_context(|| { format!( "creating DNS resolver for subnet {}", diff --git a/dev-tools/omdb/src/bin/omdb/mgs.rs b/dev-tools/omdb/src/bin/omdb/mgs.rs index 6b7c8b2641..0db1731ca6 100644 --- a/dev-tools/omdb/src/bin/omdb/mgs.rs +++ b/dev-tools/omdb/src/bin/omdb/mgs.rs @@ -21,6 +21,7 @@ use gateway_client::types::SpIgnitionInfo; use gateway_client::types::SpIgnitionSystemType; use gateway_client::types::SpState; use gateway_client::types::SpType; +use internal_dns_types::names::ServiceName; use tabled::Tabled; mod dashboard; @@ -75,7 +76,7 @@ impl MgsArgs { let addr = omdb .dns_lookup_one( log.clone(), - internal_dns::ServiceName::ManagementGatewayService, + ServiceName::ManagementGatewayService, ) .await?; format!("http://{}", addr) diff --git a/dev-tools/omdb/src/bin/omdb/nexus.rs b/dev-tools/omdb/src/bin/omdb/nexus.rs index 8429d9a446..326a6b6384 100644 --- a/dev-tools/omdb/src/bin/omdb/nexus.rs +++ b/dev-tools/omdb/src/bin/omdb/nexus.rs @@ -22,6 +22,7 @@ use clap::Subcommand; use clap::ValueEnum; use futures::future::try_join; use futures::TryStreamExt; +use internal_dns_types::names::ServiceName; use itertools::Itertools; use nexus_client::types::ActivationReason; use nexus_client::types::BackgroundTask; @@ -391,10 +392,7 @@ impl NexusArgs { "note: Nexus URL not specified. Will pick one from DNS." ); let addr = omdb - .dns_lookup_one( - log.clone(), - internal_dns::ServiceName::Nexus, - ) + .dns_lookup_one(log.clone(), ServiceName::Nexus) .await?; format!("http://{}", addr) } @@ -1129,6 +1127,14 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) { println!(" > {line}"); } + println!( + " region replacement requests set to completed ok: {}", + status.requests_completed_ok.len() + ); + for line in &status.requests_completed_ok { + println!(" > {line}"); + } + println!(" errors: {}", status.errors.len()); for line in &status.errors { println!(" > {line}"); diff --git a/dev-tools/omdb/src/bin/omdb/oximeter.rs b/dev-tools/omdb/src/bin/omdb/oximeter.rs index c068110b4c..cc1efd126f 100644 --- a/dev-tools/omdb/src/bin/omdb/oximeter.rs +++ b/dev-tools/omdb/src/bin/omdb/oximeter.rs @@ -10,6 +10,7 @@ use anyhow::Context; use clap::Args; use clap::Subcommand; use futures::TryStreamExt; +use internal_dns_types::names::ServiceName; use oximeter_client::types::ProducerEndpoint; use oximeter_client::Client; use slog::Logger; @@ -55,10 +56,7 @@ impl OximeterArgs { "note: Oximeter URL not specified. Will pick one from DNS." ); let addr = omdb - .dns_lookup_one( - log.clone(), - internal_dns::ServiceName::Oximeter, - ) + .dns_lookup_one(log.clone(), ServiceName::Oximeter) .await?; format!("http://{}", addr) } diff --git a/dev-tools/omdb/src/bin/omdb/oxql.rs b/dev-tools/omdb/src/bin/omdb/oxql.rs index 89ddae9cf2..28f405e067 100644 --- a/dev-tools/omdb/src/bin/omdb/oxql.rs +++ b/dev-tools/omdb/src/bin/omdb/oxql.rs @@ -10,6 +10,7 @@ use crate::helpers::CONNECTION_OPTIONS_HEADING; use crate::Omdb; use anyhow::Context; use clap::Args; +use internal_dns_types::names::ServiceName; use oximeter_db::{ self, shells::oxql::{self, ShellOptions}, @@ -86,7 +87,7 @@ impl OxqlArgs { Ok(SocketAddr::V6( omdb.dns_lookup_one( log.clone(), - internal_dns::ServiceName::Clickhouse, + ServiceName::Clickhouse, ) .await .context("failed looking up ClickHouse internal DNS entry")?, diff --git a/dev-tools/omdb/tests/successes.out b/dev-tools/omdb/tests/successes.out index 0d66d1ed3e..1e99dbd3a8 100644 --- a/dev-tools/omdb/tests/successes.out +++ b/dev-tools/omdb/tests/successes.out @@ -401,14 +401,14 @@ termination: Exited(0) --------------------------------------------- stdout: task: "dns_config_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms last generation found: 1 task: "dns_servers_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -418,7 +418,7 @@ task: "dns_servers_internal" [::1]:REDACTED_PORT task: "dns_propagation_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a dependent task completing started at (s ago) and ran for ms @@ -429,14 +429,14 @@ task: "dns_propagation_internal" task: "dns_config_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms last generation found: 2 task: "dns_servers_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -446,7 +446,7 @@ task: "dns_servers_external" [::1]:REDACTED_PORT task: "dns_propagation_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a dependent task completing started at (s ago) and ran for ms @@ -464,21 +464,21 @@ task: "nat_v4_garbage_collector" last completion reported error: failed to resolve addresses for Dendrite services: no record found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } task: "blueprint_loader" - configured period: every 1m s + configured period: every m s currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms last completion reported error: failed to read target blueprint: Internal Error: no target blueprint set task: "blueprint_executor" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms last completion reported error: no blueprint task: "abandoned_vmm_reaper" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -495,21 +495,21 @@ task: "bfd_manager" last completion reported error: failed to resolve addresses for Dendrite services: no record found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } task: "crdb_node_id_collector" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms last completion reported error: no blueprint task: "decommissioned_disk_cleaner" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms warning: unknown background task: "decommissioned_disk_cleaner" (don't know how to interpret details: Object {"deleted": Number(0), "error": Null, "error_count": Number(0), "found": Number(0), "not_ready_to_be_deleted": Number(0)}) task: "external_endpoints" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -526,7 +526,7 @@ task: "external_endpoints" TLS certificates: 0 task: "instance_reincarnation" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -565,7 +565,7 @@ task: "instance_watcher" stale instance metrics pruned: 0 task: "inventory_collection" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -574,7 +574,7 @@ task: "inventory_collection" last collection done: task: "lookup_region_port" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -582,7 +582,7 @@ task: "lookup_region_port" errors: 0 task: "metrics_producer_gc" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -604,16 +604,17 @@ task: "physical_disk_adoption" last completion reported error: task disabled task: "region_replacement" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms region replacement requests created ok: 0 region replacement start sagas started ok: 0 + region replacement requests set to completed ok: 0 errors: 0 task: "region_replacement_driver" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -622,7 +623,7 @@ task: "region_replacement_driver" errors: 0 task: "region_snapshot_replacement_finish" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -630,7 +631,7 @@ task: "region_snapshot_replacement_finish" errors: 0 task: "region_snapshot_replacement_garbage_collection" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -638,7 +639,7 @@ task: "region_snapshot_replacement_garbage_collection" errors: 0 task: "region_snapshot_replacement_start" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -647,7 +648,7 @@ task: "region_snapshot_replacement_start" errors: 0 task: "region_snapshot_replacement_step" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -657,7 +658,7 @@ task: "region_snapshot_replacement_step" errors: 0 task: "saga_recovery" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -678,7 +679,7 @@ task: "saga_recovery" no saga recovery failures task: "service_firewall_rule_propagation" - configured period: every 5m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -720,7 +721,7 @@ termination: Exited(0) --------------------------------------------- stdout: task: "saga_recovery" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -749,14 +750,14 @@ termination: Exited(0) --------------------------------------------- stdout: task: "blueprint_loader" - configured period: every 1m s + configured period: every m s currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms last completion reported error: failed to read target blueprint: Internal Error: no target blueprint set task: "blueprint_executor" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -771,14 +772,14 @@ termination: Exited(0) --------------------------------------------- stdout: task: "dns_config_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms last generation found: 1 task: "dns_servers_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -788,7 +789,7 @@ task: "dns_servers_internal" [::1]:REDACTED_PORT task: "dns_propagation_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a dependent task completing started at (s ago) and ran for ms @@ -807,14 +808,14 @@ termination: Exited(0) --------------------------------------------- stdout: task: "dns_config_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms last generation found: 2 task: "dns_servers_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -824,7 +825,7 @@ task: "dns_servers_external" [::1]:REDACTED_PORT task: "dns_propagation_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a dependent task completing started at (s ago) and ran for ms @@ -843,14 +844,14 @@ termination: Exited(0) --------------------------------------------- stdout: task: "dns_config_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms last generation found: 1 task: "dns_servers_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -860,7 +861,7 @@ task: "dns_servers_internal" [::1]:REDACTED_PORT task: "dns_propagation_internal" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a dependent task completing started at (s ago) and ran for ms @@ -871,14 +872,14 @@ task: "dns_propagation_internal" task: "dns_config_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms last generation found: 2 task: "dns_servers_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -888,7 +889,7 @@ task: "dns_servers_external" [::1]:REDACTED_PORT task: "dns_propagation_external" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a dependent task completing started at (s ago) and ran for ms @@ -906,21 +907,21 @@ task: "nat_v4_garbage_collector" last completion reported error: failed to resolve addresses for Dendrite services: no record found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } task: "blueprint_loader" - configured period: every 1m s + configured period: every m s currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms last completion reported error: failed to read target blueprint: Internal Error: no target blueprint set task: "blueprint_executor" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms last completion reported error: no blueprint task: "abandoned_vmm_reaper" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -937,21 +938,21 @@ task: "bfd_manager" last completion reported error: failed to resolve addresses for Dendrite services: no record found for Query { name: Name("_dendrite._tcp.control-plane.oxide.internal."), query_type: SRV, query_class: IN } task: "crdb_node_id_collector" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms last completion reported error: no blueprint task: "decommissioned_disk_cleaner" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms warning: unknown background task: "decommissioned_disk_cleaner" (don't know how to interpret details: Object {"deleted": Number(0), "error": Null, "error_count": Number(0), "found": Number(0), "not_ready_to_be_deleted": Number(0)}) task: "external_endpoints" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -968,7 +969,7 @@ task: "external_endpoints" TLS certificates: 0 task: "instance_reincarnation" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1007,7 +1008,7 @@ task: "instance_watcher" stale instance metrics pruned: 0 task: "inventory_collection" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by an explicit signal started at (s ago) and ran for ms @@ -1016,7 +1017,7 @@ task: "inventory_collection" last collection done: task: "lookup_region_port" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1024,7 +1025,7 @@ task: "lookup_region_port" errors: 0 task: "metrics_producer_gc" - configured period: every 1m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1046,16 +1047,17 @@ task: "physical_disk_adoption" last completion reported error: task disabled task: "region_replacement" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms region replacement requests created ok: 0 region replacement start sagas started ok: 0 + region replacement requests set to completed ok: 0 errors: 0 task: "region_replacement_driver" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1064,7 +1066,7 @@ task: "region_replacement_driver" errors: 0 task: "region_snapshot_replacement_finish" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1072,7 +1074,7 @@ task: "region_snapshot_replacement_finish" errors: 0 task: "region_snapshot_replacement_garbage_collection" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1080,7 +1082,7 @@ task: "region_snapshot_replacement_garbage_collection" errors: 0 task: "region_snapshot_replacement_start" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1089,7 +1091,7 @@ task: "region_snapshot_replacement_start" errors: 0 task: "region_snapshot_replacement_step" - configured period: every s + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1099,7 +1101,7 @@ task: "region_snapshot_replacement_step" errors: 0 task: "saga_recovery" - configured period: every 10m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms @@ -1120,7 +1122,7 @@ task: "saga_recovery" no saga recovery failures task: "service_firewall_rule_propagation" - configured period: every 5m + configured period: every m currently executing: no last completed activation: , triggered by a periodic timer firing started at (s ago) and ran for ms diff --git a/dev-tools/omdb/tests/test_all_output.rs b/dev-tools/omdb/tests/test_all_output.rs index be14d07765..57594fcca5 100644 --- a/dev-tools/omdb/tests/test_all_output.rs +++ b/dev-tools/omdb/tests/test_all_output.rs @@ -16,9 +16,8 @@ use nexus_types::deployment::Blueprint; use nexus_types::deployment::SledFilter; use nexus_types::deployment::UnstableReconfiguratorState; use omicron_test_utils::dev::test_cmds::path_to_executable; -use omicron_test_utils::dev::test_cmds::redact_extra; use omicron_test_utils::dev::test_cmds::run_command; -use omicron_test_utils::dev::test_cmds::ExtraRedactions; +use omicron_test_utils::dev::test_cmds::Redactor; use slog_error_chain::InlineErrorChain; use std::fmt::Write; use std::net::IpAddr; @@ -203,19 +202,21 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) { // ControlPlaneTestContext. ]; - let mut redactions = ExtraRedactions::new(); - redactions - .variable_length("tmp_path", tmppath.as_str()) - .fixed_length("blueprint_id", &initial_blueprint_id) - .variable_length( + let mut redactor = Redactor::default(); + redactor + .extra_variable_length("tmp_path", tmppath.as_str()) + .extra_fixed_length("blueprint_id", &initial_blueprint_id) + .extra_variable_length( "cockroachdb_fingerprint", &initial_blueprint.cockroachdb_fingerprint, ); + let crdb_version = initial_blueprint.cockroachdb_setting_preserve_downgrade.to_string(); if initial_blueprint.cockroachdb_setting_preserve_downgrade.is_set() { - redactions.variable_length("cockroachdb_version", &crdb_version); + redactor.extra_variable_length("cockroachdb_version", &crdb_version); } + for args in invocations { println!("running commands with args: {:?}", args); let p = postgres_url.to_string(); @@ -234,7 +235,7 @@ async fn test_omdb_success_cases(cptestctx: &ControlPlaneTestContext) { }, &cmd_path, args, - Some(&redactions), + &redactor, ) .await; } @@ -444,14 +445,7 @@ async fn do_run( ) where F: FnOnce(Exec) -> Exec + Send + 'static, { - do_run_extra( - output, - modexec, - cmd_path, - args, - Some(&ExtraRedactions::new()), - ) - .await; + do_run_extra(output, modexec, cmd_path, args, &Redactor::default()).await; } async fn do_run_no_redactions( @@ -462,7 +456,7 @@ async fn do_run_no_redactions( ) where F: FnOnce(Exec) -> Exec + Send + 'static, { - do_run_extra(output, modexec, cmd_path, args, None).await; + do_run_extra(output, modexec, cmd_path, args, &Redactor::noop()).await; } async fn do_run_extra( @@ -470,7 +464,7 @@ async fn do_run_extra( modexec: F, cmd_path: &Path, args: &[&str], - extra_redactions: Option<&ExtraRedactions<'_>>, + redactor: &Redactor<'_>, ) where F: FnOnce(Exec) -> Exec + Send + 'static, { @@ -478,14 +472,7 @@ async fn do_run_extra( output, "EXECUTING COMMAND: {} {:?}\n", cmd_path.file_name().expect("missing command").to_string_lossy(), - args.iter() - .map(|r| { - extra_redactions.map_or_else( - || r.to_string(), - |redactions| redact_extra(r, redactions), - ) - }) - .collect::>() + args.iter().map(|r| redactor.do_redact(r)).collect::>() ) .unwrap(); @@ -521,21 +508,11 @@ async fn do_run_extra( write!(output, "termination: {:?}\n", exit_status).unwrap(); write!(output, "---------------------------------------------\n").unwrap(); write!(output, "stdout:\n").unwrap(); - - if let Some(extra_redactions) = extra_redactions { - output.push_str(&redact_extra(&stdout_text, extra_redactions)); - } else { - output.push_str(&stdout_text); - } + output.push_str(&redactor.do_redact(&stdout_text)); write!(output, "---------------------------------------------\n").unwrap(); write!(output, "stderr:\n").unwrap(); - - if let Some(extra_redactions) = extra_redactions { - output.push_str(&redact_extra(&stderr_text, extra_redactions)); - } else { - output.push_str(&stderr_text); - } + output.push_str(&redactor.do_redact(&stderr_text)); write!(output, "=============================================\n").unwrap(); } diff --git a/dev-tools/openapi-manager/src/spec.rs b/dev-tools/openapi-manager/src/spec.rs index 7d734218fc..dafcebac05 100644 --- a/dev-tools/openapi-manager/src/spec.rs +++ b/dev-tools/openapi-manager/src/spec.rs @@ -27,14 +27,25 @@ pub fn all_apis() -> Vec { extra_validation: None, }, ApiSpec { - title: "ClickHouse Cluster Admin API", + title: "ClickHouse Cluster Admin Keeper API", version: "0.0.1", description: "API for interacting with the Oxide \ - control plane's ClickHouse cluster", + control plane's ClickHouse cluster keepers", boundary: ApiBoundary::Internal, api_description: - clickhouse_admin_api::clickhouse_admin_api_mod::stub_api_description, - filename: "clickhouse-admin.json", + clickhouse_admin_api::clickhouse_admin_keeper_api_mod::stub_api_description, + filename: "clickhouse-admin-keeper.json", + extra_validation: None, + }, + ApiSpec { + title: "ClickHouse Cluster Admin Server API", + version: "0.0.1", + description: "API for interacting with the Oxide \ + control plane's ClickHouse cluster replica servers", + boundary: ApiBoundary::Internal, + api_description: + clickhouse_admin_api::clickhouse_admin_server_api_mod::stub_api_description, + filename: "clickhouse-admin-server.json", extra_validation: None, }, ApiSpec { @@ -82,7 +93,7 @@ pub fn all_apis() -> Vec { }, ApiSpec { title: "Oxide Region API", - version: "20241009.0", + version: "20241204.0", description: "API for interacting with the Oxide control plane", boundary: ApiBoundary::External, api_description: diff --git a/dev-tools/reconfigurator-cli/Cargo.toml b/dev-tools/reconfigurator-cli/Cargo.toml index b24c0eef36..2aab2c2333 100644 --- a/dev-tools/reconfigurator-cli/Cargo.toml +++ b/dev-tools/reconfigurator-cli/Cargo.toml @@ -14,14 +14,14 @@ omicron-rpaths.workspace = true anyhow.workspace = true assert_matches.workspace = true camino.workspace = true +chrono.workspace = true clap.workspace = true -dns-service-client.workspace = true dropshot.workspace = true humantime.workspace = true indexmap.workspace = true +internal-dns-types.workspace = true nexus-inventory.workspace = true nexus-reconfigurator-planning.workspace = true -nexus-reconfigurator-execution.workspace = true nexus-sled-agent-shared.workspace = true nexus-types.workspace = true omicron-common.workspace = true @@ -34,6 +34,7 @@ slog-error-chain.workspace = true slog.workspace = true swrite.workspace = true tabled.workspace = true +typed-rng.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true diff --git a/dev-tools/reconfigurator-cli/src/main.rs b/dev-tools/reconfigurator-cli/src/main.rs index 188319b665..f70ac2fc23 100644 --- a/dev-tools/reconfigurator-cli/src/main.rs +++ b/dev-tools/reconfigurator-cli/src/main.rs @@ -6,24 +6,25 @@ use anyhow::{anyhow, bail, Context}; use camino::Utf8PathBuf; +use chrono::Utc; use clap::CommandFactory; use clap::FromArgMatches; use clap::ValueEnum; use clap::{Args, Parser, Subcommand}; -use dns_service_client::DnsDiff; use indexmap::IndexMap; +use internal_dns_types::diff::DnsDiff; use nexus_inventory::CollectionBuilder; -use nexus_reconfigurator_execution::blueprint_external_dns_config; -use nexus_reconfigurator_execution::blueprint_internal_dns_config; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; use nexus_reconfigurator_planning::blueprint_builder::EnsureMultiple; +use nexus_reconfigurator_planning::example::ExampleSystemBuilder; use nexus_reconfigurator_planning::planner::Planner; use nexus_reconfigurator_planning::system::{ SledBuilder, SledHwInventory, SystemDescription, }; -use nexus_sled_agent_shared::inventory::OmicronZonesConfig; -use nexus_sled_agent_shared::inventory::SledRole; use nexus_sled_agent_shared::inventory::ZoneKind; +use nexus_types::deployment::execution; +use nexus_types::deployment::execution::blueprint_external_dns_config; +use nexus_types::deployment::execution::blueprint_internal_dns_config; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::OmicronZoneNic; use nexus_types::deployment::PlanningInput; @@ -34,6 +35,8 @@ use nexus_types::internal_api::params::DnsConfigParams; use nexus_types::inventory::Collection; use omicron_common::api::external::Generation; use omicron_common::api::external::Name; +use omicron_common::policy::NEXUS_REDUNDANCY; +use omicron_uuid_kinds::CollectionKind; use omicron_uuid_kinds::CollectionUuid; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; @@ -45,6 +48,7 @@ use std::collections::BTreeMap; use std::io::BufRead; use swrite::{swriteln, SWrite}; use tabled::Tabled; +use typed_rng::TypedUuidRng; use uuid::Uuid; /// REPL state @@ -76,6 +80,9 @@ struct ReconfiguratorSim { /// External DNS zone name configured external_dns_zone_name: String, + /// RNG for collection IDs + collection_id_rng: TypedUuidRng, + /// Policy overrides num_nexus: Option, @@ -83,6 +90,57 @@ struct ReconfiguratorSim { } impl ReconfiguratorSim { + fn new(log: slog::Logger) -> Self { + Self { + system: SystemDescription::new(), + collections: IndexMap::new(), + blueprints: IndexMap::new(), + internal_dns: BTreeMap::new(), + external_dns: BTreeMap::new(), + silo_names: vec!["example-silo".parse().unwrap()], + external_dns_zone_name: String::from("oxide.example"), + collection_id_rng: TypedUuidRng::from_entropy(), + num_nexus: None, + log, + } + } + + /// Returns true if the user has made local changes to the simulated + /// system. + /// + /// This is used when the user asks to load an example system. Doing that + /// basically requires a clean slate. + fn user_made_system_changes(&self) -> bool { + // Use this pattern to ensure that if a new field is added to + // ReconfiguratorSim, it will fail to compile until it's added here. + let Self { + system, + collections, + blueprints, + internal_dns, + external_dns, + // For purposes of this method, we let these policy parameters be + // set to any arbitrary value. This lets example systems be + // generated using these values. + silo_names: _, + external_dns_zone_name: _, + collection_id_rng: _, + num_nexus: _, + log: _, + } = self; + + system.has_sleds() + || !collections.is_empty() + || !blueprints.is_empty() + || !internal_dns.is_empty() + || !external_dns.is_empty() + } + + // Reset the state of the REPL. + fn wipe(&mut self) { + *self = Self::new(self.log.clone()); + } + fn blueprint_lookup(&self, id: Uuid) -> Result<&Blueprint, anyhow::Error> { self.blueprints .get(&id) @@ -181,22 +239,12 @@ fn main() -> anyhow::Result<()> { let cmd = CmdReconfiguratorSim::parse(); let log = dropshot::ConfigLogging::StderrTerminal { - level: dropshot::ConfigLoggingLevel::Debug, + level: dropshot::ConfigLoggingLevel::Info, } .to_logger("reconfigurator-sim") .context("creating logger")?; - let mut sim = ReconfiguratorSim { - system: SystemDescription::new(), - collections: IndexMap::new(), - blueprints: IndexMap::new(), - internal_dns: BTreeMap::new(), - external_dns: BTreeMap::new(), - log, - silo_names: vec!["example-silo".parse().unwrap()], - external_dns_zone_name: String::from("oxide.example"), - num_nexus: None, - }; + let mut sim = ReconfiguratorSim::new(log); if let Some(input_file) = cmd.input_file { let file = std::fs::File::open(&input_file) @@ -310,8 +358,10 @@ fn process_entry(sim: &mut ReconfiguratorSim, entry: String) -> LoopResult { Commands::Show => cmd_show(sim), Commands::Set(args) => cmd_set(sim, args), Commands::Load(args) => cmd_load(sim, args), + Commands::LoadExample(args) => cmd_load_example(sim, args), Commands::FileContents(args) => cmd_file_contents(args), Commands::Save(args) => cmd_save(sim, args), + Commands::Wipe => cmd_wipe(sim), }; match cmd_result { @@ -380,8 +430,12 @@ enum Commands { Save(SaveArgs), /// load state from a file Load(LoadArgs), + /// generate and load an example system + LoadExample(LoadExampleArgs), /// show information about what's in a saved file FileContents(FileContentsArgs), + /// reset the state of the REPL + Wipe, } #[derive(Debug, Args)] @@ -511,6 +565,33 @@ struct LoadArgs { collection_id: Option, } +#[derive(Debug, Args)] +struct LoadExampleArgs { + /// Seed for the RNG that's used to generate the example system. + /// + /// Setting this makes it possible for callers to get deterministic + /// results. In automated tests, the seed is typically the name of the + /// test. + #[clap(long, default_value = "reconfigurator_cli_example")] + seed: String, + + /// The number of sleds in the example system. + #[clap(short = 's', long, default_value_t = ExampleSystemBuilder::DEFAULT_N_SLEDS)] + nsleds: usize, + + /// The number of disks per sled in the example system. + #[clap(short = 'd', long, default_value_t = SledBuilder::DEFAULT_NPOOLS)] + ndisks_per_sled: u8, + + /// Do not create zones in the example system. + #[clap(short = 'Z', long)] + no_zones: bool, + + /// Do not create entries for disks in the blueprint. + #[clap(long)] + no_disks_in_blueprint: bool, +} + #[derive(Debug, Args)] struct FileContentsArgs { /// input file @@ -657,25 +738,16 @@ fn cmd_inventory_list( fn cmd_inventory_generate( sim: &mut ReconfiguratorSim, ) -> anyhow::Result> { - let mut builder = + let builder = sim.system.to_collection_builder().context("generating inventory")?; - // For an inventory we just generated from thin air, pretend like each sled - // has no zones on it. - let planning_input = - sim.system.to_planning_input_builder().unwrap().build(); - for sled_id in planning_input.all_sled_ids(SledFilter::Commissioned) { - builder - .found_sled_omicron_zones( - "fake sled agent", - sled_id, - OmicronZonesConfig { - generation: Generation::new(), - zones: vec![], - }, - ) - .context("recording Omicron zones")?; - } - let inventory = builder.build(); + + // sim.system carries around Omicron zones, which will make their way into + // the inventory. + let mut inventory = builder.build(); + // Assign collection IDs from the RNG. This enables consistent results when + // callers have explicitly seeded the RNG (e.g., in tests). + inventory.id = sim.collection_id_rng.next(); + let rv = format!( "generated inventory collection {} from configured sleds", inventory.id @@ -848,7 +920,7 @@ fn cmd_blueprint_diff( // Diff'ing DNS is a little trickier. First, compute what DNS should be for // each blueprint. To do that we need to construct a list of sleds suitable // for the executor. - let sleds_by_id = make_sleds_by_id(&sim)?; + let sleds_by_id = make_sleds_by_id(&sim.system)?; let internal_dns_config1 = blueprint_internal_dns_config( &blueprint1, &sleds_by_id, @@ -881,13 +953,9 @@ fn cmd_blueprint_diff( } fn make_sleds_by_id( - sim: &ReconfiguratorSim, -) -> Result< - BTreeMap, - anyhow::Error, -> { - let collection = sim - .system + system: &SystemDescription, +) -> Result, anyhow::Error> { + let collection = system .to_collection_builder() .context( "unexpectedly failed to create collection for current set of sleds", @@ -897,10 +965,10 @@ fn make_sleds_by_id( .sled_agents .iter() .map(|(sled_id, sled_agent_info)| { - let sled = nexus_reconfigurator_execution::Sled::new( + let sled = execution::Sled::new( *sled_id, sled_agent_info.sled_agent_address, - sled_agent_info.sled_role == SledRole::Scrimlet, + sled_agent_info.sled_role, ); (*sled_id, sled) }) @@ -927,7 +995,7 @@ fn cmd_blueprint_diff_dns( let blueprint_dns_zone = match dns_group { CliDnsGroup::Internal => { - let sleds_by_id = make_sleds_by_id(sim)?; + let sleds_by_id = make_sleds_by_id(&sim.system)?; blueprint_internal_dns_config( blueprint, &sleds_by_id, @@ -1006,6 +1074,11 @@ fn cmd_save( ))) } +fn cmd_wipe(sim: &mut ReconfiguratorSim) -> anyhow::Result> { + sim.wipe(); + Ok(Some("wiped reconfigurator-sim state".to_string())) +} + fn cmd_show(sim: &mut ReconfiguratorSim) -> anyhow::Result> { let mut s = String::new(); do_print_properties(&mut s, sim); @@ -1278,6 +1351,71 @@ fn cmd_load( Ok(Some(s)) } +fn cmd_load_example( + sim: &mut ReconfiguratorSim, + args: LoadExampleArgs, +) -> anyhow::Result> { + if sim.user_made_system_changes() { + bail!( + "changes made to simulated system: run `wipe system` before \ + loading an example system" + ); + } + + // Generate the example system. + let (example, blueprint) = ExampleSystemBuilder::new(&sim.log, &args.seed) + .nsleds(args.nsleds) + .ndisks_per_sled(args.ndisks_per_sled) + .nexus_count(sim.num_nexus.map_or(NEXUS_REDUNDANCY, |n| n.into())) + .create_zones(!args.no_zones) + .create_disks_in_blueprint(!args.no_disks_in_blueprint) + .build(); + + // Generate the internal and external DNS configs based on the blueprint. + let sleds_by_id = make_sleds_by_id(&example.system)?; + let internal_dns = blueprint_internal_dns_config( + &blueprint, + &sleds_by_id, + &Default::default(), + )?; + let external_dns = blueprint_external_dns_config( + &blueprint, + &sim.silo_names, + sim.external_dns_zone_name.clone(), + ); + + // No more fallible operations from here on out: set the system state. + let collection_id = example.collection.id; + let blueprint_id = blueprint.id; + sim.system = example.system; + sim.collections.insert(collection_id, example.collection); + sim.internal_dns.insert( + blueprint.internal_dns_version, + DnsConfigParams { + generation: blueprint.internal_dns_version.into(), + time_created: Utc::now(), + zones: vec![internal_dns], + }, + ); + sim.external_dns.insert( + blueprint.external_dns_version, + DnsConfigParams { + generation: blueprint.external_dns_version.into(), + time_created: Utc::now(), + zones: vec![external_dns], + }, + ); + sim.blueprints.insert(blueprint.id, blueprint); + sim.collection_id_rng = + TypedUuidRng::from_seed(&args.seed, "reconfigurator-cli"); + + Ok(Some(format!( + "loaded example system with:\n\ + - collection: {collection_id}\n\ + - blueprint: {blueprint_id}", + ))) +} + fn cmd_file_contents(args: FileContentsArgs) -> anyhow::Result> { let loaded = read_file(&args.filename)?; diff --git a/dev-tools/reconfigurator-cli/tests/input/cmds-example.txt b/dev-tools/reconfigurator-cli/tests/input/cmds-example.txt new file mode 100644 index 0000000000..b3143ac016 --- /dev/null +++ b/dev-tools/reconfigurator-cli/tests/input/cmds-example.txt @@ -0,0 +1,26 @@ +load-example --seed test-basic +load-example --seed test-basic + +show + +sled-list +inventory-list +blueprint-list + +sled-show 2eb69596-f081-4e2d-9425-9994926e0832 +blueprint-show ade5749d-bdf3-4fab-a8ae-00bea01b3a5a + +blueprint-diff-inventory 9e187896-7809-46d0-9210-d75be1b3c4d4 ade5749d-bdf3-4fab-a8ae-00bea01b3a5a + +inventory-generate +blueprint-diff-inventory b32394d8-7d79-486f-8657-fd5219508181 ade5749d-bdf3-4fab-a8ae-00bea01b3a5a + +wipe +load-example --seed test-basic --nsleds 1 --ndisks-per-sled 4 --no-zones + +sled-list +inventory-list +blueprint-list + +sled-show 89d02b1b-478c-401a-8e28-7a26f74fa41b +blueprint-show ade5749d-bdf3-4fab-a8ae-00bea01b3a5a diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-example-stderr b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stderr new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout new file mode 100644 index 0000000000..61b6da1f73 --- /dev/null +++ b/dev-tools/reconfigurator-cli/tests/output/cmd-example-stdout @@ -0,0 +1,538 @@ +> load-example --seed test-basic +loaded example system with: +- collection: 9e187896-7809-46d0-9210-d75be1b3c4d4 +- blueprint: ade5749d-bdf3-4fab-a8ae-00bea01b3a5a + +> load-example --seed test-basic +error: changes made to simulated system: run `wipe system` before loading an example system + +> + +> show +configured external DNS zone name: oxide.example +configured silo names: example-silo +internal DNS generations: 1 +external DNS generations: 1 +target number of Nexus instances: default + + +> + +> sled-list +ID NZPOOLS SUBNET +2eb69596-f081-4e2d-9425-9994926e0832 10 fd00:1122:3344:102::/64 +32d8d836-4d8a-4e54-8fa9-f31d79c42646 10 fd00:1122:3344:103::/64 +89d02b1b-478c-401a-8e28-7a26f74fa41b 10 fd00:1122:3344:101::/64 + +> inventory-list +ID NERRORS TIME_DONE +9e187896-7809-46d0-9210-d75be1b3c4d4 0 + +> blueprint-list +ID PARENT TIME_CREATED +ade5749d-bdf3-4fab-a8ae-00bea01b3a5a 02697f74-b14a-4418-90f0-c28b2a3a6aa9 + +> + +> sled-show 2eb69596-f081-4e2d-9425-9994926e0832 +sled 2eb69596-f081-4e2d-9425-9994926e0832 +subnet fd00:1122:3344:102::/64 +zpools (10): + 088ed702-551e-453b-80d7-57700372a844 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-088ed702-551e-453b-80d7-57700372a844" }, disk_id: b2850ccb-4ac7-4034-aeab-b1cd582d407b (physical_disk), policy: InService, state: Active } + 09e51697-abad-47c0-a193-eaf74bc5d3cd (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-09e51697-abad-47c0-a193-eaf74bc5d3cd" }, disk_id: c6d1fe0d-5226-4318-a55a-e86e20612277 (physical_disk), policy: InService, state: Active } + 3a512d49-edbe-47f3-8d0b-6051bfdc4044 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-3a512d49-edbe-47f3-8d0b-6051bfdc4044" }, disk_id: 24510d37-20b1-4bdc-9ca7-c37fff39abb2 (physical_disk), policy: InService, state: Active } + 40517680-aa77-413c-bcf4-b9041dcf6612 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-40517680-aa77-413c-bcf4-b9041dcf6612" }, disk_id: 30ed317f-1717-4df6-8c1c-69f9d438705e (physical_disk), policy: InService, state: Active } + 78d3cb96-9295-4644-bf78-2e32191c71f9 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-78d3cb96-9295-4644-bf78-2e32191c71f9" }, disk_id: 5ac39660-8149-48a2-a6df-aebb0f30352a (physical_disk), policy: InService, state: Active } + 853595e7-77da-404e-bc35-aba77478d55c (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-853595e7-77da-404e-bc35-aba77478d55c" }, disk_id: 43083372-c7d0-4df3-ac4e-96c45cde28d9 (physical_disk), policy: InService, state: Active } + 8926e0e7-65d9-4e2e-ac6d-f1298af81ef1 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8926e0e7-65d9-4e2e-ac6d-f1298af81ef1" }, disk_id: 13e65865-2a6e-41f7-aa18-6ef8dff59b4e (physical_disk), policy: InService, state: Active } + 9c0b9151-17f3-4857-94cc-b5bfcd402326 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-9c0b9151-17f3-4857-94cc-b5bfcd402326" }, disk_id: 40383e60-18f6-4423-94e7-7b91ce939b43 (physical_disk), policy: InService, state: Active } + d61354fa-48d2-47c6-90bf-546e3ed1708b (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d61354fa-48d2-47c6-90bf-546e3ed1708b" }, disk_id: e02ae523-7b66-4188-93c8-c5808c01c795 (physical_disk), policy: InService, state: Active } + d792c8cb-7490-40cb-bb1c-d4917242edf4 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-d792c8cb-7490-40cb-bb1c-d4917242edf4" }, disk_id: c19e5610-a3a2-4cc6-af4d-517a49ef610b (physical_disk), policy: InService, state: Active } + + +> blueprint-show ade5749d-bdf3-4fab-a8ae-00bea01b3a5a +blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a +parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 + + sled: 2eb69596-f081-4e2d-9425-9994926e0832 (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-088ed702-551e-453b-80d7-57700372a844 + fake-vendor fake-model serial-09e51697-abad-47c0-a193-eaf74bc5d3cd + fake-vendor fake-model serial-3a512d49-edbe-47f3-8d0b-6051bfdc4044 + fake-vendor fake-model serial-40517680-aa77-413c-bcf4-b9041dcf6612 + fake-vendor fake-model serial-78d3cb96-9295-4644-bf78-2e32191c71f9 + fake-vendor fake-model serial-853595e7-77da-404e-bc35-aba77478d55c + fake-vendor fake-model serial-8926e0e7-65d9-4e2e-ac6d-f1298af81ef1 + fake-vendor fake-model serial-9c0b9151-17f3-4857-94cc-b5bfcd402326 + fake-vendor fake-model serial-d61354fa-48d2-47c6-90bf-546e3ed1708b + fake-vendor fake-model serial-d792c8cb-7490-40cb-bb1c-d4917242edf4 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse fe79023f-c5d5-4be5-ad2c-da4e9e9237e4 in service fd00:1122:3344:102::23 + crucible 054f64a5-182c-4c28-8994-d2e082550201 in service fd00:1122:3344:102::26 + crucible 3b5bffea-e5ed-44df-8468-fd4fa69757d8 in service fd00:1122:3344:102::27 + crucible 53dd7fa4-899e-49ed-9fc2-48222db3e20d in service fd00:1122:3344:102::2a + crucible 7db307d4-a6ed-4c47-bddf-6759161bf64a in service fd00:1122:3344:102::2c + crucible 95ad9a1d-4063-4874-974c-2fc92830be27 in service fd00:1122:3344:102::29 + crucible bc095417-e2f0-4e95-b390-9cc3fc6e3c6d in service fd00:1122:3344:102::28 + crucible d90401f1-fbc2-42cb-bf17-309ee0f922fe in service fd00:1122:3344:102::2b + crucible e8f994c0-0a1b-40e6-8db1-40a8ca89e503 in service fd00:1122:3344:102::2d + crucible e9bf481e-323e-466e-842f-8107078c7137 in service fd00:1122:3344:102::2e + crucible f97aa057-6485-45d0-9cb4-4af5b0831d48 in service fd00:1122:3344:102::25 + crucible_pantry eaec16c0-0d44-4847-b2d6-31a5151bae52 in service fd00:1122:3344:102::24 + internal_dns 8b8f7c02-7a18-4268-b045-2e286b464c5d in service fd00:1122:3344:1::1 + internal_ntp c67dd9a4-0d6c-4e9f-b28d-20003f211f7d in service fd00:1122:3344:102::21 + nexus 94b45ce9-d3d8-413a-a76b-865da1f67930 in service fd00:1122:3344:102::22 + + + + sled: 32d8d836-4d8a-4e54-8fa9-f31d79c42646 (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-128b0f04-229b-48dc-9c5c-555cb5723ed8 + fake-vendor fake-model serial-43ae0f4e-b0cf-4d74-8636-df0567ba01e6 + fake-vendor fake-model serial-4e9806d0-41cd-48c2-86ef-7f815c3ce3b1 + fake-vendor fake-model serial-70bb6d98-111f-4015-9d97-9ef1b2d6dcac + fake-vendor fake-model serial-7ce5029f-703c-4c08-8164-9af9cf1acf23 + fake-vendor fake-model serial-b113c11f-44e6-4fb4-a56e-1d91bd652faf + fake-vendor fake-model serial-bf149c80-2498-481c-9989-6344da914081 + fake-vendor fake-model serial-c69b6237-09f9-45aa-962c-5dbdd1d894be + fake-vendor fake-model serial-ccd5a87b-00ae-42ad-85da-b37d70436cb1 + fake-vendor fake-model serial-d7410a1c-e01d-49a4-be9c-f861f086760a + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 09937ebb-bb6a-495b-bc97-b58076b70a78 in service fd00:1122:3344:103::2c + crucible a999e5fa-3edc-4dac-919a-d7b554cdae58 in service fd00:1122:3344:103::26 + crucible b416f299-c23c-46c8-9820-be2b66ffea0a in service fd00:1122:3344:103::27 + crucible b5d5491d-b3aa-4727-8b55-f66e0581ea4f in service fd00:1122:3344:103::2b + crucible cc1dc86d-bd6f-4929-aa4a-9619012e9393 in service fd00:1122:3344:103::24 + crucible cd3bb540-e605-465f-8c62-177ac482d850 in service fd00:1122:3344:103::29 + crucible e8971ab3-fb7d-4ad8-aae3-7f2fe87c51f3 in service fd00:1122:3344:103::25 + crucible f3628f0a-2301-4fc8-bcbf-961199771731 in service fd00:1122:3344:103::2d + crucible f52aa245-7e1b-46c0-8a31-e09725f02caf in service fd00:1122:3344:103::2a + crucible fae49024-6cec-444d-a6c4-83658ab015a4 in service fd00:1122:3344:103::28 + crucible_pantry 728db429-8621-4e1e-9915-282aadfa27d1 in service fd00:1122:3344:103::23 + internal_dns e7dd3e98-7fe7-4827-be7f-395ff9a5f542 in service fd00:1122:3344:2::1 + internal_ntp 4f2eb088-7d28-4c4e-a27c-746400ec65ba in service fd00:1122:3344:103::21 + nexus c8aa84a5-a802-46c9-adcd-d61e9c8393c9 in service fd00:1122:3344:103::22 + + + + sled: 89d02b1b-478c-401a-8e28-7a26f74fa41b (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8 + fake-vendor fake-model serial-5265edc6-debf-4687-a758-a9746893ebd3 + fake-vendor fake-model serial-532fbd69-b472-4445-86af-4c4c85afb313 + fake-vendor fake-model serial-54fd6fa6-ce3c-4abe-8c9d-7e107e159e84 + fake-vendor fake-model serial-8562317c-4736-4cfc-9292-7dcab96a6fee + fake-vendor fake-model serial-9a1327e4-d11b-4d98-8454-8c41862e9832 + fake-vendor fake-model serial-bf9d6692-64bc-459a-87dd-e7a83080a210 + fake-vendor fake-model serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6 + fake-vendor fake-model serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c + fake-vendor fake-model serial-fe1d5b9f-8db7-4e2d-bf17-c4b80e1f897c + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 413d3e02-e19f-400a-9718-a662347538f0 in service fd00:1122:3344:101::24 + crucible 6cb330f9-4609-4d6c-98ad-b5cc34245813 in service fd00:1122:3344:101::29 + crucible 6d725df0-0189-4429-b270-3eeb891d39c8 in service fd00:1122:3344:101::28 + crucible b5443ebd-1f5b-448c-8edc-b4ca25c25db1 in service fd00:1122:3344:101::25 + crucible bb55534c-1042-4af4-ad2f-9590803695ac in service fd00:1122:3344:101::27 + crucible c4296f9f-f902-4fc7-b896-178e56e60732 in service fd00:1122:3344:101::2d + crucible d14c165f-6370-4cce-9dba-3c6deb762cfc in service fd00:1122:3344:101::2c + crucible de65f128-30f7-422b-a234-d1fc8dd6ef78 in service fd00:1122:3344:101::2b + crucible e135441d-637e-4de9-8023-5ea0096347f3 in service fd00:1122:3344:101::26 + crucible fee71ee6-da42-4a7f-a00e-f56b6a3327ce in service fd00:1122:3344:101::2a + crucible_pantry 315a3670-d019-425c-b7a6-c9429428b671 in service fd00:1122:3344:101::23 + internal_dns 8b47e1e8-0396-4e44-a4a5-ea891405c9f2 in service fd00:1122:3344:3::1 + internal_ntp cbe91cdc-cbb6-4760-aece-6ce08b67e85a in service fd00:1122:3344:101::21 + nexus b43ce109-90d6-46f9-9df0-8c68bfe6d4a0 in service fd00:1122:3344:101::22 + + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) + cluster.preserve_downgrade_option: (do not modify) + + METADATA: + created by::::::::::: test suite + created at::::::::::: + comment:::::::::::::: (none) + internal DNS version: 1 + external DNS version: 1 + + + +> + +> blueprint-diff-inventory 9e187896-7809-46d0-9210-d75be1b3c4d4 ade5749d-bdf3-4fab-a8ae-00bea01b3a5a +from: collection 9e187896-7809-46d0-9210-d75be1b3c4d4 +to: blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a + UNCHANGED SLEDS: + + sled 2eb69596-f081-4e2d-9425-9994926e0832 (active): + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-088ed702-551e-453b-80d7-57700372a844 + fake-vendor fake-model serial-09e51697-abad-47c0-a193-eaf74bc5d3cd + fake-vendor fake-model serial-3a512d49-edbe-47f3-8d0b-6051bfdc4044 + fake-vendor fake-model serial-40517680-aa77-413c-bcf4-b9041dcf6612 + fake-vendor fake-model serial-78d3cb96-9295-4644-bf78-2e32191c71f9 + fake-vendor fake-model serial-853595e7-77da-404e-bc35-aba77478d55c + fake-vendor fake-model serial-8926e0e7-65d9-4e2e-ac6d-f1298af81ef1 + fake-vendor fake-model serial-9c0b9151-17f3-4857-94cc-b5bfcd402326 + fake-vendor fake-model serial-d61354fa-48d2-47c6-90bf-546e3ed1708b + fake-vendor fake-model serial-d792c8cb-7490-40cb-bb1c-d4917242edf4 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse fe79023f-c5d5-4be5-ad2c-da4e9e9237e4 in service fd00:1122:3344:102::23 + crucible 054f64a5-182c-4c28-8994-d2e082550201 in service fd00:1122:3344:102::26 + crucible 3b5bffea-e5ed-44df-8468-fd4fa69757d8 in service fd00:1122:3344:102::27 + crucible 53dd7fa4-899e-49ed-9fc2-48222db3e20d in service fd00:1122:3344:102::2a + crucible 7db307d4-a6ed-4c47-bddf-6759161bf64a in service fd00:1122:3344:102::2c + crucible 95ad9a1d-4063-4874-974c-2fc92830be27 in service fd00:1122:3344:102::29 + crucible bc095417-e2f0-4e95-b390-9cc3fc6e3c6d in service fd00:1122:3344:102::28 + crucible d90401f1-fbc2-42cb-bf17-309ee0f922fe in service fd00:1122:3344:102::2b + crucible e8f994c0-0a1b-40e6-8db1-40a8ca89e503 in service fd00:1122:3344:102::2d + crucible e9bf481e-323e-466e-842f-8107078c7137 in service fd00:1122:3344:102::2e + crucible f97aa057-6485-45d0-9cb4-4af5b0831d48 in service fd00:1122:3344:102::25 + crucible_pantry eaec16c0-0d44-4847-b2d6-31a5151bae52 in service fd00:1122:3344:102::24 + internal_dns 8b8f7c02-7a18-4268-b045-2e286b464c5d in service fd00:1122:3344:1::1 + internal_ntp c67dd9a4-0d6c-4e9f-b28d-20003f211f7d in service fd00:1122:3344:102::21 + nexus 94b45ce9-d3d8-413a-a76b-865da1f67930 in service fd00:1122:3344:102::22 + + + sled 32d8d836-4d8a-4e54-8fa9-f31d79c42646 (active): + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-128b0f04-229b-48dc-9c5c-555cb5723ed8 + fake-vendor fake-model serial-43ae0f4e-b0cf-4d74-8636-df0567ba01e6 + fake-vendor fake-model serial-4e9806d0-41cd-48c2-86ef-7f815c3ce3b1 + fake-vendor fake-model serial-70bb6d98-111f-4015-9d97-9ef1b2d6dcac + fake-vendor fake-model serial-7ce5029f-703c-4c08-8164-9af9cf1acf23 + fake-vendor fake-model serial-b113c11f-44e6-4fb4-a56e-1d91bd652faf + fake-vendor fake-model serial-bf149c80-2498-481c-9989-6344da914081 + fake-vendor fake-model serial-c69b6237-09f9-45aa-962c-5dbdd1d894be + fake-vendor fake-model serial-ccd5a87b-00ae-42ad-85da-b37d70436cb1 + fake-vendor fake-model serial-d7410a1c-e01d-49a4-be9c-f861f086760a + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 09937ebb-bb6a-495b-bc97-b58076b70a78 in service fd00:1122:3344:103::2c + crucible a999e5fa-3edc-4dac-919a-d7b554cdae58 in service fd00:1122:3344:103::26 + crucible b416f299-c23c-46c8-9820-be2b66ffea0a in service fd00:1122:3344:103::27 + crucible b5d5491d-b3aa-4727-8b55-f66e0581ea4f in service fd00:1122:3344:103::2b + crucible cc1dc86d-bd6f-4929-aa4a-9619012e9393 in service fd00:1122:3344:103::24 + crucible cd3bb540-e605-465f-8c62-177ac482d850 in service fd00:1122:3344:103::29 + crucible e8971ab3-fb7d-4ad8-aae3-7f2fe87c51f3 in service fd00:1122:3344:103::25 + crucible f3628f0a-2301-4fc8-bcbf-961199771731 in service fd00:1122:3344:103::2d + crucible f52aa245-7e1b-46c0-8a31-e09725f02caf in service fd00:1122:3344:103::2a + crucible fae49024-6cec-444d-a6c4-83658ab015a4 in service fd00:1122:3344:103::28 + crucible_pantry 728db429-8621-4e1e-9915-282aadfa27d1 in service fd00:1122:3344:103::23 + internal_dns e7dd3e98-7fe7-4827-be7f-395ff9a5f542 in service fd00:1122:3344:2::1 + internal_ntp 4f2eb088-7d28-4c4e-a27c-746400ec65ba in service fd00:1122:3344:103::21 + nexus c8aa84a5-a802-46c9-adcd-d61e9c8393c9 in service fd00:1122:3344:103::22 + + + sled 89d02b1b-478c-401a-8e28-7a26f74fa41b (active): + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8 + fake-vendor fake-model serial-5265edc6-debf-4687-a758-a9746893ebd3 + fake-vendor fake-model serial-532fbd69-b472-4445-86af-4c4c85afb313 + fake-vendor fake-model serial-54fd6fa6-ce3c-4abe-8c9d-7e107e159e84 + fake-vendor fake-model serial-8562317c-4736-4cfc-9292-7dcab96a6fee + fake-vendor fake-model serial-9a1327e4-d11b-4d98-8454-8c41862e9832 + fake-vendor fake-model serial-bf9d6692-64bc-459a-87dd-e7a83080a210 + fake-vendor fake-model serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6 + fake-vendor fake-model serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c + fake-vendor fake-model serial-fe1d5b9f-8db7-4e2d-bf17-c4b80e1f897c + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 413d3e02-e19f-400a-9718-a662347538f0 in service fd00:1122:3344:101::24 + crucible 6cb330f9-4609-4d6c-98ad-b5cc34245813 in service fd00:1122:3344:101::29 + crucible 6d725df0-0189-4429-b270-3eeb891d39c8 in service fd00:1122:3344:101::28 + crucible b5443ebd-1f5b-448c-8edc-b4ca25c25db1 in service fd00:1122:3344:101::25 + crucible bb55534c-1042-4af4-ad2f-9590803695ac in service fd00:1122:3344:101::27 + crucible c4296f9f-f902-4fc7-b896-178e56e60732 in service fd00:1122:3344:101::2d + crucible d14c165f-6370-4cce-9dba-3c6deb762cfc in service fd00:1122:3344:101::2c + crucible de65f128-30f7-422b-a234-d1fc8dd6ef78 in service fd00:1122:3344:101::2b + crucible e135441d-637e-4de9-8023-5ea0096347f3 in service fd00:1122:3344:101::26 + crucible fee71ee6-da42-4a7f-a00e-f56b6a3327ce in service fd00:1122:3344:101::2a + crucible_pantry 315a3670-d019-425c-b7a6-c9429428b671 in service fd00:1122:3344:101::23 + internal_dns 8b47e1e8-0396-4e44-a4a5-ea891405c9f2 in service fd00:1122:3344:3::1 + internal_ntp cbe91cdc-cbb6-4760-aece-6ce08b67e85a in service fd00:1122:3344:101::21 + nexus b43ce109-90d6-46f9-9df0-8c68bfe6d4a0 in service fd00:1122:3344:101::22 + + + COCKROACHDB SETTINGS: ++ state fingerprint::::::::::::::::: (not present in collection) -> (none) ++ cluster.preserve_downgrade_option: (not present in collection) -> (do not modify) + + METADATA: ++ internal DNS version: (not present in collection) -> 1 ++ external DNS version: (not present in collection) -> 1 + + + +> + +> inventory-generate +generated inventory collection b32394d8-7d79-486f-8657-fd5219508181 from configured sleds + +> blueprint-diff-inventory b32394d8-7d79-486f-8657-fd5219508181 ade5749d-bdf3-4fab-a8ae-00bea01b3a5a +from: collection b32394d8-7d79-486f-8657-fd5219508181 +to: blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a + UNCHANGED SLEDS: + + sled 2eb69596-f081-4e2d-9425-9994926e0832 (active): + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-088ed702-551e-453b-80d7-57700372a844 + fake-vendor fake-model serial-09e51697-abad-47c0-a193-eaf74bc5d3cd + fake-vendor fake-model serial-3a512d49-edbe-47f3-8d0b-6051bfdc4044 + fake-vendor fake-model serial-40517680-aa77-413c-bcf4-b9041dcf6612 + fake-vendor fake-model serial-78d3cb96-9295-4644-bf78-2e32191c71f9 + fake-vendor fake-model serial-853595e7-77da-404e-bc35-aba77478d55c + fake-vendor fake-model serial-8926e0e7-65d9-4e2e-ac6d-f1298af81ef1 + fake-vendor fake-model serial-9c0b9151-17f3-4857-94cc-b5bfcd402326 + fake-vendor fake-model serial-d61354fa-48d2-47c6-90bf-546e3ed1708b + fake-vendor fake-model serial-d792c8cb-7490-40cb-bb1c-d4917242edf4 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse fe79023f-c5d5-4be5-ad2c-da4e9e9237e4 in service fd00:1122:3344:102::23 + crucible 054f64a5-182c-4c28-8994-d2e082550201 in service fd00:1122:3344:102::26 + crucible 3b5bffea-e5ed-44df-8468-fd4fa69757d8 in service fd00:1122:3344:102::27 + crucible 53dd7fa4-899e-49ed-9fc2-48222db3e20d in service fd00:1122:3344:102::2a + crucible 7db307d4-a6ed-4c47-bddf-6759161bf64a in service fd00:1122:3344:102::2c + crucible 95ad9a1d-4063-4874-974c-2fc92830be27 in service fd00:1122:3344:102::29 + crucible bc095417-e2f0-4e95-b390-9cc3fc6e3c6d in service fd00:1122:3344:102::28 + crucible d90401f1-fbc2-42cb-bf17-309ee0f922fe in service fd00:1122:3344:102::2b + crucible e8f994c0-0a1b-40e6-8db1-40a8ca89e503 in service fd00:1122:3344:102::2d + crucible e9bf481e-323e-466e-842f-8107078c7137 in service fd00:1122:3344:102::2e + crucible f97aa057-6485-45d0-9cb4-4af5b0831d48 in service fd00:1122:3344:102::25 + crucible_pantry eaec16c0-0d44-4847-b2d6-31a5151bae52 in service fd00:1122:3344:102::24 + internal_dns 8b8f7c02-7a18-4268-b045-2e286b464c5d in service fd00:1122:3344:1::1 + internal_ntp c67dd9a4-0d6c-4e9f-b28d-20003f211f7d in service fd00:1122:3344:102::21 + nexus 94b45ce9-d3d8-413a-a76b-865da1f67930 in service fd00:1122:3344:102::22 + + + sled 32d8d836-4d8a-4e54-8fa9-f31d79c42646 (active): + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-128b0f04-229b-48dc-9c5c-555cb5723ed8 + fake-vendor fake-model serial-43ae0f4e-b0cf-4d74-8636-df0567ba01e6 + fake-vendor fake-model serial-4e9806d0-41cd-48c2-86ef-7f815c3ce3b1 + fake-vendor fake-model serial-70bb6d98-111f-4015-9d97-9ef1b2d6dcac + fake-vendor fake-model serial-7ce5029f-703c-4c08-8164-9af9cf1acf23 + fake-vendor fake-model serial-b113c11f-44e6-4fb4-a56e-1d91bd652faf + fake-vendor fake-model serial-bf149c80-2498-481c-9989-6344da914081 + fake-vendor fake-model serial-c69b6237-09f9-45aa-962c-5dbdd1d894be + fake-vendor fake-model serial-ccd5a87b-00ae-42ad-85da-b37d70436cb1 + fake-vendor fake-model serial-d7410a1c-e01d-49a4-be9c-f861f086760a + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 09937ebb-bb6a-495b-bc97-b58076b70a78 in service fd00:1122:3344:103::2c + crucible a999e5fa-3edc-4dac-919a-d7b554cdae58 in service fd00:1122:3344:103::26 + crucible b416f299-c23c-46c8-9820-be2b66ffea0a in service fd00:1122:3344:103::27 + crucible b5d5491d-b3aa-4727-8b55-f66e0581ea4f in service fd00:1122:3344:103::2b + crucible cc1dc86d-bd6f-4929-aa4a-9619012e9393 in service fd00:1122:3344:103::24 + crucible cd3bb540-e605-465f-8c62-177ac482d850 in service fd00:1122:3344:103::29 + crucible e8971ab3-fb7d-4ad8-aae3-7f2fe87c51f3 in service fd00:1122:3344:103::25 + crucible f3628f0a-2301-4fc8-bcbf-961199771731 in service fd00:1122:3344:103::2d + crucible f52aa245-7e1b-46c0-8a31-e09725f02caf in service fd00:1122:3344:103::2a + crucible fae49024-6cec-444d-a6c4-83658ab015a4 in service fd00:1122:3344:103::28 + crucible_pantry 728db429-8621-4e1e-9915-282aadfa27d1 in service fd00:1122:3344:103::23 + internal_dns e7dd3e98-7fe7-4827-be7f-395ff9a5f542 in service fd00:1122:3344:2::1 + internal_ntp 4f2eb088-7d28-4c4e-a27c-746400ec65ba in service fd00:1122:3344:103::21 + nexus c8aa84a5-a802-46c9-adcd-d61e9c8393c9 in service fd00:1122:3344:103::22 + + + sled 89d02b1b-478c-401a-8e28-7a26f74fa41b (active): + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8 + fake-vendor fake-model serial-5265edc6-debf-4687-a758-a9746893ebd3 + fake-vendor fake-model serial-532fbd69-b472-4445-86af-4c4c85afb313 + fake-vendor fake-model serial-54fd6fa6-ce3c-4abe-8c9d-7e107e159e84 + fake-vendor fake-model serial-8562317c-4736-4cfc-9292-7dcab96a6fee + fake-vendor fake-model serial-9a1327e4-d11b-4d98-8454-8c41862e9832 + fake-vendor fake-model serial-bf9d6692-64bc-459a-87dd-e7a83080a210 + fake-vendor fake-model serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6 + fake-vendor fake-model serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c + fake-vendor fake-model serial-fe1d5b9f-8db7-4e2d-bf17-c4b80e1f897c + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 413d3e02-e19f-400a-9718-a662347538f0 in service fd00:1122:3344:101::24 + crucible 6cb330f9-4609-4d6c-98ad-b5cc34245813 in service fd00:1122:3344:101::29 + crucible 6d725df0-0189-4429-b270-3eeb891d39c8 in service fd00:1122:3344:101::28 + crucible b5443ebd-1f5b-448c-8edc-b4ca25c25db1 in service fd00:1122:3344:101::25 + crucible bb55534c-1042-4af4-ad2f-9590803695ac in service fd00:1122:3344:101::27 + crucible c4296f9f-f902-4fc7-b896-178e56e60732 in service fd00:1122:3344:101::2d + crucible d14c165f-6370-4cce-9dba-3c6deb762cfc in service fd00:1122:3344:101::2c + crucible de65f128-30f7-422b-a234-d1fc8dd6ef78 in service fd00:1122:3344:101::2b + crucible e135441d-637e-4de9-8023-5ea0096347f3 in service fd00:1122:3344:101::26 + crucible fee71ee6-da42-4a7f-a00e-f56b6a3327ce in service fd00:1122:3344:101::2a + crucible_pantry 315a3670-d019-425c-b7a6-c9429428b671 in service fd00:1122:3344:101::23 + internal_dns 8b47e1e8-0396-4e44-a4a5-ea891405c9f2 in service fd00:1122:3344:3::1 + internal_ntp cbe91cdc-cbb6-4760-aece-6ce08b67e85a in service fd00:1122:3344:101::21 + nexus b43ce109-90d6-46f9-9df0-8c68bfe6d4a0 in service fd00:1122:3344:101::22 + + + COCKROACHDB SETTINGS: ++ state fingerprint::::::::::::::::: (not present in collection) -> (none) ++ cluster.preserve_downgrade_option: (not present in collection) -> (do not modify) + + METADATA: ++ internal DNS version: (not present in collection) -> 1 ++ external DNS version: (not present in collection) -> 1 + + + +> + +> wipe +wiped reconfigurator-sim state + +> load-example --seed test-basic --nsleds 1 --ndisks-per-sled 4 --no-zones +loaded example system with: +- collection: 9e187896-7809-46d0-9210-d75be1b3c4d4 +- blueprint: ade5749d-bdf3-4fab-a8ae-00bea01b3a5a + +> + +> sled-list +ID NZPOOLS SUBNET +89d02b1b-478c-401a-8e28-7a26f74fa41b 4 fd00:1122:3344:101::/64 + +> inventory-list +ID NERRORS TIME_DONE +9e187896-7809-46d0-9210-d75be1b3c4d4 0 + +> blueprint-list +ID PARENT TIME_CREATED +ade5749d-bdf3-4fab-a8ae-00bea01b3a5a 02697f74-b14a-4418-90f0-c28b2a3a6aa9 + +> + +> sled-show 89d02b1b-478c-401a-8e28-7a26f74fa41b +sled 89d02b1b-478c-401a-8e28-7a26f74fa41b +subnet fd00:1122:3344:101::/64 +zpools (4): + 44fa7024-c2bc-4d2c-b478-c4997e4aece8 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8" }, disk_id: 2a15b33c-dd0e-45b7-aba9-d05f40f030ff (physical_disk), policy: InService, state: Active } + 8562317c-4736-4cfc-9292-7dcab96a6fee (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-8562317c-4736-4cfc-9292-7dcab96a6fee" }, disk_id: cad6faa6-9409-4496-9aeb-392b3c50bed4 (physical_disk), policy: InService, state: Active } + ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6 (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6" }, disk_id: 7d89a66e-0dcd-47ab-824d-62186812b8bd (physical_disk), policy: InService, state: Active } + f931ec80-a3e3-4adb-a8ba-fa5adbd2294c (zpool) + ↳ SledDisk { disk_identity: DiskIdentity { vendor: "fake-vendor", model: "fake-model", serial: "serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c" }, disk_id: 41755be9-2c77-4deb-87a4-cb53f09263fa (physical_disk), policy: InService, state: Active } + + +> blueprint-show ade5749d-bdf3-4fab-a8ae-00bea01b3a5a +blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a +parent: 02697f74-b14a-4418-90f0-c28b2a3a6aa9 + + sled: 89d02b1b-478c-401a-8e28-7a26f74fa41b (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-44fa7024-c2bc-4d2c-b478-c4997e4aece8 + fake-vendor fake-model serial-8562317c-4736-4cfc-9292-7dcab96a6fee + fake-vendor fake-model serial-ce1c13f3-bef2-4306-b0f2-4e39bd4a18b6 + fake-vendor fake-model serial-f931ec80-a3e3-4adb-a8ba-fa5adbd2294c + + + omicron zones at generation 1: + ----------------------------------------------- + zone type zone id disposition underlay IP + ----------------------------------------------- + + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) + cluster.preserve_downgrade_option: (do not modify) + + METADATA: + created by::::::::::: test suite + created at::::::::::: + comment:::::::::::::: (none) + internal DNS version: 1 + external DNS version: 1 + + + diff --git a/dev-tools/reconfigurator-cli/tests/test_basic.rs b/dev-tools/reconfigurator-cli/tests/test_basic.rs index 2f45158d30..d451996e0b 100644 --- a/dev-tools/reconfigurator-cli/tests/test_basic.rs +++ b/dev-tools/reconfigurator-cli/tests/test_basic.rs @@ -19,8 +19,8 @@ use omicron_test_utils::dev::poll::wait_for_condition; use omicron_test_utils::dev::poll::CondCheckError; use omicron_test_utils::dev::test_cmds::assert_exit_code; use omicron_test_utils::dev::test_cmds::path_to_executable; -use omicron_test_utils::dev::test_cmds::redact_variable; use omicron_test_utils::dev::test_cmds::run_command; +use omicron_test_utils::dev::test_cmds::Redactor; use omicron_test_utils::dev::test_cmds::EXIT_SUCCESS; use omicron_uuid_kinds::SledUuid; use slog::debug; @@ -43,11 +43,26 @@ fn test_basic() { let exec = Exec::cmd(path_to_cli()).arg("tests/input/cmds.txt"); let (exit_status, stdout_text, stderr_text) = run_command(exec); assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text); - let stdout_text = redact_variable(&stdout_text); + let stdout_text = Redactor::default().do_redact(&stdout_text); assert_contents("tests/output/cmd-stdout", &stdout_text); assert_contents("tests/output/cmd-stderr", &stderr_text); } +// Run tests against a loaded example system. +#[test] +fn test_example() { + let exec = Exec::cmd(path_to_cli()).arg("tests/input/cmds-example.txt"); + let (exit_status, stdout_text, stderr_text) = run_command(exec); + assert_exit_code(exit_status, EXIT_SUCCESS, &stderr_text); + + // The example system uses a fixed seed, which means that UUIDs are + // deterministic. Some of the test commands also use those UUIDs, and it's + // convenient for everyone if they aren't redacted. + let stdout_text = Redactor::default().uuids(false).do_redact(&stdout_text); + assert_contents("tests/output/cmd-example-stdout", &stdout_text); + assert_contents("tests/output/cmd-example-stderr", &stderr_text); +} + type ControlPlaneTestContext = nexus_test_utils::ControlPlaneTestContext; diff --git a/dev-tools/releng/src/main.rs b/dev-tools/releng/src/main.rs index 7cf5459088..9f95344862 100644 --- a/dev-tools/releng/src/main.rs +++ b/dev-tools/releng/src/main.rs @@ -41,7 +41,7 @@ use crate::job::Jobs; /// to as "v8", "version 8", or "release 8" to customers). The use of semantic /// versioning is mostly to hedge for perhaps wanting something more granular in /// the future. -const BASE_VERSION: Version = Version::new(11, 0, 0); +const BASE_VERSION: Version = Version::new(12, 0, 0); const RETRY_ATTEMPTS: usize = 3; diff --git a/dns-server-api/Cargo.toml b/dns-server-api/Cargo.toml index c87af14e0d..dfa384763e 100644 --- a/dns-server-api/Cargo.toml +++ b/dns-server-api/Cargo.toml @@ -10,6 +10,7 @@ workspace = true [dependencies] chrono.workspace = true dropshot.workspace = true +internal-dns-types.workspace = true omicron-workspace-hack.workspace = true schemars.workspace = true serde.workspace = true diff --git a/dns-server-api/src/lib.rs b/dns-server-api/src/lib.rs index 2c59caf0c5..8449293e5f 100644 --- a/dns-server-api/src/lib.rs +++ b/dns-server-api/src/lib.rs @@ -89,14 +89,8 @@ //! in-progress one. How large do we allow that queue to grow? At some point //! we'll need to stop queueing them. So why bother at all? -use std::{ - collections::HashMap, - net::{Ipv4Addr, Ipv6Addr}, -}; - use dropshot::{HttpError, HttpResponseOk, RequestContext}; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; +use internal_dns_types::config::{DnsConfig, DnsConfigParams}; #[dropshot::api_description] pub trait DnsServerApi { @@ -119,42 +113,3 @@ pub trait DnsServerApi { rq: dropshot::TypedBody, ) -> Result; } - -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] -pub struct DnsConfigParams { - pub generation: u64, - pub time_created: chrono::DateTime, - pub zones: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] -pub struct DnsConfig { - pub generation: u64, - pub time_created: chrono::DateTime, - pub time_applied: chrono::DateTime, - pub zones: Vec, -} - -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] -pub struct DnsConfigZone { - pub zone_name: String, - pub records: HashMap>, -} - -#[allow(clippy::upper_case_acronyms)] -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] -#[serde(tag = "type", content = "data")] -pub enum DnsRecord { - A(Ipv4Addr), - AAAA(Ipv6Addr), - SRV(SRV), -} - -#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq)] -#[serde(rename = "Srv")] -pub struct SRV { - pub prio: u16, - pub weight: u16, - pub port: u16, - pub target: String, -} diff --git a/dns-server/Cargo.toml b/dns-server/Cargo.toml index b4516b8b77..b3e7839162 100644 --- a/dns-server/Cargo.toml +++ b/dns-server/Cargo.toml @@ -20,6 +20,7 @@ hickory-proto.workspace = true hickory-resolver.workspace = true hickory-server.workspace = true http.workspace = true +internal-dns-types.workspace = true pretty-hex.workspace = true schemars.workspace = true serde.workspace = true diff --git a/dns-server/src/bin/dnsadm.rs b/dns-server/src/bin/dnsadm.rs index 76ba9bc2d4..1c6a446124 100644 --- a/dns-server/src/bin/dnsadm.rs +++ b/dns-server/src/bin/dnsadm.rs @@ -16,11 +16,12 @@ use anyhow::ensure; use anyhow::Context; use anyhow::Result; use clap::{Args, Parser, Subcommand}; -use dns_service_client::types::DnsConfig; -use dns_service_client::{ - types::{DnsConfigParams, DnsConfigZone, DnsRecord, Srv}, - Client, -}; +use dns_service_client::Client; +use internal_dns_types::config::DnsConfig; +use internal_dns_types::config::DnsConfigParams; +use internal_dns_types::config::DnsConfigZone; +use internal_dns_types::config::DnsRecord; +use internal_dns_types::config::Srv; use slog::{Drain, Logger}; use std::collections::BTreeMap; use std::collections::HashMap; diff --git a/dns-server/src/dns_server.rs b/dns-server/src/dns_server.rs index 4ecbe382c8..34750719c1 100644 --- a/dns-server/src/dns_server.rs +++ b/dns-server/src/dns_server.rs @@ -12,7 +12,6 @@ use crate::storage::QueryError; use crate::storage::Store; use anyhow::anyhow; use anyhow::Context; -use dns_server_api::DnsRecord; use hickory_proto::op::Header; use hickory_proto::op::ResponseCode; use hickory_proto::rr::rdata::SRV; @@ -26,6 +25,8 @@ use hickory_resolver::Name; use hickory_server::authority::MessageRequest; use hickory_server::authority::MessageResponse; use hickory_server::authority::MessageResponseBuilder; +use internal_dns_types::config::DnsRecord; +use internal_dns_types::config::Srv; use pretty_hex::*; use serde::Deserialize; use slog::{debug, error, info, o, trace, Logger}; @@ -231,7 +232,7 @@ fn dns_record_to_record( Ok(a) } - DnsRecord::AAAA(addr) => { + DnsRecord::Aaaa(addr) => { let mut aaaa = Record::new(); aaaa.set_name(name.clone()) .set_rr_type(RecordType::AAAA) @@ -239,7 +240,7 @@ fn dns_record_to_record( Ok(aaaa) } - DnsRecord::SRV(dns_server_api::SRV { prio, weight, port, target }) => { + DnsRecord::Srv(Srv { prio, weight, port, target }) => { let tgt = Name::from_str(&target).map_err(|error| { RequestError::ServFail(anyhow!( "serialization failed due to bad SRV target {:?}: {:#}", diff --git a/dns-server/src/http_server.rs b/dns-server/src/http_server.rs index f9f56d9326..87d576258f 100644 --- a/dns-server/src/http_server.rs +++ b/dns-server/src/http_server.rs @@ -5,11 +5,12 @@ //! Dropshot server for configuring DNS namespace use crate::storage::{self, UpdateError}; -use dns_server_api::{DnsConfig, DnsConfigParams, DnsServerApi}; +use dns_server_api::DnsServerApi; use dns_service_client::{ ERROR_CODE_BAD_UPDATE_GENERATION, ERROR_CODE_UPDATE_IN_PROGRESS, }; use dropshot::RequestContext; +use internal_dns_types::config::{DnsConfig, DnsConfigParams}; pub struct Context { store: storage::Store, diff --git a/dns-server/src/lib.rs b/dns-server/src/lib.rs index 8abd3b945e..88549e9982 100644 --- a/dns-server/src/lib.rs +++ b/dns-server/src/lib.rs @@ -52,6 +52,7 @@ use hickory_resolver::config::Protocol; use hickory_resolver::config::ResolverConfig; use hickory_resolver::config::ResolverOpts; use hickory_resolver::TokioAsyncResolver; +use internal_dns_types::config::DnsConfigParams; use slog::o; use std::net::SocketAddr; @@ -148,7 +149,7 @@ impl TransientServer { pub async fn initialize_with_config( &self, log: &slog::Logger, - dns_config: &dns_service_client::types::DnsConfigParams, + dns_config: &DnsConfigParams, ) -> Result<(), anyhow::Error> { let dns_config_client = dns_service_client::Client::new( &format!("http://{}", self.dropshot_server.local_addr()), diff --git a/dns-server/src/storage.rs b/dns-server/src/storage.rs index b3141f6751..6c58af4978 100644 --- a/dns-server/src/storage.rs +++ b/dns-server/src/storage.rs @@ -94,9 +94,11 @@ use anyhow::{anyhow, Context}; use camino::Utf8PathBuf; -use dns_server_api::{DnsConfig, DnsConfigParams, DnsConfigZone, DnsRecord}; use hickory_proto::rr::LowerName; use hickory_resolver::Name; +use internal_dns_types::config::{ + DnsConfig, DnsConfigParams, DnsConfigZone, DnsRecord, +}; use serde::{Deserialize, Serialize}; use sled::transaction::ConflictableTransactionError; use slog::{debug, error, info, o, warn}; @@ -781,11 +783,11 @@ mod test { use anyhow::Context; use camino::Utf8PathBuf; use camino_tempfile::Utf8TempDir; - use dns_server_api::DnsConfigParams; - use dns_server_api::DnsConfigZone; - use dns_server_api::DnsRecord; use hickory_proto::rr::LowerName; use hickory_resolver::Name; + use internal_dns_types::config::DnsConfigParams; + use internal_dns_types::config::DnsConfigZone; + use internal_dns_types::config::DnsRecord; use omicron_test_utils::dev::test_setup_log; use std::collections::BTreeSet; use std::collections::HashMap; @@ -897,7 +899,7 @@ mod test { expect(&tc.store, "gen8_name.zone8.internal", Expect::NoZone); // Update to generation 1, which contains one zone with one name. - let dummy_record = DnsRecord::AAAA(Ipv6Addr::LOCALHOST); + let dummy_record = DnsRecord::Aaaa(Ipv6Addr::LOCALHOST); let update1 = DnsConfigParams { time_created: chrono::Utc::now(), generation: 1, @@ -1066,7 +1068,7 @@ mod test { assert!(config.zones.is_empty()); // Make one normal update. - let dummy_record = DnsRecord::AAAA(Ipv6Addr::LOCALHOST); + let dummy_record = DnsRecord::Aaaa(Ipv6Addr::LOCALHOST); let update1 = DnsConfigParams { time_created: chrono::Utc::now(), generation: 1, @@ -1188,7 +1190,7 @@ mod test { let after = chrono::Utc::now(); // Concurrently attempt another update. - let dummy_record = DnsRecord::AAAA(Ipv6Addr::LOCALHOST); + let dummy_record = DnsRecord::Aaaa(Ipv6Addr::LOCALHOST); let update2 = DnsConfigParams { time_created: chrono::Utc::now(), generation: 1, diff --git a/dns-server/tests/basic_test.rs b/dns-server/tests/basic_test.rs index fa5bfea468..c72bb4b3ac 100644 --- a/dns-server/tests/basic_test.rs +++ b/dns-server/tests/basic_test.rs @@ -4,10 +4,7 @@ use anyhow::{Context, Result}; use camino_tempfile::Utf8TempDir; -use dns_service_client::{ - types::{DnsConfigParams, DnsConfigZone, DnsRecord, Srv}, - Client, -}; +use dns_service_client::Client; use dropshot::{test_util::LogContext, HandlerTaskMode}; use hickory_resolver::error::ResolveErrorKind; use hickory_resolver::TokioAsyncResolver; @@ -15,6 +12,9 @@ use hickory_resolver::{ config::{NameServerConfig, Protocol, ResolverConfig, ResolverOpts}, proto::op::ResponseCode, }; +use internal_dns_types::config::{ + DnsConfigParams, DnsConfigZone, DnsRecord, Srv, +}; use omicron_test_utils::dev::test_setup_log; use slog::o; use std::{ diff --git a/docs/debugging-time-sync.adoc b/docs/debugging-time-sync.adoc new file mode 100644 index 0000000000..4c165bb50a --- /dev/null +++ b/docs/debugging-time-sync.adoc @@ -0,0 +1,304 @@ +:showtitle: +:numbered: +:toc: left + += Omicron Time Synchronization Debugging Guide + +THis guide is aimed at helping Omicron developers debug common time +synchronisation problems. If you run into a problem that's not covered here, +please consider adding it! + +== Overview + +In any Oxide control plane deployment, there is a single NTP zone per sled. +Up to two of these will be configured as *boundary* NTP zones and these are the +ones that reach out of the rack to communicate with upstream NTP servers on an +external network. In lab environments this is often a public NTP server pool on +the Internet. On sleds that are not running one of the two boundary NTP zones, +there are *internal* NTP zones, which talk to the boundary NTP zones in order +to synchronise time. + +The external, boundary and internal NTP zones form a hierarchy: + + +[source,text] +---- + +----------+ + | External | + | Source | + +----------+ + / \ + / \ + v v + +----------+ +----------+ + | Boundary | | Boundary | + | NTP | | NTP | + | Zone | | Zone | + +----------+ +----------+ + ||| ||| + vvv vvv + +----------+ +----------+ +----------+ +----------+ + | Internal | | Internal | | Internal | | Internal | + | NTP | | NTP |....| NTP | | NTP | + | Zone | | Zone | | Zone | | Zone | + +----------+ +----------+ +----------+ +----------+ +---- + +== Debugging + +Time synchronisation is required to be established before other services, such +as the database, can be brought online. The sled agent will usually be +repeatedly reporting: + +[source,text] +---- +2024-04-01T23:14:24.799Z WARN SledAgent (RSS): Time is not yet synchronized + error = "Time is synchronized on 0/10 sleds" +---- + +You will first need to find one of the boundary NTP zones. In the case of a +single or dual sled deployment this is obviously easy -- both sleds will have a +boundary NTP zone -- but otherwise you are looking for an NTP zone in which the +NTP `boundary` property in the `chrony-setup` service is true: + +[source,text] +---- +sled0# svcprop -p config/boundary -z `zoneadm list | grep oxz_ntp` chrony-setup +true +---- + +Having found a boundary zone, log into it and check the following things: + +===== Basic networking + +First, confirm that the zone has basic networking configuration and +connectivity with the outside world. In some environments this may be limited +due to the configuration of external network devices such as firewalls, but for +the purposes of this guide I assume that it's unrestricted: + +The zone should have an OPTE interface: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# dladm +LINK CLASS MTU STATE BRIDGE OVER +opte0 misc 1500 up -- -- +oxControlService1 vnic 9000 up -- ? +---- + +and that interface should have successfully obtained an IP address via DHCP: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# ipadm show-addr opte0/public +ADDROBJ TYPE STATE ADDR +opte0/public dhcp ok 172.30.3.6/32 +---- + +and a corresponding default route: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# route -n get default + route to: default +destination: default + mask: default + gateway: 172.30.3.1 + interface: opte0 + flags: + recvpipe sendpipe ssthresh rtt,ms rttvar,ms hopcount mtu expire + 0 0 0 0 0 0 1500 0 +---- + +The zone should be able to ping the Internet, by number: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# ping -sn 1.1.1.1 54 1 +PING 1.1.1.1 (1.1.1.1): 54 data bytes +62 bytes from 1.1.1.1: icmp_seq=0. time=1.953 ms + +----1.1.1.1 PING Statistics---- +1 packets transmitted, 1 packets received, 0% packet loss +round-trip (ms) min/avg/max/stddev = 1.953/1.953/1.953/-nan +---- + +and by name: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# ping -sn oxide.computer 56 1 +PING oxide.computer (76.76.21.21): 56 data bytes +64 bytes from 76.76.21.21: icmp_seq=0. time=1.373 ms + +----oxide.computer PING Statistics---- +1 packets transmitted, 1 packets received, 0% packet loss +round-trip (ms) min/avg/max/stddev = 1.373/1.373/1.373/-nan +---- + +Perform arbitrary DNS lookups via dig: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# dig 0.pool.ntp.org @1.1.1.1 + +; <<>> DiG 9.18.14 <<>> 0.pool.ntp.org @1.1.1.1 +;; global options: +cmd +;; Got answer: +;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 3283 +;; flags: qr rd ra; QUERY: 1, ANSWER: 4, AUTHORITY: 0, ADDITIONAL: 1 + +;; OPT PSEUDOSECTION: +; EDNS: version: 0, flags:; udp: 1232 +;; QUESTION SECTION: +;0.pool.ntp.org. IN A + +;; ANSWER SECTION: +0.pool.ntp.org. 68 IN A 23.186.168.1 +0.pool.ntp.org. 68 IN A 204.17.205.27 +0.pool.ntp.org. 68 IN A 23.186.168.2 +0.pool.ntp.org. 68 IN A 198.60.22.240 + +;; Query time: 2 msec +;; SERVER: 1.1.1.1#53(1.1.1.1) (UDP) +;; WHEN: Wed Oct 02 16:51:29 UTC 2024 +;; MSG SIZE rcvd: 107 +---- + +and via the local resolver: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# getent hosts time.cloudflare.com +162.159.200.123 time.cloudflare.com +162.159.200.1 time.cloudflare.com +---- + +===== NTP Service (chrony) + +Having established that basic networking and DNS are working, now look at the +running NTP service (chrony). + +First, confirm that it is indeed running. There should be two processes +associated with the service. + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# svcs -vp ntp +STATE NSTATE STIME CTID FMRI +online - 1986 217 svc:/oxide/ntp:default + 1986 2551 chronyd + 1986 2552 chronyd +---- + +Check if it has been able to synchronise time. + +Here is example output for a server that cannot synchronise: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# chronyc -n tracking +Reference ID : 7F7F0101 () +Stratum : 10 +Ref time (UTC) : Mon Apr 01 23:14:59 +System time : 0.000000000 seconds +Last offset : +0.000000000 seconds +RMS offset : 0.000000000 seconds +Frequency : 0.000 ppm slow +Residual freq : +0.000 ppm +Skew : 0.000 ppm +Root delay : 0.000000000 seconds +Root dispersion : 0.000000000 seconds +Update interval : 0.0 seconds +Leap status : Normal +---- + +and an example of one that can: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# chronyc -n tracking +Reference ID : A29FC87B (162.159.200.123) +Stratum : 4 +Ref time (UTC) : Wed Oct 02 16:57:11 2024 +System time : 0.000004693 seconds fast of NTP time +Last offset : +0.000000982 seconds +RMS offset : 0.000002580 seconds +Frequency : 32.596 ppm slow +Residual freq : +0.002 ppm +Skew : 0.111 ppm +Root delay : 0.030124957 seconds +Root dispersion : 0.000721255 seconds +Update interval : 8.1 seconds +Leap status : Normal +---- + +Similarly, check the active time synchronisation source list: + +Example of output for a server that cannot synchronise: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# chronyc -n sources -a +MS Name/IP address Stratum Poll Reach LastRx Last sample +================================================================ +---- + +and one that can: + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# chronyc -n sources -a +MS Name/IP address Stratum Poll Reach LastRx Last sample +=============================================================================== +^? ID#0000000001 0 0 0 - +0ns[ +0ns] +/- 0ns +^* 162.159.200.123 3 3 377 4 +45us[ +44us] +/- 16ms +^? ID#0000000003 0 0 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000004 0 0 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000005 0 0 0 - +0ns[ +0ns] +/- 0ns +^? 2606:4700:f1::1 0 3 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000007 0 0 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000008 0 0 0 - +0ns[ +0ns] +/- 0ns +^+ 162.159.200.1 3 3 377 5 -64us[ -65us] +/- 16ms +^? ID#0000000010 0 0 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000011 0 0 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000012 0 0 0 - +0ns[ +0ns] +/- 0ns +^? 2606:4700:f1::123 0 3 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000014 0 0 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000015 0 0 0 - +0ns[ +0ns] +/- 0ns +^? ID#0000000016 0 0 0 - +0ns[ +0ns] +/- 0ns +---- + +Note that the `Reach` column is shown in octal and is a representation of +bits showing the past 8 communication attempts. `377` means that all of the +previous 8 attempts succeeded. + +Chrony generates log files under `/var/log/chrony/` which can help with +diagnosing failures. + +At this point, if time is not synchronising. Stop the chrony daemon and attempt +to synchronise manually, from the command line. + +[source,text] +---- +root@oxz_ntp_cb901d3e:~# svcadm disable ntp +root@oxz_ntp_cb901d3e:~# /usr/sbin/chronyd -t 10 -ddQ 'pool time.cloudflare.com iburst maxdelay 0.1' +2024-10-02T17:02:54Z chronyd version 4.5 starting (+CMDMON +NTP +REFCLOCK -RTC ++PRIVDROP -SCFILTER +SIGND +ASYNCDNS -NTS +SECHASH +IPV6 -DEBUG) +2024-10-02T17:02:54Z Disabled control of system clock +2024-10-02T17:02:59Z System clock wrong by -0.000015 seconds (ignored) +2024-10-02T17:02:59Z chronyd exiting +---- + +== Post-mortem CI Debugging + +If time synchronisation fails in CI, such as in the omicron `deploy` job, then +the information above will have been collected as evidence and uploaded to +buildomat for inspection. You should be able to perform the same diagnosis by +finding the output of the above commands in there and determining whether basic +networking is present and correct, and then whether chrony is behaving as +expected. + +If there's something that would be useful in post mortem but is not being +collected, add it to the deploy job script. + diff --git a/illumos-utils/Cargo.toml b/illumos-utils/Cargo.toml index 3d17745b7e..e1421bd3ab 100644 --- a/illumos-utils/Cargo.toml +++ b/illumos-utils/Cargo.toml @@ -17,7 +17,9 @@ camino.workspace = true camino-tempfile.workspace = true cfg-if.workspace = true crucible-smf.workspace = true +dropshot.workspace = true futures.workspace = true +http.workspace = true ipnetwork.workspace = true libc.workspace = true macaddr.workspace = true @@ -29,6 +31,7 @@ oxnet.workspace = true schemars.workspace = true serde.workspace = true slog.workspace = true +slog-error-chain.workspace = true smf.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/illumos-utils/src/lib.rs b/illumos-utils/src/lib.rs index 48a5767f41..fec515e673 100644 --- a/illumos-utils/src/lib.rs +++ b/illumos-utils/src/lib.rs @@ -4,6 +4,8 @@ //! Wrappers around illumos-specific commands. +use dropshot::HttpError; +use slog_error_chain::InlineErrorChain; #[allow(unused)] use std::sync::atomic::{AtomicBool, Ordering}; @@ -71,6 +73,18 @@ pub enum ExecutionError { NotRunning, } +impl From for HttpError { + fn from(err: ExecutionError) -> Self { + let message = InlineErrorChain::new(&err).to_string(); + HttpError { + status_code: http::StatusCode::INTERNAL_SERVER_ERROR, + error_code: Some(String::from("Internal")), + external_message: message.clone(), + internal_message: message, + } + } +} + // We wrap this method in an inner module to make it possible to mock // these free functions. #[cfg_attr(any(test, feature = "testing"), mockall::automock, allow(dead_code))] diff --git a/illumos-utils/src/opte/mod.rs b/illumos-utils/src/opte/mod.rs index 9a86711ae6..3b3208e718 100644 --- a/illumos-utils/src/opte/mod.rs +++ b/illumos-utils/src/opte/mod.rs @@ -86,11 +86,23 @@ fn net_to_cidr(net: IpNet) -> IpCidr { } /// Convert a nexus `RouterTarget` to an OPTE `RouterTarget`. -fn router_target_opte(target: &shared::RouterTarget) -> RouterTarget { +/// +/// Currently, we strip InternetGateway IDs from any routes targeting +/// non-instance NICs, because we need to actively store the full set +/// (and division of) SNAT/ephemeral/floating IPs. +/// Sled-agent only holds this today for instances, because these are +/// reconfigurable at run-time (whereas services and probes are fixed). +fn router_target_opte( + target: &shared::RouterTarget, + is_instance: bool, +) -> RouterTarget { use shared::RouterTarget::*; match target { Drop => RouterTarget::Drop, - InternetGateway => RouterTarget::InternetGateway, + InternetGateway(id) if is_instance => { + RouterTarget::InternetGateway(*id) + } + InternetGateway(_) => RouterTarget::InternetGateway(None), Ip(ip) => RouterTarget::Ip((*ip).into()), VpcSubnet(net) => RouterTarget::VpcSubnet(net_to_cidr(*net)), } diff --git a/illumos-utils/src/opte/port_manager.rs b/illumos-utils/src/opte/port_manager.rs index 735428907e..3ce4546c6a 100644 --- a/illumos-utils/src/opte/port_manager.rs +++ b/illumos-utils/src/opte/port_manager.rs @@ -13,6 +13,7 @@ use crate::opte::Port; use crate::opte::Vni; use ipnetwork::IpNetwork; use omicron_common::api::external; +use omicron_common::api::internal::shared::ExternalIpGatewayMap; use omicron_common::api::internal::shared::NetworkInterface; use omicron_common::api::internal::shared::NetworkInterfaceKind; use omicron_common::api::internal::shared::ResolvedVpcFirewallRule; @@ -77,6 +78,12 @@ struct PortManagerInner { /// Map of all current resolved routes. routes: Mutex>, + + /// Mappings of associated Internet Gateways for all External IPs + /// attached to each NIC. + /// + /// IGW IDs are specific to the VPC of each NIC. + eip_gateways: Mutex>>>, } impl PortManagerInner { @@ -116,6 +123,7 @@ impl PortManager { underlay_ip, ports: Mutex::new(BTreeMap::new()), routes: Mutex::new(Default::default()), + eip_gateways: Mutex::new(Default::default()), }); Self { inner } @@ -249,7 +257,7 @@ impl PortManager { }; let vpc_cfg = VpcCfg { - ip_cfg, + ip_cfg: ip_cfg.clone(), guest_mac: MacAddr::from(nic.mac.into_array()), gateway_mac: MacAddr::from(gateway.mac.into_array()), vni, @@ -329,26 +337,15 @@ impl PortManager { // create a record to show that we're interested in receiving // those routes. let mut routes = self.inner.routes.lock().unwrap(); - let system_routes = - routes.entry(port.system_router_key()).or_insert_with(|| { - let mut routes = HashSet::new(); - - // Services do not talk to one another via OPTE, but do need - // to reach out over the Internet *before* nexus is up to give - // us real rules. The easiest bet is to instantiate these here. - if is_service { - routes.insert(ResolvedVpcRoute { - dest: "0.0.0.0/0".parse().unwrap(), - target: ApiRouterTarget::InternetGateway, - }); - routes.insert(ResolvedVpcRoute { - dest: "::/0".parse().unwrap(), - target: ApiRouterTarget::InternetGateway, - }); - } + let system_routes = match &ip_cfg { + IpCfg::Ipv4(_) => system_routes_v4(is_service, &mut routes, &port), + IpCfg::Ipv6(_) => system_routes_v6(is_service, &mut routes, &port), + IpCfg::DualStack { .. } => { + system_routes_v4(is_service, &mut routes, &port); + system_routes_v6(is_service, &mut routes, &port) + } + }; - RouteSet { version: None, routes, active_ports: 0 } - }); system_routes.active_ports += 1; // Clone is needed to get borrowck on our side, sadly. let system_routes = system_routes.clone(); @@ -371,7 +368,16 @@ impl PortManager { class, port_name: port_name.clone(), dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), + target: super::router_target_opte( + &route.target, + // This option doesn't make any difference here: + // We don't yet know any associated InetGw IDs for + // the IPs attached to this interface. + // We might have this knowledge at create-time in + // future, but assume for now that the control plane + // will backfill this. + false, + ), }; #[cfg(target_os = "illumos")] @@ -443,9 +449,11 @@ impl PortManager { ) -> Result<(), Error> { let mut routes = self.inner.routes.lock().unwrap(); let mut deltas = HashMap::new(); + slog::debug!(self.inner.log, "new routes: {new_routes:#?}"); for new in new_routes { // Disregard any route information for a subnet we don't have. let Some(old) = routes.get(&new.id) else { + slog::warn!(self.inner.log, "ignoring route {new:#?}"); continue; }; @@ -458,6 +466,13 @@ impl PortManager { (Some(old_vers), Some(new_vers)) if !old_vers.is_replaced_by(&new_vers) => { + slog::info!( + self.inner.log, + "skipping delta compute for subnet"; + "subnet" => ?new.id, + "old_vers" => ?old_vers, + "new_vers" => ?new_vers, + ); continue; } _ => ( @@ -486,28 +501,41 @@ impl PortManager { let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?; // Propagate deltas out to all ports. - for port in ports.values() { + for (interface_id, port) in ports.iter() { let system_id = port.system_router_key(); let system_delta = deltas.get(&system_id); let custom_id = port.custom_router_key(); let custom_delta = deltas.get(&custom_id); + let is_instance = + matches!(interface_id.1, NetworkInterfaceKind::Instance { .. }); + #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] for (class, delta) in [ (RouterClass::System, system_delta), (RouterClass::Custom, custom_delta), ] { let Some((to_add, to_delete)) = delta else { + debug!(self.inner.log, "vpc route ensure: no delta"); continue; }; + debug!(self.inner.log, "vpc route ensure to_add: {to_add:#?}"); + debug!( + self.inner.log, + "vpc router ensure to_delete: {to_delete:#?}" + ); + for route in to_delete { let route = DelRouterEntryReq { class, port_name: port.name().into(), dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), + target: super::router_target_opte( + &route.target, + is_instance, + ), }; #[cfg(target_os = "illumos")] @@ -526,7 +554,10 @@ impl PortManager { class, port_name: port.name().into(), dest: super::net_to_cidr(route.dest), - target: super::router_target_opte(&route.target), + target: super::router_target_opte( + &route.target, + is_instance, + ), }; #[cfg(target_os = "illumos")] @@ -545,6 +576,20 @@ impl PortManager { Ok(()) } + /// Set Internet Gateway mappings for all external IPs in use + /// by attached `NetworkInterface`s. + /// + /// Returns whether the internal mappings were changed. + pub fn set_eip_gateways(&self, mappings: ExternalIpGatewayMap) -> bool { + let mut gateways = self.inner.eip_gateways.lock().unwrap(); + + let changed = &*gateways != &mappings.mappings; + + *gateways = mappings.mappings; + + changed + } + /// Ensure external IPs for an OPTE port are up to date. #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))] pub fn external_ips_ensure( @@ -555,6 +600,10 @@ impl PortManager { ephemeral_ip: Option, floating_ips: &[IpAddr], ) -> Result<(), Error> { + let egw_lock = self.inner.eip_gateways.lock().unwrap(); + let inet_gw_map = egw_lock.get(&nic_id).cloned(); + drop(egw_lock); + let ports = self.inner.ports.lock().unwrap(); let port = ports.get(&(nic_id, nic_kind)).ok_or_else(|| { Error::ExternalIpUpdateMissingPort(nic_id, nic_kind) @@ -645,10 +694,21 @@ impl PortManager { } } + let inet_gw_map = if let Some(map) = inet_gw_map { + Some( + map.into_iter() + .map(|(k, v)| (k.into(), v.into_iter().collect())) + .collect(), + ) + } else { + None + }; + let req = SetExternalIpsReq { port_name: port.name().into(), external_ips_v4: v4_cfg, external_ips_v6: v6_cfg, + inet_gw_map, }; #[cfg(target_os = "illumos")] @@ -939,3 +999,43 @@ impl Drop for PortTicket { let _ = self.release_inner(); } } + +fn system_routes_v4<'a>( + is_service: bool, + routes: &'a mut HashMap, + port: &Port, +) -> &'a mut RouteSet { + let routes = routes.entry(port.system_router_key()).or_insert_with(|| { + let routes = HashSet::new(); + RouteSet { version: None, routes, active_ports: 0 } + }); + + if is_service { + routes.routes.insert(ResolvedVpcRoute { + dest: "0.0.0.0/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway(None), + }); + } + + routes +} + +fn system_routes_v6<'a>( + is_service: bool, + routes: &'a mut HashMap, + port: &Port, +) -> &'a mut RouteSet { + let routes = routes.entry(port.system_router_key()).or_insert_with(|| { + let routes = HashSet::new(); + RouteSet { version: None, routes, active_ports: 0 } + }); + + if is_service { + routes.routes.insert(ResolvedVpcRoute { + dest: "::/0".parse().unwrap(), + target: ApiRouterTarget::InternetGateway(None), + }); + } + + routes +} diff --git a/illumos-utils/src/svcadm.rs b/illumos-utils/src/svcadm.rs index 0d472187df..7e970dea04 100644 --- a/illumos-utils/src/svcadm.rs +++ b/illumos-utils/src/svcadm.rs @@ -18,4 +18,11 @@ impl Svcadm { execute(cmd)?; Ok(()) } + + pub fn enable_service(fmri: String) -> Result<(), ExecutionError> { + let mut cmd = std::process::Command::new(PFEXEC); + let cmd = cmd.args(&[SVCADM, "enable", "-s", &fmri]); + execute(cmd)?; + Ok(()) + } } diff --git a/internal-dns-cli/Cargo.toml b/internal-dns/cli/Cargo.toml similarity index 82% rename from internal-dns-cli/Cargo.toml rename to internal-dns/cli/Cargo.toml index 3e34c21622..93375ef7a0 100644 --- a/internal-dns-cli/Cargo.toml +++ b/internal-dns/cli/Cargo.toml @@ -12,7 +12,8 @@ anyhow.workspace = true clap.workspace = true dropshot.workspace = true hickory-resolver.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true omicron-common.workspace = true slog.workspace = true tokio.workspace = true diff --git a/internal-dns-cli/src/bin/dnswait.rs b/internal-dns/cli/src/bin/dnswait.rs similarity index 85% rename from internal-dns-cli/src/bin/dnswait.rs rename to internal-dns/cli/src/bin/dnswait.rs index f9875e71a0..8d7c0e2683 100644 --- a/internal-dns-cli/src/bin/dnswait.rs +++ b/internal-dns/cli/src/bin/dnswait.rs @@ -8,8 +8,8 @@ use anyhow::Context; use anyhow::Result; use clap::Parser; use clap::ValueEnum; -use internal_dns::resolver::ResolveError; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::ResolveError; +use internal_dns_resolver::Resolver; use slog::{info, warn}; use std::net::SocketAddr; @@ -40,15 +40,17 @@ enum ServiceName { ClickhouseServer, } -impl From for internal_dns::ServiceName { +impl From for internal_dns_types::names::ServiceName { fn from(value: ServiceName) -> Self { match value { - ServiceName::Cockroach => internal_dns::ServiceName::Cockroach, + ServiceName::Cockroach => { + internal_dns_types::names::ServiceName::Cockroach + } ServiceName::ClickhouseServer => { - internal_dns::ServiceName::ClickhouseServer + internal_dns_types::names::ServiceName::ClickhouseServer } ServiceName::ClickhouseKeeper => { - internal_dns::ServiceName::ClickhouseKeeper + internal_dns_types::names::ServiceName::ClickhouseKeeper } } } @@ -79,7 +81,8 @@ async fn main() -> Result<()> { let result = omicron_common::backoff::retry_notify( omicron_common::backoff::retry_policy_internal_service(), || async { - let dns_name = internal_dns::ServiceName::from(opt.srv_name); + let dns_name = + internal_dns_types::names::ServiceName::from(opt.srv_name); resolver.lookup_srv(dns_name).await.map_err(|error| match error { ResolveError::Resolve(_) | ResolveError::NotFound(_) diff --git a/internal-dns/Cargo.toml b/internal-dns/resolver/Cargo.toml similarity index 90% rename from internal-dns/Cargo.toml rename to internal-dns/resolver/Cargo.toml index c12035e2cb..7c89fe41c4 100644 --- a/internal-dns/Cargo.toml +++ b/internal-dns/resolver/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "internal-dns" +name = "internal-dns-resolver" version = "0.1.0" edition = "2021" license = "MPL-2.0" @@ -8,26 +8,25 @@ license = "MPL-2.0" workspace = true [dependencies] -anyhow.workspace = true -chrono.workspace = true -dns-service-client.workspace = true futures.workspace = true -hyper.workspace = true +hickory-resolver.workspace = true +internal-dns-types.workspace = true omicron-common.workspace = true -omicron-uuid-kinds.workspace = true +omicron-workspace-hack.workspace = true +qorb.workspace = true reqwest = { workspace = true, features = ["rustls-tls", "stream"] } slog.workspace = true thiserror.workspace = true -hickory-resolver.workspace = true -uuid.workspace = true -omicron-workspace-hack.workspace = true [dev-dependencies] +anyhow.workspace = true assert_matches.workspace = true dropshot.workspace = true dns-server.workspace = true +dns-service-client.workspace = true expectorate.workspace = true omicron-test-utils.workspace = true +omicron-uuid-kinds.workspace = true progenitor.workspace = true serde = { workspace = true, features = ["derive"] } serde_json.workspace = true diff --git a/internal-dns/resolver/src/lib.rs b/internal-dns/resolver/src/lib.rs new file mode 100644 index 0000000000..795e2ea998 --- /dev/null +++ b/internal-dns/resolver/src/lib.rs @@ -0,0 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! A resolver for internal DNS names (see RFD 248). + +mod resolver; + +pub use resolver::*; diff --git a/internal-dns/src/resolver.rs b/internal-dns/resolver/src/resolver.rs similarity index 95% rename from internal-dns/src/resolver.rs rename to internal-dns/resolver/src/resolver.rs index 5675575a18..2378e62c65 100644 --- a/internal-dns/src/resolver.rs +++ b/internal-dns/resolver/src/resolver.rs @@ -7,26 +7,65 @@ use hickory_resolver::config::{ }; use hickory_resolver::lookup::SrvLookup; use hickory_resolver::TokioAsyncResolver; +use internal_dns_types::names::ServiceName; use omicron_common::address::{ get_internal_dns_server_addresses, Ipv6Subnet, AZ_PREFIX, DNS_PORT, }; use slog::{debug, error, info, trace}; use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; -pub type DnsError = dns_service_client::Error; - #[derive(Debug, Clone, thiserror::Error)] pub enum ResolveError { #[error(transparent)] Resolve(#[from] hickory_resolver::error::ResolveError), #[error("Record not found for SRV key: {}", .0.dns_name())] - NotFound(crate::ServiceName), + NotFound(ServiceName), #[error("Record not found for {0}")] NotFoundByString(String), } +/// A wrapper around a set of bootstrap DNS addresses, providing a convenient +/// way to construct a [`qorb::resolvers::dns::DnsResolver`] for specific +/// services. +#[derive(Debug, Clone)] +pub struct QorbResolver { + bootstrap_dns_ips: Vec, +} + +impl QorbResolver { + pub fn new(bootstrap_dns_ips: Vec) -> Self { + Self { bootstrap_dns_ips } + } + + pub fn bootstrap_dns_ips(&self) -> &[SocketAddr] { + &self.bootstrap_dns_ips + } + + pub fn for_service( + &self, + service: ServiceName, + ) -> qorb::resolver::BoxedResolver { + let config = qorb::resolvers::dns::DnsResolverConfig { + // Ignore the TTL returned by our servers, primarily to avoid + // thrashing if they return a TTL of 0 (which they currently do: + // https://github.com/oxidecomputer/omicron/issues/6790). + hardcoded_ttl: Some(std::time::Duration::MAX), + // We don't currently run additional internal DNS servers that + // themselves need to be found via a set of bootstrap DNS IPs, but + // if we did, we'd populate `resolver_service` here to tell qorb how + // to find them. + ..Default::default() + }; + Box::new(qorb::resolvers::dns::DnsResolver::new( + qorb::service::Name(service.srv_name()), + self.bootstrap_dns_ips.clone(), + config, + )) + } +} + /// A wrapper around a DNS resolver, providing a way to conveniently /// look up IP addresses of services based on their SRV keys. #[derive(Clone)] @@ -161,7 +200,7 @@ impl Resolver { /// need to be looked up to find A/AAAA records. pub async fn lookup_srv( &self, - srv: crate::ServiceName, + srv: ServiceName, ) -> Result, ResolveError> { let name = srv.srv_name(); trace!(self.log, "lookup_srv"; "dns_name" => &name); @@ -181,7 +220,7 @@ impl Resolver { pub async fn lookup_all_ipv6( &self, - srv: crate::ServiceName, + srv: ServiceName, ) -> Result, ResolveError> { let name = srv.srv_name(); trace!(self.log, "lookup_all_ipv6 srv"; "dns_name" => &name); @@ -217,7 +256,7 @@ impl Resolver { // API that can be improved upon later. pub async fn lookup_socket_v6( &self, - service: crate::ServiceName, + service: ServiceName, ) -> Result { let name = service.srv_name(); trace!(self.log, "lookup_socket_v6 srv"; "dns_name" => &name); @@ -241,7 +280,7 @@ impl Resolver { /// targets and return a list of [`SocketAddrV6`]. pub async fn lookup_all_socket_v6( &self, - service: crate::ServiceName, + service: ServiceName, ) -> Result, ResolveError> { let name = service.srv_name(); trace!(self.log, "lookup_all_socket_v6 srv"; "dns_name" => &name); @@ -286,7 +325,7 @@ impl Resolver { pub async fn lookup_ip( &self, - srv: crate::ServiceName, + srv: ServiceName, ) -> Result { let name = srv.srv_name(); debug!(self.log, "lookup srv"; "dns_name" => &name); @@ -366,15 +405,16 @@ impl Resolver { mod test { use super::ResolveError; use super::Resolver; - use crate::DNS_ZONE; - use crate::{DnsConfigBuilder, ServiceName}; use anyhow::Context; use assert_matches::assert_matches; - use dns_service_client::types::DnsConfigParams; use dropshot::{ endpoint, ApiDescription, HandlerTaskMode, HttpError, HttpResponseOk, RequestContext, }; + use internal_dns_types::config::DnsConfigBuilder; + use internal_dns_types::config::DnsConfigParams; + use internal_dns_types::names::ServiceName; + use internal_dns_types::names::DNS_ZONE; use omicron_test_utils::dev::test_setup_log; use omicron_uuid_kinds::OmicronZoneUuid; use slog::{o, Logger}; @@ -811,7 +851,7 @@ mod test { // // We'll use the SRV record for Nexus, even though it's just our // standalone test server. - let dns_name = crate::ServiceName::Nexus.srv_name(); + let dns_name = ServiceName::Nexus.srv_name(); let reqwest_client = reqwest::ClientBuilder::new() .dns_resolver(resolver.clone().into()) .build() @@ -891,7 +931,7 @@ mod test { // // We'll use the SRV record for Nexus, even though it's just our // standalone test server. - let dns_name = crate::ServiceName::Nexus.srv_name(); + let dns_name = ServiceName::Nexus.srv_name(); let reqwest_client = reqwest::ClientBuilder::new() .dns_resolver(resolver.clone().into()) .build() diff --git a/internal-dns/tests/output/test-server.json b/internal-dns/resolver/tests/output/test-server.json similarity index 100% rename from internal-dns/tests/output/test-server.json rename to internal-dns/resolver/tests/output/test-server.json diff --git a/internal-dns/types/Cargo.toml b/internal-dns/types/Cargo.toml new file mode 100644 index 0000000000..612aafde04 --- /dev/null +++ b/internal-dns/types/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "internal-dns-types" +version = "0.1.0" +edition = "2021" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +chrono.workspace = true +omicron-common.workspace = true +omicron-workspace-hack.workspace = true +omicron-uuid-kinds.workspace = true +schemars.workspace = true +serde.workspace = true + +[dev-dependencies] +expectorate.workspace = true +serde_json.workspace = true diff --git a/internal-dns/src/config.rs b/internal-dns/types/src/config.rs similarity index 86% rename from internal-dns/src/config.rs rename to internal-dns/types/src/config.rs index 8318337f6d..a7f223caee 100644 --- a/internal-dns/src/config.rs +++ b/internal-dns/types/src/config.rs @@ -63,12 +63,13 @@ use crate::names::{ServiceName, BOUNDARY_NTP_DNS_NAME, DNS_ZONE}; use anyhow::{anyhow, ensure}; use core::fmt; -use dns_service_client::types::{DnsConfigParams, DnsConfigZone, DnsRecord}; -use omicron_common::address::CLICKHOUSE_TCP_PORT; +use omicron_common::address::{CLICKHOUSE_ADMIN_PORT, CLICKHOUSE_TCP_PORT}; use omicron_common::api::external::Generation; use omicron_uuid_kinds::{OmicronZoneUuid, SledUuid}; -use std::collections::BTreeMap; -use std::net::Ipv6Addr; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, HashMap}; +use std::net::{Ipv4Addr, Ipv6Addr}; /// Used to construct the DNS name for a control plane host #[derive(Clone, Debug, PartialEq, PartialOrd)] @@ -175,7 +176,7 @@ impl Zone { Host::Zone(self.clone()) } - pub(crate) fn dns_name(&self) -> String { + pub fn dns_name(&self) -> String { self.to_host().dns_name() } } @@ -411,6 +412,9 @@ impl DnsConfigBuilder { /// this zone, and `http_port` is the associated port for that service. The /// native service is added automatically, using its default port. /// + /// For `ClickhouseServer` zones we also need to add a + /// `ClickhouseAdminServer` service. + /// /// # Errors /// /// This fails if the provided `http_service` is not for a ClickHouse @@ -435,6 +439,45 @@ impl DnsConfigBuilder { ServiceName::ClickhouseNative, &zone, CLICKHOUSE_TCP_PORT, + )?; + + if http_service == ServiceName::ClickhouseServer { + self.service_backend_zone( + ServiceName::ClickhouseAdminServer, + &zone, + CLICKHOUSE_ADMIN_PORT, + )?; + } + + Ok(()) + } + + /// Higher-level shorthand for adding a ClickhouseKeeper zone with several + /// services. + /// + /// # Errors + /// + /// This fails if the provided `http_service` is not for a ClickhouseKeeper + /// replica server. It also fails if the given zone has already been added + /// to the configuration. + pub fn host_zone_clickhouse_keeper( + &mut self, + zone_id: OmicronZoneUuid, + underlay_address: Ipv6Addr, + service: ServiceName, + port: u16, + ) -> anyhow::Result<()> { + anyhow::ensure!( + service == ServiceName::ClickhouseKeeper, + "This method is only valid for ClickHouse keeper servers, \ + but we were provided the service '{service:?}'", + ); + let zone = self.host_zone(zone_id, underlay_address)?; + self.service_backend_zone(service, &zone, port)?; + self.service_backend_zone( + ServiceName::ClickhouseAdminKeeper, + &zone, + CLICKHOUSE_ADMIN_PORT, ) } @@ -481,7 +524,7 @@ impl DnsConfigBuilder { let records = zone2port .into_iter() .map(|(zone, port)| { - DnsRecord::Srv(dns_service_client::types::Srv { + DnsRecord::Srv(Srv { prio: 0, weight: 0, port, @@ -500,7 +543,7 @@ impl DnsConfigBuilder { let records = sled2port .into_iter() .map(|(sled, port)| { - DnsRecord::Srv(dns_service_client::types::Srv { + DnsRecord::Srv(Srv { prio: 0, weight: 0, port, @@ -536,16 +579,129 @@ impl DnsConfigBuilder { } } +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct DnsConfigParams { + pub generation: u64, + pub time_created: chrono::DateTime, + pub zones: Vec, +} + +impl DnsConfigParams { + /// Given a high-level DNS configuration, return a reference to its sole + /// DNS zone. + /// + /// # Errors + /// + /// Returns an error if there are 0 or more than one zones in this + /// configuration. + pub fn sole_zone(&self) -> Result<&DnsConfigZone, anyhow::Error> { + ensure!( + self.zones.len() == 1, + "expected exactly one DNS zone, but found {}", + self.zones.len() + ); + Ok(&self.zones[0]) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] +pub struct DnsConfig { + pub generation: u64, + pub time_created: chrono::DateTime, + pub time_applied: chrono::DateTime, + pub zones: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema, PartialEq, Eq)] +pub struct DnsConfigZone { + pub zone_name: String, + pub records: HashMap>, +} + +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialEq, + Eq, + PartialOrd, + Ord, +)] +#[serde(tag = "type", content = "data")] +pub enum DnsRecord { + A(Ipv4Addr), + // The renames are because openapi-lint complains about `Aaaa` and `Srv` + // not being in screaming snake case. `Aaaa` and `Srv` are the idiomatic + // Rust casings, though. + #[serde(rename = "AAAA")] + Aaaa(Ipv6Addr), + #[serde(rename = "SRV")] + Srv(Srv), +} + +// The `From` and `From` implementations are very slightly +// dubious, because a v4 or v6 address could also theoretically map to a DNS +// PTR record +// (https://www.cloudflare.com/learning/dns/dns-records/dns-ptr-record/). +// However, we don't support PTR records at the moment, so this is fine. Would +// certainly be worth revisiting if we do in the future, though. + +impl From for DnsRecord { + fn from(ip: Ipv4Addr) -> Self { + DnsRecord::A(ip) + } +} + +impl From for DnsRecord { + fn from(ip: Ipv6Addr) -> Self { + DnsRecord::Aaaa(ip) + } +} + +impl From for DnsRecord { + fn from(srv: Srv) -> Self { + DnsRecord::Srv(srv) + } +} + +#[derive( + Clone, + Debug, + Serialize, + Deserialize, + JsonSchema, + PartialEq, + Eq, + PartialOrd, + Ord, +)] +pub struct Srv { + pub prio: u16, + pub weight: u16, + pub port: u16, + pub target: String, +} + #[cfg(test)] mod test { use super::{DnsConfigBuilder, Host, ServiceName}; - use crate::{config::Zone, DNS_ZONE}; + use crate::{config::Zone, names::DNS_ZONE}; use omicron_uuid_kinds::{OmicronZoneUuid, SledUuid}; use std::{collections::BTreeMap, io::Write, net::Ipv6Addr}; #[test] fn display_srv_service() { assert_eq!(ServiceName::Clickhouse.dns_name(), "_clickhouse._tcp",); + assert_eq!( + ServiceName::ClickhouseAdminKeeper.dns_name(), + "_clickhouse-admin-keeper._tcp", + ); + assert_eq!( + ServiceName::ClickhouseAdminServer.dns_name(), + "_clickhouse-admin-server._tcp", + ); assert_eq!( ServiceName::ClickhouseKeeper.dns_name(), "_clickhouse-keeper._tcp", diff --git a/clients/dns-service-client/src/diff.rs b/internal-dns/types/src/diff.rs similarity index 97% rename from clients/dns-service-client/src/diff.rs rename to internal-dns/types/src/diff.rs index 2ae7036c86..85cd38642f 100644 --- a/clients/dns-service-client/src/diff.rs +++ b/internal-dns/types/src/diff.rs @@ -2,12 +2,13 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::types::DnsConfigZone; -use crate::types::DnsRecord; -use crate::types::Srv; -use crate::DnsRecords; use anyhow::ensure; use std::collections::BTreeSet; +use std::collections::HashMap; + +use crate::config::DnsConfigZone; +use crate::config::DnsRecord; +use crate::config::Srv; #[derive(Debug)] enum NameDiff<'a> { @@ -17,6 +18,8 @@ enum NameDiff<'a> { Unchanged(&'a str, &'a [DnsRecord]), } +type DnsRecords = HashMap>; + /// Compare the DNS records contained in two sets of DNS configuration #[derive(Debug)] pub struct DnsDiff<'a> { @@ -216,8 +219,8 @@ impl<'a> std::fmt::Display for DnsDiff<'a> { #[cfg(test)] mod test { use super::DnsDiff; - use crate::types::DnsConfigZone; - use crate::types::DnsRecord; + use crate::config::DnsConfigZone; + use crate::config::DnsRecord; use std::collections::HashMap; use std::net::Ipv4Addr; diff --git a/internal-dns/src/lib.rs b/internal-dns/types/src/lib.rs similarity index 51% rename from internal-dns/src/lib.rs rename to internal-dns/types/src/lib.rs index cc84b6aa76..5dfccd324e 100644 --- a/internal-dns/src/lib.rs +++ b/internal-dns/types/src/lib.rs @@ -2,13 +2,8 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -//! Working with Omicron-internal DNS (see RFD 248) +//! Common types for internal DNS resolution. pub mod config; +pub mod diff; pub mod names; -pub mod resolver; - -// We export these names out to the root for compatibility. -pub use config::DnsConfigBuilder; -pub use names::ServiceName; -pub use names::DNS_ZONE; diff --git a/internal-dns/src/names.rs b/internal-dns/types/src/names.rs similarity index 90% rename from internal-dns/src/names.rs rename to internal-dns/types/src/names.rs index 78db87fbab..ae4885c980 100644 --- a/internal-dns/src/names.rs +++ b/internal-dns/types/src/names.rs @@ -25,6 +25,10 @@ pub const DNS_ZONE_EXTERNAL_TESTING: &str = "oxide-dev.test"; pub enum ServiceName { /// The HTTP interface to a single-node ClickHouse server. Clickhouse, + /// The HTTP interface for managing clickhouse keepers + ClickhouseAdminKeeper, + /// The HTTP interface for managing replicated clickhouse servers + ClickhouseAdminServer, /// The native TCP interface to a ClickHouse server. /// /// NOTE: This is used for either single-node or a replicated cluster. @@ -55,6 +59,8 @@ impl ServiceName { fn service_kind(&self) -> &'static str { match self { ServiceName::Clickhouse => "clickhouse", + ServiceName::ClickhouseAdminKeeper => "clickhouse-admin-keeper", + ServiceName::ClickhouseAdminServer => "clickhouse-admin-server", ServiceName::ClickhouseNative => "clickhouse-native", ServiceName::ClickhouseKeeper => "clickhouse-keeper", ServiceName::ClickhouseServer => "clickhouse-server", @@ -82,6 +88,8 @@ impl ServiceName { pub fn dns_name(&self) -> String { match self { ServiceName::Clickhouse + | ServiceName::ClickhouseAdminKeeper + | ServiceName::ClickhouseAdminServer | ServiceName::ClickhouseNative | ServiceName::ClickhouseKeeper | ServiceName::ClickhouseServer diff --git a/clients/dns-service-client/tests/output/diff_example_different.out b/internal-dns/types/tests/output/diff_example_different.out similarity index 100% rename from clients/dns-service-client/tests/output/diff_example_different.out rename to internal-dns/types/tests/output/diff_example_different.out diff --git a/clients/dns-service-client/tests/output/diff_example_different_reversed.out b/internal-dns/types/tests/output/diff_example_different_reversed.out similarity index 100% rename from clients/dns-service-client/tests/output/diff_example_different_reversed.out rename to internal-dns/types/tests/output/diff_example_different_reversed.out diff --git a/clients/dns-service-client/tests/output/diff_example_empty.out b/internal-dns/types/tests/output/diff_example_empty.out similarity index 100% rename from clients/dns-service-client/tests/output/diff_example_empty.out rename to internal-dns/types/tests/output/diff_example_empty.out diff --git a/internal-dns/tests/output/internal-dns-zone.txt b/internal-dns/types/tests/output/internal-dns-zone.txt similarity index 89% rename from internal-dns/tests/output/internal-dns-zone.txt rename to internal-dns/types/tests/output/internal-dns-zone.txt index d87805f677..b23553130c 100644 --- a/internal-dns/tests/output/internal-dns-zone.txt +++ b/internal-dns/types/tests/output/internal-dns-zone.txt @@ -72,10 +72,10 @@ builder: "non_trivial" { "type": "SRV", "data": { - "port": 127, "prio": 0, - "target": "001de000-c04e-4000-8000-000000000002.host.control-plane.oxide.internal", - "weight": 0 + "weight": 0, + "port": 127, + "target": "001de000-c04e-4000-8000-000000000002.host.control-plane.oxide.internal" } } ], @@ -83,19 +83,19 @@ builder: "non_trivial" { "type": "SRV", "data": { - "port": 123, "prio": 0, - "target": "001de000-c04e-4000-8000-000000000001.host.control-plane.oxide.internal", - "weight": 0 + "weight": 0, + "port": 123, + "target": "001de000-c04e-4000-8000-000000000001.host.control-plane.oxide.internal" } }, { "type": "SRV", "data": { - "port": 124, "prio": 0, - "target": "001de000-c04e-4000-8000-000000000002.host.control-plane.oxide.internal", - "weight": 0 + "weight": 0, + "port": 124, + "target": "001de000-c04e-4000-8000-000000000002.host.control-plane.oxide.internal" } } ], @@ -103,19 +103,19 @@ builder: "non_trivial" { "type": "SRV", "data": { - "port": 125, "prio": 0, - "target": "001de000-c04e-4000-8000-000000000002.host.control-plane.oxide.internal", - "weight": 0 + "weight": 0, + "port": 125, + "target": "001de000-c04e-4000-8000-000000000002.host.control-plane.oxide.internal" } }, { "type": "SRV", "data": { - "port": 126, "prio": 0, - "target": "001de000-c04e-4000-8000-000000000003.host.control-plane.oxide.internal", - "weight": 0 + "weight": 0, + "port": 126, + "target": "001de000-c04e-4000-8000-000000000003.host.control-plane.oxide.internal" } } ], @@ -123,10 +123,10 @@ builder: "non_trivial" { "type": "SRV", "data": { - "port": 123, "prio": 0, - "target": "001de000-51ed-4000-8000-000000000001.sled.control-plane.oxide.internal", - "weight": 0 + "weight": 0, + "port": 123, + "target": "001de000-51ed-4000-8000-000000000001.sled.control-plane.oxide.internal" } } ], diff --git a/live-tests/Cargo.toml b/live-tests/Cargo.toml index f731f248d0..ac0b39a5a5 100644 --- a/live-tests/Cargo.toml +++ b/live-tests/Cargo.toml @@ -17,7 +17,8 @@ anyhow.workspace = true assert_matches.workspace = true dropshot.workspace = true futures.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true live-tests-macros.workspace = true nexus-client.workspace = true nexus-config.workspace = true diff --git a/live-tests/tests/common/mod.rs b/live-tests/tests/common/mod.rs index 28f677f5ed..360e07235a 100644 --- a/live-tests/tests/common/mod.rs +++ b/live-tests/tests/common/mod.rs @@ -6,8 +6,8 @@ pub mod reconfigurator; use anyhow::{anyhow, ensure, Context}; use dropshot::test_util::LogContext; -use internal_dns::resolver::Resolver; -use internal_dns::ServiceName; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use nexus_config::PostgresConfigWithUrl; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; @@ -105,7 +105,7 @@ fn create_resolver(log: &slog::Logger) -> Result { // default value used here. let subnet = Ipv6Subnet::new("fd00:1122:3344:0100::".parse().unwrap()); eprintln!("note: using DNS server for subnet {}", subnet.net()); - internal_dns::resolver::Resolver::new_from_subnet(log.clone(), subnet) + internal_dns_resolver::Resolver::new_from_subnet(log.clone(), subnet) .with_context(|| { format!("creating DNS resolver for subnet {}", subnet.net()) }) diff --git a/nexus-sled-agent-shared/src/inventory.rs b/nexus-sled-agent-shared/src/inventory.rs index bda102c38e..2ed654b944 100644 --- a/nexus-sled-agent-shared/src/inventory.rs +++ b/nexus-sled-agent-shared/src/inventory.rs @@ -99,6 +99,7 @@ pub struct Inventory { pub usable_hardware_threads: u32, pub usable_physical_ram: ByteCount, pub reservoir_size: ByteCount, + pub omicron_zones: OmicronZonesConfig, pub disks: Vec, pub zpools: Vec, pub datasets: Vec, diff --git a/nexus/Cargo.toml b/nexus/Cargo.toml index 1fc4d00abf..d430009360 100644 --- a/nexus/Cargo.toml +++ b/nexus/Cargo.toml @@ -22,6 +22,8 @@ camino.workspace = true camino-tempfile.workspace = true clap.workspace = true chrono.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true cockroach-admin-client.workspace = true crucible-agent-client.workspace = true crucible-pantry-client.workspace = true @@ -40,7 +42,8 @@ http.workspace = true http-body-util.workspace = true hyper.workspace = true illumos-utils.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true ipnetwork.workspace = true itertools.workspace = true macaddr.workspace = true @@ -66,6 +69,7 @@ paste.workspace = true pq-sys = "*" progenitor-client.workspace = true propolis-client.workspace = true +qorb.workspace = true rand.workspace = true ref-cast.workspace = true reqwest = { workspace = true, features = ["json"] } diff --git a/nexus/auth/src/authz/api_resources.rs b/nexus/auth/src/authz/api_resources.rs index f4c91dc544..df1a1efc29 100644 --- a/nexus/auth/src/authz/api_resources.rs +++ b/nexus/auth/src/authz/api_resources.rs @@ -754,6 +754,30 @@ authz_resource! { polar_snippet = InProject, } +authz_resource! { + name = "InternetGateway", + parent = "Vpc", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + +authz_resource! { + name = "InternetGatewayIpPool", + parent = "InternetGateway", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + +authz_resource! { + name = "InternetGatewayIpAddress", + parent = "InternetGateway", + primary_key = Uuid, + roles_allowed = false, + polar_snippet = InProject, +} + authz_resource! { name = "FloatingIp", parent = "Project", diff --git a/nexus/auth/src/authz/oso_generic.rs b/nexus/auth/src/authz/oso_generic.rs index 383a06e985..acd74b2167 100644 --- a/nexus/auth/src/authz/oso_generic.rs +++ b/nexus/auth/src/authz/oso_generic.rs @@ -130,6 +130,9 @@ pub fn make_omicron_oso(log: &slog::Logger) -> Result { InstanceNetworkInterface::init(), Vpc::init(), VpcRouter::init(), + InternetGateway::init(), + InternetGatewayIpPool::init(), + InternetGatewayIpAddress::init(), RouterRoute::init(), VpcSubnet::init(), FloatingIp::init(), diff --git a/nexus/db-fixed-data/src/project.rs b/nexus/db-fixed-data/src/project.rs index 6b9f005916..7784390820 100644 --- a/nexus/db-fixed-data/src/project.rs +++ b/nexus/db-fixed-data/src/project.rs @@ -3,7 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use nexus_db_model as model; -use nexus_types::external_api::params; +use nexus_types::{external_api::params, silo::INTERNAL_SILO_ID}; use omicron_common::api::external::IdentityMetadataCreateParams; use once_cell::sync::Lazy; @@ -21,7 +21,7 @@ pub static SERVICES_PROJECT_ID: Lazy = Lazy::new(|| { pub static SERVICES_PROJECT: Lazy = Lazy::new(|| { model::Project::new_with_id( *SERVICES_PROJECT_ID, - *super::silo::INTERNAL_SILO_ID, + INTERNAL_SILO_ID, params::ProjectCreate { identity: IdentityMetadataCreateParams { name: SERVICES_DB_NAME.parse().unwrap(), diff --git a/nexus/db-fixed-data/src/silo.rs b/nexus/db-fixed-data/src/silo.rs index ebc6776923..10c624e30e 100644 --- a/nexus/db-fixed-data/src/silo.rs +++ b/nexus/db-fixed-data/src/silo.rs @@ -3,26 +3,26 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use nexus_db_model as model; -use nexus_types::external_api::{params, shared}; +use nexus_types::{ + external_api::{params, shared}, + silo::{ + default_silo_name, internal_silo_name, DEFAULT_SILO_ID, + INTERNAL_SILO_ID, + }, +}; use omicron_common::api::external::IdentityMetadataCreateParams; use once_cell::sync::Lazy; -pub static DEFAULT_SILO_ID: Lazy = Lazy::new(|| { - "001de000-5110-4000-8000-000000000000" - .parse() - .expect("invalid uuid for builtin silo id") -}); - /// "Default" Silo /// /// This was historically used for demos and the unit tests. The plan is to /// remove it per omicron#2305. pub static DEFAULT_SILO: Lazy = Lazy::new(|| { model::Silo::new_with_id( - *DEFAULT_SILO_ID, + DEFAULT_SILO_ID, params::SiloCreate { identity: IdentityMetadataCreateParams { - name: "default-silo".parse().unwrap(), + name: default_silo_name().clone(), description: "default silo".to_string(), }, // This quota is actually _unused_ because the default silo @@ -38,21 +38,14 @@ pub static DEFAULT_SILO: Lazy = Lazy::new(|| { .unwrap() }); -/// UUID of built-in internal silo. -pub static INTERNAL_SILO_ID: Lazy = Lazy::new(|| { - "001de000-5110-4000-8000-000000000001" - .parse() - .expect("invalid uuid for builtin silo id") -}); - /// Built-in Silo to house internal resources. It contains no users and /// can't be logged into. pub static INTERNAL_SILO: Lazy = Lazy::new(|| { model::Silo::new_with_id( - *INTERNAL_SILO_ID, + INTERNAL_SILO_ID, params::SiloCreate { identity: IdentityMetadataCreateParams { - name: "oxide-internal".parse().unwrap(), + name: internal_silo_name().clone(), description: "Built-in internal Silo.".to_string(), }, // The internal silo contains no virtual resources, so it has no allotted capacity. diff --git a/nexus/db-fixed-data/src/silo_user.rs b/nexus/db-fixed-data/src/silo_user.rs index defaa9bd52..e6e6d7d0e5 100644 --- a/nexus/db-fixed-data/src/silo_user.rs +++ b/nexus/db-fixed-data/src/silo_user.rs @@ -5,7 +5,7 @@ use super::role_builtin; use nexus_db_model as model; -use nexus_types::identity::Asset; +use nexus_types::{identity::Asset, silo::DEFAULT_SILO_ID}; use once_cell::sync::Lazy; /// Test user that's granted all privileges, used for automated testing and @@ -15,7 +15,7 @@ use once_cell::sync::Lazy; // not automatically at Nexus startup. See omicron#2305. pub static USER_TEST_PRIVILEGED: Lazy = Lazy::new(|| { model::SiloUser::new( - *crate::silo::DEFAULT_SILO_ID, + DEFAULT_SILO_ID, // "4007" looks a bit like "root". "001de000-05e4-4000-8000-000000004007".parse().unwrap(), "privileged".into(), @@ -39,7 +39,7 @@ pub static ROLE_ASSIGNMENTS_PRIVILEGED: Lazy> = model::IdentityType::SiloUser, USER_TEST_PRIVILEGED.id(), role_builtin::SILO_ADMIN.resource_type, - *crate::silo::DEFAULT_SILO_ID, + DEFAULT_SILO_ID, role_builtin::SILO_ADMIN.role_name, ), ] @@ -51,7 +51,7 @@ pub static ROLE_ASSIGNMENTS_PRIVILEGED: Lazy> = // not automatically at Nexus startup. See omicron#2305. pub static USER_TEST_UNPRIVILEGED: Lazy = Lazy::new(|| { model::SiloUser::new( - *crate::silo::DEFAULT_SILO_ID, + DEFAULT_SILO_ID, // 60001 is the decimal uid for "nobody" on Helios. "001de000-05e4-4000-8000-000000060001".parse().unwrap(), "unprivileged".into(), diff --git a/nexus/db-fixed-data/src/vpc.rs b/nexus/db-fixed-data/src/vpc.rs index d5940a976e..64a2563305 100644 --- a/nexus/db-fixed-data/src/vpc.rs +++ b/nexus/db-fixed-data/src/vpc.rs @@ -23,20 +23,27 @@ pub static SERVICES_VPC_ROUTER_ID: Lazy = Lazy::new(|| { .expect("invalid uuid for builtin services vpc router id") }); -/// UUID of default IPv4 route for built-in Services VPC. -pub static SERVICES_VPC_DEFAULT_V4_ROUTE_ID: Lazy = +/// UUID of InternetGateway for built-in Services VPC. +pub static SERVICES_INTERNET_GATEWAY_ID: Lazy = Lazy::new(|| { + "001de000-074c-4000-8000-000000000002" + .parse() + .expect("invalid uuid for builtin services internet gateway id") +}); + +/// UUID of InternetGateway IPv4 default route for built-in Services VPC. +pub static SERVICES_INTERNET_GATEWAY_DEFAULT_ROUTE_V4: Lazy = Lazy::new(|| { - "001de000-074c-4000-8000-000000000002" - .parse() - .expect("invalid uuid for builtin services vpc default route id") + "001de000-074c-4000-8000-000000000003" + .parse() + .expect("invalid uuid for builtin services internet gateway default route v4") }); -/// UUID of default IPv6 route for built-in Services VPC. -pub static SERVICES_VPC_DEFAULT_V6_ROUTE_ID: Lazy = +/// UUID of InternetGateway IPv6 default route for built-in Services VPC. +pub static SERVICES_INTERNET_GATEWAY_DEFAULT_ROUTE_V6: Lazy = Lazy::new(|| { - "001de000-074c-4000-8000-000000000003" - .parse() - .expect("invalid uuid for builtin services vpc default route id") + "001de000-074c-4000-8000-000000000004" + .parse() + .expect("invalid uuid for builtin services internet gateway default route v4") }); /// Built-in VPC for internal services on the rack. diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs index fab129e6ba..500d72dc9a 100644 --- a/nexus/db-model/src/deployment.rs +++ b/nexus/db-model/src/deployment.rs @@ -813,18 +813,20 @@ impl BpClickhouseClusterConfig { .max_used_server_id .0 .try_into() - .context("more than 2^63 IDs in use")?, + .context("more than 2^63 clickhouse server IDs in use")?, max_used_keeper_id: config .max_used_keeper_id .0 .try_into() - .context("more than 2^63 IDs in use")?, + .context("more than 2^63 clickhouse keeper IDs in use")?, cluster_name: config.cluster_name.clone(), cluster_secret: config.cluster_secret.clone(), highest_seen_keeper_leader_committed_log_index: config .highest_seen_keeper_leader_committed_log_index .try_into() - .context("more than 2^63 IDs in use")?, + .context( + "more than 2^63 clickhouse keeper log indexes in use", + )?, }) } } diff --git a/nexus/db-model/src/instance.rs b/nexus/db-model/src/instance.rs index 3bc9d3f993..e7aa989971 100644 --- a/nexus/db-model/src/instance.rs +++ b/nexus/db-model/src/instance.rs @@ -417,35 +417,38 @@ impl InstanceAutoRestart { // N.B. that this may become more complex in the future if we grow // additional auto-restart policies that require additional logic // (such as restart limits...) - dsl::auto_restart_policy + (dsl::auto_restart_policy .eq(InstanceAutoRestartPolicy::BestEffort) - // An instance whose last reincarnation was within the cooldown - // interval from now must remain in _bardo_ --- the liminal - // state between death and rebirth --- before its next - // reincarnation. - .and( - // If the instance has never previously been reincarnated, then - // it's allowed to reincarnate. - dsl::time_last_auto_restarted - .is_null() - // Or, if it has an overridden cooldown period, has that elapsed? - .or(dsl::auto_restart_cooldown.is_not_null().and( - dsl::time_last_auto_restarted - .le(now.nullable() - dsl::auto_restart_cooldown), - )) - // Or, finally, if it does not have an overridden cooldown - // period, has the default cooldown period elapsed? - .or(dsl::auto_restart_cooldown.is_null().and( - dsl::time_last_auto_restarted - .le((now - Self::DEFAULT_COOLDOWN).nullable()), - )), - ) - // Deleted instances may not be reincarnated. - .and(dsl::time_deleted.is_null()) - // If the instance is currently in the process of being updated, - // let's not mess with it for now and try to restart it on another - // pass. - .and(dsl::updater_id.is_null()) + // If the auto-restart policy is null, then it should + // default to "best effort". + .or(dsl::auto_restart_policy.is_null())) + // An instance whose last reincarnation was within the cooldown + // interval from now must remain in _bardo_ --- the liminal + // state between death and rebirth --- before its next + // reincarnation. + .and( + // If the instance has never previously been reincarnated, then + // it's allowed to reincarnate. + dsl::time_last_auto_restarted + .is_null() + // Or, if it has an overridden cooldown period, has that elapsed? + .or(dsl::auto_restart_cooldown.is_not_null().and( + dsl::time_last_auto_restarted + .le(now.nullable() - dsl::auto_restart_cooldown), + )) + // Or, finally, if it does not have an overridden cooldown + // period, has the default cooldown period elapsed? + .or(dsl::auto_restart_cooldown.is_null().and( + dsl::time_last_auto_restarted + .le((now - Self::DEFAULT_COOLDOWN).nullable()), + )), + ) + // Deleted instances may not be reincarnated. + .and(dsl::time_deleted.is_null()) + // If the instance is currently in the process of being updated, + // let's not mess with it for now and try to restart it on another + // pass. + .and(dsl::updater_id.is_null()) } } @@ -534,4 +537,9 @@ mod optional_time_delta { pub struct InstanceUpdate { #[diesel(column_name = boot_disk_id)] pub boot_disk_id: Option, + + /// The auto-restart policy for this instance. If this is `None`, it will + /// set the instance's auto-restart policy to `NULL`. + #[diesel(column_name = auto_restart_policy)] + pub auto_restart_policy: Option, } diff --git a/nexus/db-model/src/internet_gateway.rs b/nexus/db-model/src/internet_gateway.rs new file mode 100644 index 0000000000..6ead292fc5 --- /dev/null +++ b/nexus/db-model/src/internet_gateway.rs @@ -0,0 +1,124 @@ +use super::Generation; +use crate::schema::{ + internet_gateway, internet_gateway_ip_address, internet_gateway_ip_pool, +}; +use crate::DatastoreCollectionConfig; +use db_macros::Resource; +use ipnetwork::IpNetwork; +use nexus_types::external_api::{params, views}; +use nexus_types::identity::Resource; +use omicron_common::api::external::IdentityMetadataCreateParams; +use uuid::Uuid; + +#[derive(Queryable, Insertable, Clone, Debug, Selectable, Resource)] +#[diesel(table_name = internet_gateway)] +pub struct InternetGateway { + #[diesel(embed)] + identity: InternetGatewayIdentity, + + pub vpc_id: Uuid, + pub rcgen: Generation, + pub resolved_version: i64, +} + +impl InternetGateway { + pub fn new( + gateway_id: Uuid, + vpc_id: Uuid, + params: params::InternetGatewayCreate, + ) -> Self { + let identity = + InternetGatewayIdentity::new(gateway_id, params.identity); + Self { identity, vpc_id, rcgen: Generation::new(), resolved_version: 0 } + } +} + +impl From for views::InternetGateway { + fn from(value: InternetGateway) -> Self { + Self { identity: value.identity(), vpc_id: value.vpc_id } + } +} + +#[derive(Queryable, Insertable, Clone, Debug, Selectable, Resource)] +#[diesel(table_name = internet_gateway_ip_pool)] +pub struct InternetGatewayIpPool { + #[diesel(embed)] + identity: InternetGatewayIpPoolIdentity, + + pub internet_gateway_id: Uuid, + pub ip_pool_id: Uuid, +} + +impl InternetGatewayIpPool { + pub fn new( + id: Uuid, + ip_pool_id: Uuid, + internet_gateway_id: Uuid, + identity: IdentityMetadataCreateParams, + ) -> Self { + let identity = InternetGatewayIpPoolIdentity::new(id, identity); + Self { identity, internet_gateway_id, ip_pool_id } + } +} + +impl From for views::InternetGatewayIpPool { + fn from(value: InternetGatewayIpPool) -> Self { + Self { + identity: value.identity(), + internet_gateway_id: value.internet_gateway_id, + ip_pool_id: value.ip_pool_id, + } + } +} + +#[derive(Queryable, Insertable, Clone, Debug, Selectable, Resource)] +#[diesel(table_name = internet_gateway_ip_address)] +pub struct InternetGatewayIpAddress { + #[diesel(embed)] + identity: InternetGatewayIpAddressIdentity, + + pub internet_gateway_id: Uuid, + pub address: IpNetwork, +} + +impl InternetGatewayIpAddress { + pub fn new( + pool_id: Uuid, + internet_gateway_id: Uuid, + params: params::InternetGatewayIpAddressCreate, + ) -> Self { + let identity = + InternetGatewayIpAddressIdentity::new(pool_id, params.identity); + Self { + identity, + internet_gateway_id, + address: IpNetwork::from(params.address), + } + } +} + +impl From for views::InternetGatewayIpAddress { + fn from(value: InternetGatewayIpAddress) -> Self { + Self { + identity: value.identity(), + internet_gateway_id: value.internet_gateway_id, + address: value.address.ip(), + } + } +} + +impl DatastoreCollectionConfig for InternetGateway { + type CollectionId = Uuid; + type GenerationNumberColumn = internet_gateway::dsl::rcgen; + type CollectionTimeDeletedColumn = internet_gateway::dsl::time_deleted; + type CollectionIdColumn = + internet_gateway_ip_pool::dsl::internet_gateway_id; +} + +impl DatastoreCollectionConfig for InternetGateway { + type CollectionId = Uuid; + type GenerationNumberColumn = internet_gateway::dsl::rcgen; + type CollectionTimeDeletedColumn = internet_gateway::dsl::time_deleted; + type CollectionIdColumn = + internet_gateway_ip_address::dsl::internet_gateway_id; +} diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs index 79cf303cfd..95f7e8b5f7 100644 --- a/nexus/db-model/src/inventory.rs +++ b/nexus/db-model/src/inventory.rs @@ -6,11 +6,12 @@ use crate::omicron_zone_config::{self, OmicronZoneNic}; use crate::schema::{ - hw_baseboard_id, inv_caboose, inv_collection, inv_collection_error, - inv_dataset, inv_nvme_disk_firmware, inv_omicron_zone, - inv_omicron_zone_nic, inv_physical_disk, inv_root_of_trust, - inv_root_of_trust_page, inv_service_processor, inv_sled_agent, - inv_sled_omicron_zones, inv_zpool, sw_caboose, sw_root_of_trust_page, + hw_baseboard_id, inv_caboose, inv_clickhouse_keeper_membership, + inv_collection, inv_collection_error, inv_dataset, inv_nvme_disk_firmware, + inv_omicron_zone, inv_omicron_zone_nic, inv_physical_disk, + inv_root_of_trust, inv_root_of_trust_page, inv_service_processor, + inv_sled_agent, inv_sled_omicron_zones, inv_zpool, sw_caboose, + sw_root_of_trust_page, }; use crate::typed_uuid::DbTypedUuid; use crate::PhysicalDiskKind; @@ -21,6 +22,7 @@ use crate::{ use anyhow::{anyhow, bail, Context, Result}; use chrono::DateTime; use chrono::Utc; +use clickhouse_admin_types::{ClickhouseKeeperClusterMembership, KeeperId}; use diesel::backend::Backend; use diesel::deserialize::{self, FromSql}; use diesel::expression::AsExpression; @@ -29,9 +31,7 @@ use diesel::serialize::ToSql; use diesel::{serialize, sql_types}; use ipnetwork::IpNetwork; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; -use nexus_sled_agent_shared::inventory::{ - OmicronZoneConfig, OmicronZoneType, OmicronZonesConfig, -}; +use nexus_sled_agent_shared::inventory::{OmicronZoneConfig, OmicronZoneType}; use nexus_types::inventory::{ BaseboardId, Caboose, Collection, NvmeFirmware, PowerState, RotPage, RotSlot, @@ -46,6 +46,7 @@ use omicron_uuid_kinds::ZpoolKind; use omicron_uuid_kinds::ZpoolUuid; use omicron_uuid_kinds::{CollectionKind, OmicronZoneKind}; use omicron_uuid_kinds::{CollectionUuid, OmicronZoneUuid}; +use std::collections::BTreeSet; use std::net::{IpAddr, SocketAddrV6}; use thiserror::Error; use uuid::Uuid; @@ -1175,7 +1176,11 @@ impl From for nexus_types::inventory::Dataset { } } -/// See [`nexus_types::inventory::OmicronZonesFound`]. +/// Information about a sled's Omicron zones, part of +/// [`nexus_types::inventory::SledAgent`]. +/// +/// TODO: This table is vestigial and can be combined with `InvSledAgent`. See +/// [issue #6770](https://github.com/oxidecomputer/omicron/issues/6770). #[derive(Queryable, Clone, Debug, Selectable, Insertable)] #[diesel(table_name = inv_sled_omicron_zones)] pub struct InvSledOmicronZones { @@ -1189,28 +1194,14 @@ pub struct InvSledOmicronZones { impl InvSledOmicronZones { pub fn new( inv_collection_id: CollectionUuid, - zones_found: &nexus_types::inventory::OmicronZonesFound, + sled_agent: &nexus_types::inventory::SledAgent, ) -> InvSledOmicronZones { InvSledOmicronZones { inv_collection_id: inv_collection_id.into(), - time_collected: zones_found.time_collected, - source: zones_found.source.clone(), - sled_id: zones_found.sled_id.into(), - generation: Generation(zones_found.zones.generation), - } - } - - pub fn into_uninit_zones_found( - self, - ) -> nexus_types::inventory::OmicronZonesFound { - nexus_types::inventory::OmicronZonesFound { - time_collected: self.time_collected, - source: self.source, - sled_id: self.sled_id.into(), - zones: OmicronZonesConfig { - generation: *self.generation, - zones: Vec::new(), - }, + time_collected: sled_agent.time_collected, + source: sled_agent.source.clone(), + sled_id: sled_agent.sled_id.into(), + generation: Generation(sled_agent.omicron_zones.generation), } } } @@ -1726,6 +1717,68 @@ impl InvOmicronZoneNic { } } +#[derive(Queryable, Clone, Debug, Selectable, Insertable)] +#[diesel(table_name = inv_clickhouse_keeper_membership)] +pub struct InvClickhouseKeeperMembership { + pub inv_collection_id: DbTypedUuid, + pub queried_keeper_id: i64, + pub leader_committed_log_index: i64, + pub raft_config: Vec, +} + +impl TryFrom + for ClickhouseKeeperClusterMembership +{ + type Error = anyhow::Error; + + fn try_from(value: InvClickhouseKeeperMembership) -> anyhow::Result { + let err_msg = "clickhouse keeper ID is negative"; + let mut raft_config = BTreeSet::new(); + // We are not worried about duplicates here, as each + // `clickhouse-admin-keeper` reports about its local, unique keeper. + // This uniqueness is guaranteed by the blueprint generation mechanism. + for id in value.raft_config { + raft_config.insert(KeeperId(id.try_into().context(err_msg)?)); + } + Ok(ClickhouseKeeperClusterMembership { + queried_keeper: KeeperId( + value.queried_keeper_id.try_into().context(err_msg)?, + ), + leader_committed_log_index: value + .leader_committed_log_index + .try_into() + .context("log index is negative")?, + raft_config, + }) + } +} + +impl InvClickhouseKeeperMembership { + pub fn new( + inv_collection_id: CollectionUuid, + membership: ClickhouseKeeperClusterMembership, + ) -> anyhow::Result { + let err_msg = "clickhouse keeper ID > 2^63"; + let mut raft_config = Vec::with_capacity(membership.raft_config.len()); + for id in membership.raft_config { + raft_config.push(id.0.try_into().context(err_msg)?); + } + Ok(InvClickhouseKeeperMembership { + inv_collection_id: inv_collection_id.into(), + queried_keeper_id: membership + .queried_keeper + .0 + .try_into() + .context(err_msg)?, + leader_committed_log_index: membership + .leader_committed_log_index + .try_into() + .context("log index > 2^63")?, + raft_config, + }) + } +} + #[cfg(test)] mod test { use nexus_types::inventory::NvmeFirmware; diff --git a/nexus/db-model/src/lib.rs b/nexus/db-model/src/lib.rs index 250fbcb369..001c97b6f6 100644 --- a/nexus/db-model/src/lib.rs +++ b/nexus/db-model/src/lib.rs @@ -37,6 +37,7 @@ mod instance; mod instance_auto_restart_policy; mod instance_cpu_count; mod instance_state; +mod internet_gateway; mod inventory; mod ip_pool; mod ipv4net; @@ -153,6 +154,7 @@ pub use instance::*; pub use instance_auto_restart_policy::*; pub use instance_cpu_count::*; pub use instance_state::*; +pub use internet_gateway::*; pub use inventory::*; pub use ip_pool::*; pub use ipv4_nat_entry::*; diff --git a/nexus/db-model/src/oximeter_info.rs b/nexus/db-model/src/oximeter_info.rs index 5579425a63..017c1bf0ba 100644 --- a/nexus/db-model/src/oximeter_info.rs +++ b/nexus/db-model/src/oximeter_info.rs @@ -9,7 +9,9 @@ use nexus_types::internal_api; use uuid::Uuid; /// A record representing a registered `oximeter` collector. -#[derive(Queryable, Insertable, Debug, Clone, Copy, PartialEq, Eq)] +#[derive( + Queryable, Insertable, Selectable, Debug, Clone, Copy, PartialEq, Eq, +)] #[diesel(table_name = oximeter)] pub struct OximeterInfo { /// The ID for this oximeter instance. diff --git a/nexus/db-model/src/region_replacement.rs b/nexus/db-model/src/region_replacement.rs index 995c55001c..57ead4b68e 100644 --- a/nexus/db-model/src/region_replacement.rs +++ b/nexus/db-model/src/region_replacement.rs @@ -116,6 +116,13 @@ impl std::str::FromStr for RegionReplacementState { /// "finish" notification is seen by the region replacement drive background /// task. This check is done before invoking the region replacement drive saga. /// +/// If the volume whose region is being replaced is soft-deleted or +/// hard-deleted, then the replacement request will be transitioned along the +/// states to Complete while avoiding operations that are meant to operate on +/// that volume. If the volume is soft-deleted or hard-deleted while the +/// replacement request is in the "Requested" state, the replacement request +/// will transition straight to Complete, and no operations will be performed. +/// /// See also: RegionReplacementStep records #[derive( Queryable, diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index d9e2c43e75..a51fd04c8e 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -188,7 +188,7 @@ table! { dst -> Inet, gw -> Inet, vid -> Nullable, - local_pref -> Nullable, + local_pref -> Nullable, } } @@ -1147,6 +1147,46 @@ table! { } } +table! { + internet_gateway(id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + vpc_id -> Uuid, + rcgen -> Int8, + resolved_version -> Int8, + } +} + +table! { + internet_gateway_ip_pool(id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + internet_gateway_id -> Uuid, + ip_pool_id -> Uuid, + } +} + +table! { + internet_gateway_ip_address(id) { + id -> Uuid, + name -> Text, + description -> Text, + time_created -> Timestamptz, + time_modified -> Timestamptz, + time_deleted -> Nullable, + internet_gateway_id -> Uuid, + address -> Inet, + } +} + table! { use diesel::sql_types::*; @@ -1528,6 +1568,15 @@ table! { } } +table! { + inv_clickhouse_keeper_membership (inv_collection_id, queried_keeper_id) { + inv_collection_id -> Uuid, + queried_keeper_id -> Int8, + leader_committed_log_index -> Int8, + raft_config -> Array, + } +} + /* blueprints */ table! { @@ -1936,6 +1985,9 @@ allow_tables_to_appear_in_same_query!( role_builtin, role_assignment, probe, + internet_gateway, + internet_gateway_ip_pool, + internet_gateway_ip_address, ); allow_tables_to_appear_in_same_query!(dns_zone, dns_version, dns_name); @@ -1944,6 +1996,20 @@ allow_tables_to_appear_in_same_query!(dns_zone, dns_version, dns_name); allow_tables_to_appear_in_same_query!(external_ip, instance); allow_tables_to_appear_in_same_query!(external_ip, project); allow_tables_to_appear_in_same_query!(external_ip, ip_pool_resource); +allow_tables_to_appear_in_same_query!(external_ip, vmm); +allow_tables_to_appear_in_same_query!(external_ip, network_interface); +allow_tables_to_appear_in_same_query!(external_ip, inv_omicron_zone); +allow_tables_to_appear_in_same_query!(external_ip, inv_omicron_zone_nic); +allow_tables_to_appear_in_same_query!(inv_omicron_zone, inv_omicron_zone_nic); +allow_tables_to_appear_in_same_query!(network_interface, inv_omicron_zone); +allow_tables_to_appear_in_same_query!(network_interface, inv_omicron_zone_nic); +allow_tables_to_appear_in_same_query!(network_interface, inv_collection); +allow_tables_to_appear_in_same_query!(inv_omicron_zone, inv_collection); +allow_tables_to_appear_in_same_query!(inv_omicron_zone_nic, inv_collection); +allow_tables_to_appear_in_same_query!(external_ip, inv_collection); +allow_tables_to_appear_in_same_query!(external_ip, internet_gateway); +allow_tables_to_appear_in_same_query!(external_ip, internet_gateway_ip_pool); +allow_tables_to_appear_in_same_query!(external_ip, internet_gateway_ip_address); allow_tables_to_appear_in_same_query!( switch_port, diff --git a/nexus/db-model/src/schema_versions.rs b/nexus/db-model/src/schema_versions.rs index 12e03e6d4e..d1d82b25fb 100644 --- a/nexus/db-model/src/schema_versions.rs +++ b/nexus/db-model/src/schema_versions.rs @@ -17,7 +17,7 @@ use std::collections::BTreeMap; /// /// This must be updated when you change the database schema. Refer to /// schema/crdb/README.adoc in the root of this repository for details. -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(107, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(109, 0, 0); /// List of all past database schema versions, in *reverse* order /// @@ -29,6 +29,8 @@ static KNOWN_VERSIONS: Lazy> = Lazy::new(|| { // | leaving the first copy as an example for the next person. // v // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"), + KnownVersion::new(109, "inv-clickhouse-keeper-membership"), + KnownVersion::new(108, "internet-gateway"), KnownVersion::new(107, "add-instance-boot-disk"), KnownVersion::new(106, "dataset-kinds-update"), KnownVersion::new(105, "inventory-nvme-firmware"), diff --git a/nexus/db-model/src/sled.rs b/nexus/db-model/src/sled.rs index ca2b292711..b586ad0fc5 100644 --- a/nexus/db-model/src/sled.rs +++ b/nexus/db-model/src/sled.rs @@ -11,11 +11,13 @@ use crate::sled_policy::DbSledPolicy; use chrono::{DateTime, Utc}; use db_macros::Asset; use nexus_sled_agent_shared::inventory::SledRole; +use nexus_types::deployment::execution; use nexus_types::{ external_api::{shared, views}, identity::Asset, internal_api::params, }; +use omicron_uuid_kinds::{GenericUuid, SledUuid}; use std::net::Ipv6Addr; use std::net::SocketAddrV6; use uuid::Uuid; @@ -140,6 +142,20 @@ impl From for views::Sled { } } +impl From for execution::Sled { + fn from(sled: Sled) -> Self { + Self::new( + SledUuid::from_untyped_uuid(sled.id()), + sled.address(), + if sled.is_scrimlet { + SledRole::Scrimlet + } else { + SledRole::Gimlet + }, + ) + } +} + impl From for params::SledAgentInfo { fn from(sled: Sled) -> Self { let role = if sled.is_scrimlet { diff --git a/nexus/db-model/src/switch_port.rs b/nexus/db-model/src/switch_port.rs index f91f107a11..2420482cce 100644 --- a/nexus/db-model/src/switch_port.rs +++ b/nexus/db-model/src/switch_port.rs @@ -559,7 +559,8 @@ pub struct SwitchPortRouteConfig { pub dst: IpNetwork, pub gw: IpNetwork, pub vid: Option, - pub local_pref: Option, + #[diesel(column_name = local_pref)] + pub rib_priority: Option, } impl SwitchPortRouteConfig { @@ -569,9 +570,9 @@ impl SwitchPortRouteConfig { dst: IpNetwork, gw: IpNetwork, vid: Option, - local_pref: Option, + rib_priority: Option, ) -> Self { - Self { port_settings_id, interface_name, dst, gw, vid, local_pref } + Self { port_settings_id, interface_name, dst, gw, vid, rib_priority } } } @@ -583,7 +584,7 @@ impl Into for SwitchPortRouteConfig { dst: self.dst.into(), gw: self.gw.into(), vlan_id: self.vid.map(Into::into), - local_pref: self.local_pref.map(Into::into), + rib_priority: self.rib_priority.map(Into::into), } } } diff --git a/nexus/db-queries/Cargo.toml b/nexus/db-queries/Cargo.toml index c6c5caab6a..a8d7983959 100644 --- a/nexus/db-queries/Cargo.toml +++ b/nexus/db-queries/Cargo.toml @@ -16,12 +16,14 @@ async-bb8-diesel.workspace = true async-trait.workspace = true camino.workspace = true chrono.workspace = true +clickhouse-admin-types.workspace = true const_format.workspace = true diesel.workspace = true diesel-dtrace.workspace = true dropshot.workspace = true futures.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true ipnetwork.workspace = true macaddr.workspace = true once_cell.workspace = true @@ -77,7 +79,7 @@ expectorate.workspace = true hyper-rustls.workspace = true gateway-client.workspace = true illumos-utils.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true itertools.workspace = true nexus-inventory.workspace = true nexus-reconfigurator-planning.workspace = true diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 07c32ef818..e43a50a291 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -20,6 +20,7 @@ use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use chrono::DateTime; use chrono::Utc; +use clickhouse_admin_types::{KeeperId, ServerId}; use diesel::expression::SelectableHelper; use diesel::pg::Pg; use diesel::query_builder::AstPass; @@ -36,6 +37,9 @@ use diesel::OptionalExtension; use diesel::QueryDsl; use diesel::RunQueryDsl; use nexus_db_model::Blueprint as DbBlueprint; +use nexus_db_model::BpClickhouseClusterConfig; +use nexus_db_model::BpClickhouseKeeperZoneIdToNodeId; +use nexus_db_model::BpClickhouseServerZoneIdToNodeId; use nexus_db_model::BpOmicronPhysicalDisk; use nexus_db_model::BpOmicronZone; use nexus_db_model::BpOmicronZoneNic; @@ -49,6 +53,7 @@ use nexus_types::deployment::BlueprintPhysicalDisksConfig; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::BlueprintZonesConfig; +use nexus_types::deployment::ClickhouseClusterConfig; use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::external_api::views::SledState; use omicron_common::api::external::DataPageParams; @@ -58,6 +63,7 @@ use omicron_common::api::external::LookupType; use omicron_common::api::external::ResourceType; use omicron_common::bail_unless; use omicron_uuid_kinds::GenericUuid; +use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use std::collections::BTreeMap; use uuid::Uuid; @@ -180,6 +186,42 @@ impl DataStore { }) .collect::, _>>()?; + let clickhouse_tables: Option<(_, _, _)> = if let Some(config) = + &blueprint.clickhouse_cluster_config + { + let mut keepers = vec![]; + for (zone_id, keeper_id) in &config.keepers { + let keeper = BpClickhouseKeeperZoneIdToNodeId::new( + blueprint_id, + *zone_id, + *keeper_id, + ) + .with_context(|| format!("zone {zone_id}, keeper {keeper_id}")) + .map_err(|e| Error::internal_error(&format!("{:#}", e)))?; + keepers.push(keeper) + } + + let mut servers = vec![]; + for (zone_id, server_id) in &config.servers { + let server = BpClickhouseServerZoneIdToNodeId::new( + blueprint_id, + *zone_id, + *server_id, + ) + .with_context(|| format!("zone {zone_id}, server {server_id}")) + .map_err(|e| Error::internal_error(&format!("{:#}", e)))?; + servers.push(server); + } + + let cluster_config = + BpClickhouseClusterConfig::new(blueprint_id, config) + .map_err(|e| Error::internal_error(&format!("{:#}", e)))?; + + Some((cluster_config, keepers, servers)) + } else { + None + }; + // This implementation inserts all records associated with the // blueprint in one transaction. This is required: we don't want // any planner or executor to see a half-inserted blueprint, nor do we @@ -258,7 +300,33 @@ impl DataStore { .await?; } + // Insert all clickhouse cluster related tables if necessary + if let Some((clickhouse_cluster_config, keepers, servers)) = clickhouse_tables { + { + use db::schema::bp_clickhouse_cluster_config::dsl; + let _ = diesel::insert_into(dsl::bp_clickhouse_cluster_config) + .values(clickhouse_cluster_config) + .execute_async(&conn) + .await?; + } + { + use db::schema::bp_clickhouse_keeper_zone_id_to_node_id::dsl; + let _ = diesel::insert_into(dsl::bp_clickhouse_keeper_zone_id_to_node_id) + .values(keepers) + .execute_async(&conn) + .await?; + } + { + use db::schema::bp_clickhouse_server_zone_id_to_node_id::dsl; + let _ = diesel::insert_into(dsl::bp_clickhouse_server_zone_id_to_node_id) + .values(servers) + .execute_async(&conn) + .await?; + } + } + Ok(()) + }) .await .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; @@ -622,6 +690,156 @@ impl DataStore { disks_config.disks.sort_unstable_by_key(|d| d.id); } + // Load our `ClickhouseClusterConfig` if it exists + let clickhouse_cluster_config: Option = { + use db::schema::bp_clickhouse_cluster_config::dsl; + + let res = dsl::bp_clickhouse_cluster_config + .filter(dsl::blueprint_id.eq(blueprint_id)) + .select(BpClickhouseClusterConfig::as_select()) + .get_result_async(&*conn) + .await + .optional() + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + match res { + None => None, + Some(bp_config) => { + // Load our clickhouse keeper configs for the given blueprint + let keepers: BTreeMap = { + use db::schema::bp_clickhouse_keeper_zone_id_to_node_id::dsl; + let mut keepers = BTreeMap::new(); + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + let batch = paginated( + dsl::bp_clickhouse_keeper_zone_id_to_node_id, + dsl::omicron_zone_id, + &p.current_pagparams(), + ) + .filter(dsl::blueprint_id.eq(blueprint_id)) + .select( + BpClickhouseKeeperZoneIdToNodeId::as_select(), + ) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + })?; + + paginator = + p.found_batch(&batch, &|k| k.omicron_zone_id); + + for k in batch { + let keeper_id = KeeperId( + u64::try_from(k.keeper_id).map_err( + |_| { + Error::internal_error(&format!( + "keeper id is negative: {}", + k.keeper_id + )) + }, + )?, + ); + keepers.insert( + k.omicron_zone_id.into(), + keeper_id, + ); + } + } + keepers + }; + + // Load our clickhouse server configs for the given blueprint + let servers: BTreeMap = { + use db::schema::bp_clickhouse_server_zone_id_to_node_id::dsl; + let mut servers = BTreeMap::new(); + let mut paginator = Paginator::new(SQL_BATCH_SIZE); + while let Some(p) = paginator.next() { + let batch = paginated( + dsl::bp_clickhouse_server_zone_id_to_node_id, + dsl::omicron_zone_id, + &p.current_pagparams(), + ) + .filter(dsl::blueprint_id.eq(blueprint_id)) + .select( + BpClickhouseServerZoneIdToNodeId::as_select(), + ) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::Server, + ) + })?; + + paginator = + p.found_batch(&batch, &|s| s.omicron_zone_id); + + for s in batch { + let server_id = ServerId( + u64::try_from(s.server_id).map_err( + |_| { + Error::internal_error(&format!( + "server id is negative: {}", + s.server_id + )) + }, + )?, + ); + servers.insert( + s.omicron_zone_id.into(), + server_id, + ); + } + } + servers + }; + + Some(ClickhouseClusterConfig { + generation: bp_config.generation.into(), + max_used_server_id: ServerId( + u64::try_from(bp_config.max_used_server_id) + .map_err(|_| { + Error::internal_error(&format!( + "max server id is negative: {}", + bp_config.max_used_server_id + )) + })?, + ), + max_used_keeper_id: KeeperId( + u64::try_from(bp_config.max_used_keeper_id) + .map_err(|_| { + Error::internal_error(&format!( + "max keeper id is negative: {}", + bp_config.max_used_keeper_id + )) + })?, + ), + cluster_name: bp_config.cluster_name, + cluster_secret: bp_config.cluster_secret, + highest_seen_keeper_leader_committed_log_index: + u64::try_from( + bp_config.highest_seen_keeper_leader_committed_log_index, + ) + .map_err(|_| { + Error::internal_error(&format!( + "max server id is negative: {}", + bp_config.highest_seen_keeper_leader_committed_log_index + )) + })?, + keepers, + servers, + }) + } + } + }; + Ok(Blueprint { id: blueprint_id, blueprint_zones, @@ -632,6 +850,7 @@ impl DataStore { external_dns_version, cockroachdb_fingerprint, cockroachdb_setting_preserve_downgrade, + clickhouse_cluster_config, time_created, creator, comment, @@ -663,6 +882,9 @@ impl DataStore { nsled_agent_zones, nzones, nnics, + nclickhouse_cluster_configs, + nclickhouse_keepers, + nclickhouse_servers, ) = conn .transaction_async(|conn| async move { // Ensure that blueprint we're about to delete is not the @@ -759,6 +981,34 @@ impl DataStore { .await? }; + let nclickhouse_cluster_configs = { + use db::schema::bp_clickhouse_cluster_config::dsl; + diesel::delete( + dsl::bp_clickhouse_cluster_config + .filter(dsl::blueprint_id.eq(blueprint_id)), + ) + .execute_async(&conn) + .await? + }; + + let nclickhouse_keepers = { + use db::schema::bp_clickhouse_keeper_zone_id_to_node_id::dsl; + diesel::delete(dsl::bp_clickhouse_keeper_zone_id_to_node_id + .filter(dsl::blueprint_id.eq(blueprint_id)), + ) + .execute_async(&conn) + .await? + }; + + let nclickhouse_servers = { + use db::schema::bp_clickhouse_server_zone_id_to_node_id::dsl; + diesel::delete(dsl::bp_clickhouse_server_zone_id_to_node_id + .filter(dsl::blueprint_id.eq(blueprint_id)), + ) + .execute_async(&conn) + .await? + }; + Ok(( nblueprints, nsled_states, @@ -767,6 +1017,9 @@ impl DataStore { nsled_agent_zones, nzones, nnics, + nclickhouse_cluster_configs, + nclickhouse_keepers, + nclickhouse_servers, )) }) .await @@ -786,6 +1039,9 @@ impl DataStore { "nsled_agent_zones" => nsled_agent_zones, "nzones" => nzones, "nnics" => nnics, + "nclickhouse_cluster_configs" => nclickhouse_cluster_configs, + "nclickhouse_keepers" => nclickhouse_keepers, + "nclickhouse_servers" => nclickhouse_servers ); Ok(()) @@ -1613,7 +1869,7 @@ mod tests { ) -> (Collection, PlanningInput, Blueprint) { // We'll start with an example system. let (mut base_collection, planning_input, mut blueprint) = - example(log, test_name, 3); + example(log, test_name); // Take a more thorough collection representative (includes SPs, // etc.)... @@ -1626,10 +1882,6 @@ mod tests { &mut collection.sled_agents, &mut base_collection.sled_agents, ); - mem::swap( - &mut collection.omicron_zones, - &mut base_collection.omicron_zones, - ); // Treat this blueprint as the initial blueprint for the system. blueprint.parent_blueprint_id = None; @@ -1743,7 +1995,7 @@ mod tests { ); assert_eq!( blueprint1.blueprint_zones.len(), - collection.omicron_zones.len() + collection.sled_agents.len() ); assert_eq!( blueprint1.all_omicron_zones(BlueprintZoneFilter::All).count(), diff --git a/nexus/db-queries/src/db/datastore/instance.rs b/nexus/db-queries/src/db/datastore/instance.rs index e89cd8f234..8fdb16b2f3 100644 --- a/nexus/db-queries/src/db/datastore/instance.rs +++ b/nexus/db-queries/src/db/datastore/instance.rs @@ -223,16 +223,23 @@ impl From for external::Instance { }, ); - let policy = value - .instance - .auto_restart - .policy - .unwrap_or(InstanceAutoRestart::DEFAULT_POLICY); - let enabled = match policy { + let policy = value.instance.auto_restart.policy; + // The active policy for this instance --- either its configured + // policy or the default. We report the configured policy as the + // instance's policy, but we must use this to determine whether it + // will be auto-restarted, since it may have no configured policy. + let active_policy = + policy.unwrap_or(InstanceAutoRestart::DEFAULT_POLICY); + + let enabled = match active_policy { InstanceAutoRestartPolicy::Never => false, InstanceAutoRestartPolicy::BestEffort => true, }; - external::InstanceAutoRestartStatus { enabled, cooldown_expiration } + external::InstanceAutoRestartStatus { + enabled, + policy: policy.map(Into::into), + cooldown_expiration, + } }; Self { @@ -572,6 +579,27 @@ impl DataStore { ) -> LookupResult { opctx.authorize(authz::Action::Read, authz_instance).await?; + self.instance_fetch_with_vmm_on_conn( + &*self.pool_connection_authorized(opctx).await?, + authz_instance, + ) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Instance, + LookupType::ById(authz_instance.id()), + ), + ) + }) + } + + async fn instance_fetch_with_vmm_on_conn( + &self, + conn: &async_bb8_diesel::Connection, + authz_instance: &authz::Instance, + ) -> Result { use db::schema::instance::dsl as instance_dsl; use db::schema::vmm::dsl as vmm_dsl; @@ -585,19 +613,8 @@ impl DataStore { .and(vmm_dsl::time_deleted.is_null())), ) .select((Instance::as_select(), Option::::as_select())) - .get_result_async::<(Instance, Option)>( - &*self.pool_connection_authorized(opctx).await?, - ) - .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::NotFoundByLookup( - ResourceType::Instance, - LookupType::ById(authz_instance.id()), - ), - ) - })?; + .get_result_async::<(Instance, Option)>(conn) + .await?; Ok(InstanceAndActiveVmm { instance, vmm }) } @@ -1013,134 +1030,39 @@ impl DataStore { ) -> Result { opctx.authorize(authz::Action::Modify, authz_instance).await?; - use crate::db::model::InstanceState; - - use db::schema::disk::dsl as disk_dsl; use db::schema::instance::dsl as instance_dsl; - use db::schema::vmm::dsl as vmm_dsl; let err = OptionalError::new(); let conn = self.pool_connection_authorized(opctx).await?; - let (instance, vmm) = self + let instance_and_vmm = self .transaction_retry_wrapper("reconfigure_instance") .transaction(&conn, |conn| { let err = err.clone(); - let update = update.clone(); + let InstanceUpdate { boot_disk_id, auto_restart_policy } = + update.clone(); async move { - // * Allow reconfiguration in NoVmm because there is no VMM - // to contend with. - // * Allow reconfiguration in Failed to allow changing the - // boot disk of a failed instance and free its boot disk - // for detach. - // * Allow reconfiguration in Creating because one of the - // last steps of instance creation, while the instance is - // still in Creating, is to reconfigure the instance to - // the desired boot disk. - let ok_to_reconfigure_instance_states = [ - InstanceState::NoVmm, - InstanceState::Failed, - InstanceState::Creating, - ]; - - let instance_state = instance_dsl::instance - .filter(instance_dsl::id.eq(authz_instance.id())) - .filter(instance_dsl::time_deleted.is_null()) - .select(instance_dsl::state) - .first_async::(&conn) - .await; - - match instance_state { - Ok(state) => { - let state_ok = ok_to_reconfigure_instance_states - .contains(&state); - - if !state_ok { - return Err(err.bail(Error::conflict( - "instance must be stopped to update", - ))); - } - } - Err(diesel::NotFound) => { - // If the instance simply doesn't exist, we - // shouldn't retry. Bail with a useful error. - return Err(err.bail(Error::not_found_by_id( - ResourceType::Instance, - &authz_instance.id(), - ))); - } - Err(e) => { - return Err(e); - } - } - - if let Some(disk_id) = update.boot_disk_id { - // Ensure the disk is currently attached before updating - // the database. - let expected_state = api::external::DiskState::Attached( - authz_instance.id(), - ); - - let attached_disk: Option = disk_dsl::disk - .filter(disk_dsl::id.eq(disk_id)) - .filter( - disk_dsl::attach_instance_id - .eq(authz_instance.id()), - ) - .filter( - disk_dsl::disk_state.eq(expected_state.label()), - ) - .select(disk_dsl::id) - .first_async::(&conn) - .await - .optional()?; - - if attached_disk.is_none() { - return Err(err.bail(Error::conflict( - "boot disk must be attached", - ))); - } - } - - // if and when `Update` can update other fields, set them - // here. - // - // NOTE: from this point forward it is OK if we update the - // instance's `boot_disk_id` column with the updated value - // again. It will have already been assigned with constraint - // checking performed above, so updates will just be - // repetitive, not harmful. - - // Update the row. We don't care about the returned - // UpdateStatus, either way the database has been updated - // with the state we're setting. + // Set the auto-restart policy. diesel::update(instance_dsl::instance) .filter(instance_dsl::id.eq(authz_instance.id())) - .set(update) + .set( + instance_dsl::auto_restart_policy + .eq(auto_restart_policy), + ) .execute_async(&conn) .await?; - // TODO: dedupe this query and `instance_fetch_with_vmm`. - // At the moment, we're only allowing instance - // reconfiguration in states that would have no VMM, but - // load it anyway so that we return correct data if this is - // relaxed in the future... - let (instance, vmm) = instance_dsl::instance - .filter(instance_dsl::id.eq(authz_instance.id())) - .filter(instance_dsl::time_deleted.is_null()) - .left_join( - vmm_dsl::vmm.on(vmm_dsl::id - .nullable() - .eq(instance_dsl::active_propolis_id) - .and(vmm_dsl::time_deleted.is_null())), - ) - .select(( - Instance::as_select(), - Option::::as_select(), - )) - .get_result_async(&conn) - .await?; + // Next, set the boot disk if needed. + self.instance_set_boot_disk_on_conn( + &conn, + &err, + authz_instance, + boot_disk_id, + ) + .await?; - Ok((instance, vmm)) + // Finally, fetch the new instance state. + self.instance_fetch_with_vmm_on_conn(&conn, authz_instance) + .await } }) .await @@ -1152,7 +1074,145 @@ impl DataStore { public_error_from_diesel(e, ErrorHandler::Server) })?; - Ok(InstanceAndActiveVmm { instance, vmm }) + Ok(instance_and_vmm) + } + + pub async fn instance_set_boot_disk( + &self, + opctx: &OpContext, + authz_instance: &authz::Instance, + boot_disk_id: Option, + ) -> Result<(), Error> { + opctx.authorize(authz::Action::Modify, authz_instance).await?; + + let err = OptionalError::new(); + let conn = self.pool_connection_authorized(opctx).await?; + self.transaction_retry_wrapper("instance_set_boot_disk") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + self.instance_set_boot_disk_on_conn( + &conn, + &err, + authz_instance, + boot_disk_id, + ) + .await?; + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + return err; + } + + public_error_from_diesel(e, ErrorHandler::Server) + }) + } + + /// Set an instance's boot disk to the provided `boot_disk_id` (or unset it, + /// if `boot_disk_id` is `None`), within an existing transaction. + /// + /// This is factored out as it is used by both + /// [`DataStore::instance_reconfigure`], which mutates many instance fields, + /// and [`DataStore::instance_set_boot_disk`], which only touches the boot + /// disk. + async fn instance_set_boot_disk_on_conn( + &self, + conn: &async_bb8_diesel::Connection, + err: &OptionalError, + authz_instance: &authz::Instance, + boot_disk_id: Option, + ) -> Result<(), diesel::result::Error> { + use db::schema::disk::dsl as disk_dsl; + use db::schema::instance::dsl as instance_dsl; + + // * Allow setting the boot disk in NoVmm because there is no VMM to + // contend with. + // * Allow setting the boot disk in Failed to allow changing the boot + // disk of a failed instance and free its boot disk for detach. + // * Allow setting the boot disk in Creating because one of the last + // steps of instance creation, while the instance is still in + // Creating, is to reconfigure the instance to the desired boot disk. + const OK_TO_SET_BOOT_DISK_STATES: &'static [InstanceState] = &[ + InstanceState::NoVmm, + InstanceState::Failed, + InstanceState::Creating, + ]; + + let maybe_instance = instance_dsl::instance + .filter(instance_dsl::id.eq(authz_instance.id())) + .filter(instance_dsl::time_deleted.is_null()) + .select(Instance::as_select()) + .first_async::(conn) + .await; + let instance = match maybe_instance { + Ok(i) => i, + Err(diesel::NotFound) => { + // If the instance simply doesn't exist, we + // shouldn't retry. Bail with a useful error. + return Err(err.bail(Error::not_found_by_id( + ResourceType::Instance, + &authz_instance.id(), + ))); + } + Err(e) => return Err(e), + }; + + // If the desired boot disk is already set, we're good here, and can + // elide the check that the instance is in an acceptable state to change + // the boot disk. + if instance.boot_disk_id == boot_disk_id { + return Ok(()); + } + + if let Some(disk_id) = boot_disk_id { + // Ensure the disk is currently attached before updating + // the database. + let expected_state = + api::external::DiskState::Attached(authz_instance.id()); + + let attached_disk: Option = disk_dsl::disk + .filter(disk_dsl::id.eq(disk_id)) + .filter(disk_dsl::attach_instance_id.eq(authz_instance.id())) + .filter(disk_dsl::disk_state.eq(expected_state.label())) + .select(disk_dsl::id) + .first_async::(conn) + .await + .optional()?; + + if attached_disk.is_none() { + return Err( + err.bail(Error::conflict("boot disk must be attached")) + ); + } + } + // + // NOTE: from this point forward it is OK if we update the + // instance's `boot_disk_id` column with the updated value + // again. It will have already been assigned with constraint + // checking performed above, so updates will just be + // repetitive, not harmful. + + let r = diesel::update(instance_dsl::instance) + .filter(instance_dsl::id.eq(authz_instance.id())) + .filter(instance_dsl::state.eq_any(OK_TO_SET_BOOT_DISK_STATES)) + .set(instance_dsl::boot_disk_id.eq(boot_disk_id)) + .check_if_exists::(authz_instance.id()) + .execute_and_check(&conn) + .await?; + match r.status { + UpdateStatus::NotUpdatedButExists => { + // This should be the only reason the query would fail... + debug_assert!(!OK_TO_SET_BOOT_DISK_STATES + .contains(&r.found.runtime().nexus_state)); + Err(err.bail(Error::conflict( + "instance must be stopped to set boot disk", + ))) + } + UpdateStatus::Updated => Ok(()), + } } pub async fn project_delete_instance( @@ -1894,6 +1954,7 @@ mod tests { use nexus_test_utils::db::test_setup_database; use nexus_types::external_api::params; use nexus_types::identity::Asset; + use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::api::external; use omicron_common::api::external::ByteCount; use omicron_common::api::external::IdentityMetadataCreateParams; @@ -1903,7 +1964,7 @@ mod tests { datastore: &DataStore, opctx: &OpContext, ) -> (authz::Project, Project) { - let silo_id = *nexus_db_fixed_data::silo::DEFAULT_SILO_ID; + let silo_id = DEFAULT_SILO_ID; let project_id = Uuid::new_v4(); datastore .project_create( diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index 5b72068b76..3fdf38f19a 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -16,6 +16,7 @@ use anyhow::Context; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use async_bb8_diesel::AsyncSimpleConnection; +use clickhouse_admin_types::ClickhouseKeeperClusterMembership; use diesel::expression::SelectableHelper; use diesel::sql_types::Nullable; use diesel::BoolExpressionMethods; @@ -36,6 +37,7 @@ use nexus_db_model::HwPowerStateEnum; use nexus_db_model::HwRotSlot; use nexus_db_model::HwRotSlotEnum; use nexus_db_model::InvCaboose; +use nexus_db_model::InvClickhouseKeeperMembership; use nexus_db_model::InvCollection; use nexus_db_model::InvCollectionError; use nexus_db_model::InvDataset; @@ -60,6 +62,7 @@ use nexus_db_model::SqlU16; use nexus_db_model::SqlU32; use nexus_db_model::SwCaboose; use nexus_db_model::SwRotPage; +use nexus_sled_agent_shared::inventory::OmicronZonesConfig; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::Collection; use nexus_types::inventory::PhysicalDiskFirmware; @@ -160,6 +163,32 @@ impl DataStore { } } + // Pull Omicron zone-related metadata out of all sled agents. + // + // TODO: InvSledOmicronZones is a vestigial table kept for backwards + // compatibility -- the only unique data within it (the generation + // number) can be moved into `InvSledAgent` in the future. See + // oxidecomputer/omicron#6770. + let sled_omicron_zones = collection + .sled_agents + .values() + .map(|sled_agent| { + InvSledOmicronZones::new(collection_id, sled_agent) + }) + .collect::>(); + + // Pull Omicron zones out of all sled agents. + let omicron_zones: Vec<_> = collection + .sled_agents + .iter() + .flat_map(|(sled_id, sled_agent)| { + sled_agent.omicron_zones.zones.iter().map(|zone| { + InvOmicronZone::new(collection_id, *sled_id, zone) + .map_err(|e| Error::internal_error(&e.to_string())) + }) + }) + .collect::, _>>()?; + // Pull disks out of all sled agents let disks: Vec<_> = collection .sled_agents @@ -212,30 +241,11 @@ impl DataStore { }) .collect::, Error>>()?; - let sled_omicron_zones = collection - .omicron_zones - .values() - .map(|found| InvSledOmicronZones::new(collection_id, found)) - .collect::>(); - let omicron_zones = collection - .omicron_zones - .values() - .flat_map(|found| { - found.zones.zones.iter().map(|found_zone| { - InvOmicronZone::new( - collection_id, - found.sled_id, - found_zone, - ) - .map_err(|e| Error::internal_error(&e.to_string())) - }) - }) - .collect::, Error>>()?; let omicron_zone_nics = collection - .omicron_zones + .sled_agents .values() - .flat_map(|found| { - found.zones.zones.iter().filter_map(|found_zone| { + .flat_map(|sled_agent| { + sled_agent.omicron_zones.zones.iter().filter_map(|found_zone| { InvOmicronZoneNic::new(collection_id, found_zone) .with_context(|| format!("zone {:?}", found_zone.id)) .map_err(|e| Error::internal_error(&format!("{:#}", e))) @@ -244,6 +254,17 @@ impl DataStore { }) .collect::, _>>()?; + let mut inv_clickhouse_keeper_memberships = Vec::new(); + for membership in &collection.clickhouse_keeper_cluster_membership { + inv_clickhouse_keeper_memberships.push( + InvClickhouseKeeperMembership::new( + collection_id, + membership.clone(), + ) + .map_err(|e| Error::internal_error(&e.to_string()))?, + ); + } + // This implementation inserts all records associated with the // collection in one transaction. This is primarily for simplicity. It // means we don't have to worry about other readers seeing a @@ -953,6 +974,15 @@ impl DataStore { .await?; } + // Insert the clickhouse keeper memberships we've received + { + use db::schema::inv_clickhouse_keeper_membership::dsl; + diesel::insert_into(dsl::inv_clickhouse_keeper_membership) + .values(inv_clickhouse_keeper_memberships) + .execute_async(&conn) + .await?; + } + // Finally, insert the list of errors. { use db::schema::inv_collection_error::dsl as errors_dsl; @@ -1221,6 +1251,7 @@ impl DataStore { nnics, nzpools, nerrors, + nclickhouse_keeper_membership, ) = conn .transaction_async(|conn| async move { // Remove the record describing the collection itself. @@ -1374,6 +1405,18 @@ impl DataStore { .await? }; + // Remove rows for clickhouse keeper membership + let nclickhouse_keeper_membership = { + use db::schema::inv_clickhouse_keeper_membership::dsl; + diesel::delete( + dsl::inv_clickhouse_keeper_membership.filter( + dsl::inv_collection_id.eq(db_collection_id), + ), + ) + .execute_async(&conn) + .await? + }; + Ok(( ncollections, nsps, @@ -1389,6 +1432,7 @@ impl DataStore { nnics, nzpools, nerrors, + nclickhouse_keeper_membership, )) }) .await @@ -1415,6 +1459,7 @@ impl DataStore { "nnics" => nnics, "nzpools" => nzpools, "nerrors" => nerrors, + "nclickhouse_keeper_membership" => nclickhouse_keeper_membership ); Ok(()) @@ -1862,56 +1907,6 @@ impl DataStore { }) }) .collect::, _>>()?; - let sled_agents: BTreeMap<_, _> = sled_agent_rows - .into_iter() - .map(|s: InvSledAgent| { - let sled_id = SledUuid::from(s.sled_id); - let baseboard_id = s - .hw_baseboard_id - .map(|id| { - baseboards_by_id.get(&id).cloned().ok_or_else(|| { - Error::internal_error( - "missing baseboard that we should have fetched", - ) - }) - }) - .transpose()?; - let sled_agent = nexus_types::inventory::SledAgent { - time_collected: s.time_collected, - source: s.source, - sled_id, - baseboard_id, - sled_agent_address: std::net::SocketAddrV6::new( - std::net::Ipv6Addr::from(s.sled_agent_ip), - u16::from(s.sled_agent_port), - 0, - 0, - ), - sled_role: s.sled_role.into(), - usable_hardware_threads: u32::from( - s.usable_hardware_threads, - ), - usable_physical_ram: s.usable_physical_ram.into(), - reservoir_size: s.reservoir_size.into(), - disks: physical_disks - .get(&sled_id) - .map(|disks| disks.to_vec()) - .unwrap_or_default(), - zpools: zpools - .get(sled_id.as_untyped_uuid()) - .map(|zpools| zpools.to_vec()) - .unwrap_or_default(), - datasets: datasets - .get(sled_id.as_untyped_uuid()) - .map(|datasets| datasets.to_vec()) - .unwrap_or_default(), - }; - Ok((sled_id, sled_agent)) - }) - .collect::, - Error, - >>()?; // Fetch records of cabooses found. let inv_caboose_rows = { @@ -2153,7 +2148,10 @@ impl DataStore { zones.extend(batch.into_iter().map(|sled_zones_config| { ( sled_zones_config.sled_id.into(), - sled_zones_config.into_uninit_zones_found(), + OmicronZonesConfig { + generation: sled_zones_config.generation.into(), + zones: Vec::new(), + }, ) })) } @@ -2261,15 +2259,128 @@ impl DataStore { .map_err(|e| { Error::internal_error(&format!("{:#}", e.to_string())) })?; - map.zones.zones.push(zone); + map.zones.push(zone); } + // Now load the clickhouse keeper cluster memberships + let clickhouse_keeper_cluster_membership = { + use db::schema::inv_clickhouse_keeper_membership::dsl; + let mut memberships = BTreeSet::new(); + let mut paginator = Paginator::new(batch_size); + while let Some(p) = paginator.next() { + let batch = paginated( + dsl::inv_clickhouse_keeper_membership, + dsl::queried_keeper_id, + &p.current_pagparams(), + ) + .filter(dsl::inv_collection_id.eq(db_id)) + .select(InvClickhouseKeeperMembership::as_select()) + .load_async(&*conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + paginator = p.found_batch(&batch, &|row| row.queried_keeper_id); + for membership in batch.into_iter() { + memberships.insert( + ClickhouseKeeperClusterMembership::try_from(membership) + .map_err(|e| { + Error::internal_error(&format!("{e:#}",)) + })?, + ); + } + } + memberships + }; + bail_unless!( omicron_zone_nics.is_empty(), "found extra Omicron zone NICs: {:?}", omicron_zone_nics.keys() ); + // Finally, build up the sled-agent map using the sled agent and + // omicron zone rows. A for loop is easier to understand than into_iter + // + filter_map + return Result + collect. + let mut sled_agents = BTreeMap::new(); + for s in sled_agent_rows { + let sled_id = SledUuid::from(s.sled_id); + let baseboard_id = s + .hw_baseboard_id + .map(|id| { + baseboards_by_id.get(&id).cloned().ok_or_else(|| { + Error::internal_error( + "missing baseboard that we should have fetched", + ) + }) + }) + .transpose()?; + + // Look up the Omicron zones. + // + // Older versions of Nexus fetched the Omicron zones in a separate + // request from the other sled agent data. The database model stil + // accounts for the possibility that for a given (collection, sled) + // pair, one of those queries succeeded while the other failed. But + // this has since been changed to fetch all the data in a single + // query, which means that newer collections will either have both + // sets of data or neither of them. + // + // If it _is_ the case that one of the pieces of data is missing, + // log that as a warning and drop the sled from the collection. + // This should only happen for old collections, and only in the + // unlikely case that exactly one of the two related requests + // failed. + // + // TODO: Update the database model to reflect the new reality + // (oxidecomputer/omicron#6770). + let Some(omicron_zones) = omicron_zones.remove(&sled_id) else { + warn!( + self.log, + "no sled Omicron zone data present -- assuming that collection was done + by an old Nexus version and dropping sled from it"; + "collection" => %id, + "sled_id" => %sled_id, + ); + continue; + }; + + let sled_agent = nexus_types::inventory::SledAgent { + time_collected: s.time_collected, + source: s.source, + sled_id, + baseboard_id, + sled_agent_address: std::net::SocketAddrV6::new( + std::net::Ipv6Addr::from(s.sled_agent_ip), + u16::from(s.sled_agent_port), + 0, + 0, + ), + sled_role: s.sled_role.into(), + usable_hardware_threads: u32::from(s.usable_hardware_threads), + usable_physical_ram: s.usable_physical_ram.into(), + reservoir_size: s.reservoir_size.into(), + omicron_zones, + // For disks, zpools, and datasets, the map for a sled ID is + // only populated if there is at least one disk/zpool/dataset + // for that sled. The `unwrap_or_default` calls cover the case + // where there are no disks/zpools/datasets for a sled. + disks: physical_disks + .get(&sled_id) + .map(|disks| disks.to_vec()) + .unwrap_or_default(), + zpools: zpools + .get(sled_id.as_untyped_uuid()) + .map(|zpools| zpools.to_vec()) + .unwrap_or_default(), + datasets: datasets + .get(sled_id.as_untyped_uuid()) + .map(|datasets| datasets.to_vec()) + .unwrap_or_default(), + }; + sled_agents.insert(sled_id, sled_agent); + } + Ok(Collection { id, errors, @@ -2284,10 +2395,7 @@ impl DataStore { cabooses_found, rot_pages_found, sled_agents, - omicron_zones, - // Currently unused - // See: https://github.com/oxidecomputer/omicron/issues/6578 - clickhouse_keeper_cluster_membership: BTreeMap::new(), + clickhouse_keeper_cluster_membership, }) } } diff --git a/nexus/db-queries/src/db/datastore/ip_pool.rs b/nexus/db-queries/src/db/datastore/ip_pool.rs index 08db5ef38c..7fa38badb0 100644 --- a/nexus/db-queries/src/db/datastore/ip_pool.rs +++ b/nexus/db-queries/src/db/datastore/ip_pool.rs @@ -36,12 +36,17 @@ use chrono::Utc; use diesel::prelude::*; use diesel::result::Error as DieselError; use ipnetwork::IpNetwork; +use nexus_db_model::InternetGateway; +use nexus_db_model::InternetGatewayIpPool; +use nexus_db_model::Project; +use nexus_db_model::Vpc; use nexus_types::external_api::shared::IpRange; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::DeleteResult; use omicron_common::api::external::Error; +use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::InternalContext; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; @@ -484,9 +489,11 @@ impl DataStore { .authorize(authz::Action::CreateChild, &authz::IP_POOL_LIST) .await?; - diesel::insert_into(dsl::ip_pool_resource) + let conn = self.pool_connection_authorized(opctx).await?; + + let result = diesel::insert_into(dsl::ip_pool_resource) .values(ip_pool_resource.clone()) - .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .get_result_async(&*conn) .await .map_err(|e| { public_error_from_diesel( @@ -501,7 +508,174 @@ impl DataStore { ) ), ) - }) + })?; + + if ip_pool_resource.is_default { + self.link_default_gateway( + opctx, + ip_pool_resource.resource_id, + ip_pool_resource.ip_pool_id, + &conn, + ) + .await?; + } + + Ok(result) + } + + async fn link_default_gateway( + &self, + opctx: &OpContext, + silo_id: Uuid, + ip_pool_id: Uuid, + conn: &async_bb8_diesel::Connection, + ) -> UpdateResult<()> { + use db::schema::internet_gateway::dsl as igw_dsl; + use db::schema::internet_gateway_ip_pool::dsl as igw_ip_pool_dsl; + use db::schema::project::dsl as project_dsl; + use db::schema::vpc::dsl as vpc_dsl; + + let projects = project_dsl::project + .filter(project_dsl::time_deleted.is_null()) + .filter(project_dsl::silo_id.eq(silo_id)) + .select(Project::as_select()) + .load_async(conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + for project in &projects { + let vpcs = vpc_dsl::vpc + .filter(vpc_dsl::time_deleted.is_null()) + .filter(vpc_dsl::project_id.eq(project.id())) + .select(Vpc::as_select()) + .load_async(conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + for vpc in &vpcs { + let igws = igw_dsl::internet_gateway + .filter(igw_dsl::time_deleted.is_null()) + .filter(igw_dsl::name.eq("default")) + .filter(igw_dsl::vpc_id.eq(vpc.id())) + .select(InternetGateway::as_select()) + .load_async(conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + for igw in &igws { + let igw_pool = InternetGatewayIpPool::new( + Uuid::new_v4(), + ip_pool_id, + igw.id(), + IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: String::from( + "Default internet gateway ip pool", + ), + }, + ); + + let _ipp: InternetGatewayIpPool = + match InternetGateway::insert_resource( + igw.id(), + diesel::insert_into( + igw_ip_pool_dsl::internet_gateway_ip_pool, + ) + .values(igw_pool), + ) + .insert_and_get_result_async(&conn) + .await { + Ok(x) => x, + Err(e) => match e { + AsyncInsertError::CollectionNotFound => { + return Err(Error::not_found_by_name( + ResourceType::InternetGateway, + &"default".parse().unwrap(), + )) + } + AsyncInsertError::DatabaseError(e) => match e { + diesel::result::Error::DatabaseError(diesel::result::DatabaseErrorKind::UniqueViolation, _) => + { + return Ok(()); + } + _ => return Err(public_error_from_diesel( + e, + ErrorHandler::Server, + )), + }, + } + }; + } + self.vpc_increment_rpw_version(opctx, vpc.id()).await?; + } + } + Ok(()) + } + + async fn unlink_ip_pool_gateway( + &self, + opctx: &OpContext, + silo_id: Uuid, + ip_pool_id: Uuid, + conn: &async_bb8_diesel::Connection, + ) -> UpdateResult<()> { + use db::schema::internet_gateway::dsl as igw_dsl; + use db::schema::internet_gateway_ip_pool::dsl as igw_ip_pool_dsl; + use db::schema::project::dsl as project_dsl; + use db::schema::vpc::dsl as vpc_dsl; + + let projects = project_dsl::project + .filter(project_dsl::time_deleted.is_null()) + .filter(project_dsl::silo_id.eq(silo_id)) + .select(Project::as_select()) + .load_async(conn) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + for project in &projects { + let vpcs = vpc_dsl::vpc + .filter(vpc_dsl::time_deleted.is_null()) + .filter(vpc_dsl::project_id.eq(project.id())) + .select(Vpc::as_select()) + .load_async(conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + for vpc in &vpcs { + let igws = igw_dsl::internet_gateway + .filter(igw_dsl::time_deleted.is_null()) + .filter(igw_dsl::vpc_id.eq(vpc.id())) + .select(InternetGateway::as_select()) + .load_async(conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + + for igw in &igws { + diesel::update(igw_ip_pool_dsl::internet_gateway_ip_pool) + .filter(igw_ip_pool_dsl::time_deleted.is_null()) + .filter( + igw_ip_pool_dsl::internet_gateway_id.eq(igw.id()), + ) + .filter(igw_ip_pool_dsl::ip_pool_id.eq(ip_pool_id)) + .set(igw_ip_pool_dsl::time_deleted.eq(Utc::now())) + .execute_async(conn) + .await + .map_err(|e| { + public_error_from_diesel(e, ErrorHandler::Server) + })?; + } + self.vpc_increment_rpw_version(opctx, vpc.id()).await?; + } + } + Ok(()) } pub async fn ip_pool_set_default( @@ -725,10 +899,12 @@ impl DataStore { self.ensure_no_floating_ips_outstanding(opctx, authz_pool, authz_silo) .await?; + let conn = self.pool_connection_authorized(opctx).await?; + diesel::delete(ip_pool_resource::table) .filter(ip_pool_resource::ip_pool_id.eq(authz_pool.id())) .filter(ip_pool_resource::resource_id.eq(authz_silo.id())) - .execute_async(&*self.pool_connection_authorized(opctx).await?) + .execute_async(&*conn) .await .map(|_rows_deleted| ()) .map_err(|e| { @@ -736,7 +912,17 @@ impl DataStore { "error deleting IP pool association to resource: {:?}", e )) - }) + })?; + + self.unlink_ip_pool_gateway( + opctx, + authz_silo.id(), + authz_pool.id(), + &conn, + ) + .await?; + + Ok(()) } pub async fn ip_pool_list_ranges( diff --git a/nexus/db-queries/src/db/datastore/migration.rs b/nexus/db-queries/src/db/datastore/migration.rs index 584f00f084..0866226d69 100644 --- a/nexus/db-queries/src/db/datastore/migration.rs +++ b/nexus/db-queries/src/db/datastore/migration.rs @@ -184,6 +184,7 @@ mod tests { use nexus_db_model::Project; use nexus_test_utils::db::test_setup_database; use nexus_types::external_api::params; + use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::api::external::ByteCount; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_test_utils::dev; @@ -194,7 +195,7 @@ mod tests { datastore: &DataStore, opctx: &OpContext, ) -> authz::Instance { - let silo_id = *nexus_db_fixed_data::silo::DEFAULT_SILO_ID; + let silo_id = DEFAULT_SILO_ID; let project_id = Uuid::new_v4(); let instance_id = InstanceUuid::new_v4(); diff --git a/nexus/db-queries/src/db/datastore/mod.rs b/nexus/db-queries/src/db/datastore/mod.rs index ec317c184f..258e43f18c 100644 --- a/nexus/db-queries/src/db/datastore/mod.rs +++ b/nexus/db-queries/src/db/datastore/mod.rs @@ -444,11 +444,11 @@ mod test { use futures::StreamExt; use nexus_config::RegionAllocationStrategy; use nexus_db_fixed_data::silo::DEFAULT_SILO; - use nexus_db_fixed_data::silo::DEFAULT_SILO_ID; use nexus_db_model::IpAttachState; use nexus_db_model::{to_db_typed_uuid, Generation}; use nexus_test_utils::db::test_setup_database; use nexus_types::external_api::params; + use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::api::external::{ ByteCount, Error, IdentityMetadataCreateParams, LookupType, Name, }; @@ -552,8 +552,8 @@ mod test { // Associate silo with user let authz_silo = authz::Silo::new( authz::FLEET, - *DEFAULT_SILO_ID, - LookupType::ById(*DEFAULT_SILO_ID), + DEFAULT_SILO_ID, + LookupType::ById(DEFAULT_SILO_ID), ); datastore .silo_user_create( @@ -572,7 +572,7 @@ mod test { .fetch() .await .unwrap(); - assert_eq!(*DEFAULT_SILO_ID, db_silo_user.silo_id); + assert_eq!(DEFAULT_SILO_ID, db_silo_user.silo_id); // fetch the one we just created let (.., fetched) = LookupPath::new(&opctx, &datastore) @@ -630,7 +630,7 @@ mod test { Arc::new(authz::Authz::new(&logctx.log)), authn::Context::for_test_user( silo_user_id, - *DEFAULT_SILO_ID, + DEFAULT_SILO_ID, SiloAuthnPolicy::try_from(&*DEFAULT_SILO).unwrap(), ), Arc::clone(&datastore) as Arc, @@ -1726,8 +1726,8 @@ mod test { // Create a new Silo user so that we can lookup their keys. let authz_silo = authz::Silo::new( authz::FLEET, - *DEFAULT_SILO_ID, - LookupType::ById(*DEFAULT_SILO_ID), + DEFAULT_SILO_ID, + LookupType::ById(DEFAULT_SILO_ID), ); let silo_user_id = Uuid::new_v4(); datastore @@ -1777,7 +1777,7 @@ mod test { .fetch() .await .unwrap(); - assert_eq!(authz_silo.id(), *DEFAULT_SILO_ID); + assert_eq!(authz_silo.id(), DEFAULT_SILO_ID); assert_eq!(authz_silo_user.id(), silo_user_id); assert_eq!(found.silo_user_id, ssh_key.silo_user_id); assert_eq!(found.public_key, ssh_key.public_key); diff --git a/nexus/db-queries/src/db/datastore/oximeter.rs b/nexus/db-queries/src/db/datastore/oximeter.rs index 0c4b5077f2..f43ab4a051 100644 --- a/nexus/db-queries/src/db/datastore/oximeter.rs +++ b/nexus/db-queries/src/db/datastore/oximeter.rs @@ -22,12 +22,11 @@ use chrono::Utc; use diesel::prelude::*; use diesel::result::DatabaseErrorKind; use diesel::result::Error as DieselError; -use diesel::sql_types; -use nexus_db_model::ProducerKindEnum; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::ResourceType; +use omicron_common::api::internal; use uuid::Uuid; /// Type returned when reassigning producers from an Oximeter collector. @@ -168,88 +167,29 @@ impl DataStore { } } - /// Create a record for a new producer endpoint - pub async fn producer_endpoint_create( + /// Create or update a record for a producer endpoint + /// + /// If the endpoint is being created, a randomly-chosen Oximeter instance + /// will be assigned. If the endpoint is being updated, it will keep its + /// existing Oximeter assignment. + /// + /// Returns the oximeter ID assigned to this producer (either the + /// randomly-chosen one, if newly inserted, or the previously-chosen, if + /// updated). + pub async fn producer_endpoint_upsert_and_assign( &self, opctx: &OpContext, - producer: &ProducerEndpoint, - ) -> Result<(), Error> { - // Our caller has already chosen an Oximeter instance for this producer, - // but we don't want to allow it to use a nonexistent or expunged - // Oximeter. This query turns into a `SELECT all_the_fields_of_producer - // WHERE producer.oximeter_id is legal` in a diesel-compatible way. I'm - // not aware of a helper method to generate "all the fields of - // `producer`", so instead we have a big tuple of its fields that must - // stay in sync with the `table!` definition and field ordering for the - // `metric_producer` table. The compiler will catch any mistakes - // _except_ incorrect orderings where the types still line up (e.g., - // swapping two Uuid columns), which is not ideal but is hopefully good - // enough. - let producer_subquery = { - use db::schema::oximeter::dsl; - - dsl::oximeter - .select(( - producer.id().into_sql::(), - producer - .time_created() - .into_sql::(), - producer - .time_modified() - .into_sql::(), - producer.kind.into_sql::(), - producer.ip.into_sql::(), - producer.port.into_sql::(), - producer.interval.into_sql::(), - producer.oximeter_id.into_sql::(), - )) - .filter( - dsl::id - .eq(producer.oximeter_id) - .and(dsl::time_expunged.is_null()), - ) - }; - - use db::schema::metric_producer::dsl; - - // TODO: see https://github.com/oxidecomputer/omicron/issues/323 - let n = diesel::insert_into(dsl::metric_producer) - .values(producer_subquery) - .on_conflict(dsl::id) - .do_update() - .set(( - dsl::time_modified.eq(Utc::now()), - dsl::kind.eq(producer.kind), - dsl::ip.eq(producer.ip), - dsl::port.eq(producer.port), - dsl::interval.eq(producer.interval), - )) - .execute_async(&*self.pool_connection_authorized(opctx).await?) + producer: &internal::nexus::ProducerEndpoint, + ) -> Result { + match queries::oximeter::upsert_producer(producer) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) .await - .map_err(|e| { - public_error_from_diesel( - e, - ErrorHandler::Conflict( - ResourceType::MetricProducer, - "Producer Endpoint", - ), - ) - })?; - - // We expect `n` to basically always be 1 (1 row was inserted or - // updated). It can be 0 if `producer.oximeter_id` doesn't exist or has - // been expunged. It can never be 2 or greater because - // `producer_subquery` filters on finding an exact row for its Oximeter - // instance's ID. - match n { - 0 => Err(Error::not_found_by_id( - ResourceType::Oximeter, - &producer.oximeter_id, + { + Ok(info) => Ok(info), + Err(DieselError::NotFound) => Err(Error::unavail( + "no Oximeter instances available for assignment", )), - 1 => Ok(()), - _ => Err(Error::internal_error(&format!( - "multiple rows inserted ({n}) in `producer_endpoint_create`" - ))), + Err(e) => Err(public_error_from_diesel(e, ErrorHandler::Server)), } } @@ -355,7 +295,6 @@ mod tests { use db::datastore::pub_test_utils::datastore_test; use nexus_test_utils::db::test_setup_database; use nexus_types::internal_api::params; - use omicron_common::api::external::LookupType; use omicron_common::api::internal::nexus; use omicron_test_utils::dev; use std::time::Duration; @@ -513,10 +452,137 @@ mod tests { } #[tokio::test] - async fn test_producer_endpoint_create_rejects_expunged_oximeters() { + async fn test_producer_endpoint_reassigns_if_oximeter_expunged() { + // Setup + let logctx = dev::test_setup_log( + "test_producer_endpoint_reassigns_if_oximeter_expunged", + ); + let mut db = test_setup_database(&logctx.log).await; + let (opctx, datastore) = + datastore_test(&logctx, &db, Uuid::new_v4()).await; + + // Insert an Oximeter collector. + let oximeter1_id = Uuid::new_v4(); + datastore + .oximeter_create( + &opctx, + &OximeterInfo::new(¶ms::OximeterInfo { + collector_id: oximeter1_id, + address: "[::1]:0".parse().unwrap(), // unused + }), + ) + .await + .expect("inserted collector"); + + // Insert a producer. + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), + interval: Duration::from_secs(0), + }; + let chosen_oximeter = datastore + .producer_endpoint_upsert_and_assign(&opctx, &producer) + .await + .expect("inserted producer"); + assert_eq!(chosen_oximeter.id, oximeter1_id); + + // Grab the inserted producer (so we have its time_modified for checks + // below). + let producer_info = datastore + .producers_list_by_oximeter_id( + &opctx, + oximeter1_id, + &DataPageParams::max_page(), + ) + .await + .expect("listed producers") + .pop() + .expect("got producer"); + assert_eq!(producer_info.id(), producer.id); + + // Expunge the oximeter. + datastore + .oximeter_expunge(&opctx, oximeter1_id) + .await + .expect("expunged oximeter"); + + // Attempting to upsert our producer again should fail; our oximeter has + // been expunged, and our time modified should be unchanged. + let err = datastore + .producer_endpoint_upsert_and_assign(&opctx, &producer) + .await + .expect_err("producer upsert failed") + .to_string(); + assert!( + err.contains("no Oximeter instances available for assignment"), + "unexpected error: {err}" + ); + { + let check_info = datastore + .producers_list_by_oximeter_id( + &opctx, + oximeter1_id, + &DataPageParams::max_page(), + ) + .await + .expect("listed producers") + .pop() + .expect("got producer"); + assert_eq!( + producer_info, check_info, + "unexpected modification in failed upsert" + ); + } + + // Add a new, non-expunged Oximeter. + let oximeter2_id = Uuid::new_v4(); + datastore + .oximeter_create( + &opctx, + &OximeterInfo::new(¶ms::OximeterInfo { + collector_id: oximeter2_id, + address: "[::1]:0".parse().unwrap(), // unused + }), + ) + .await + .expect("inserted collector"); + + // Retry updating our existing producer; it should get reassigned to a + // the new Oximeter. + let chosen_oximeter = datastore + .producer_endpoint_upsert_and_assign(&opctx, &producer) + .await + .expect("inserted producer"); + assert_eq!(chosen_oximeter.id, oximeter2_id); + { + let check_info = datastore + .producers_list_by_oximeter_id( + &opctx, + oximeter2_id, + &DataPageParams::max_page(), + ) + .await + .expect("listed producers") + .pop() + .expect("got producer"); + assert_eq!(check_info.id(), producer_info.id()); + assert!( + check_info.time_modified() > producer_info.time_modified(), + "producer time modified was not advanced" + ); + } + + // Cleanup + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn test_producer_endpoint_upsert_rejects_expunged_oximeters() { // Setup let logctx = dev::test_setup_log( - "test_producer_endpoint_create_rejects_expunged_oximeters", + "test_producer_endpoint_upsert_rejects_expunged_oximeters", ); let mut db = test_setup_database(&logctx.log).await; let (opctx, datastore) = @@ -535,77 +601,87 @@ mod tests { .expect("inserted collector"); } - // We can insert metric producers for each collector. - for &collector_id in &collector_ids { - let producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_id, - ); - datastore - .producer_endpoint_create(&opctx, &producer) + // Creating a producer randomly chooses one of our collectors. Create + // 1000 and check that we saw each collector at least once. + let mut seen_collector_counts = vec![0; collector_ids.len()]; + for _ in 0..1000 { + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; + let collector_id = datastore + .producer_endpoint_upsert_and_assign(&opctx, &producer) .await - .expect("created producer"); + .expect("inserted producer") + .id; + let i = collector_ids + .iter() + .position(|id| *id == collector_id) + .expect("found collector position"); + seen_collector_counts[i] += 1; + } + eprintln!("saw collector counts: {seen_collector_counts:?}"); + for count in seen_collector_counts { + assert_ne!(count, 0); } - // Delete the first collector. + // Expunge the first collector. datastore .oximeter_expunge(&opctx, collector_ids[0]) .await .expect("expunged collector"); - // Attempting to insert a producer assigned to the first collector - // should fail, now that it's expunged. - let err = { - let producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_ids[0], - ); - datastore - .producer_endpoint_create(&opctx, &producer) + // Repeat the test above; we should never see collector 0 chosen. + let mut seen_collector_counts = vec![0; collector_ids.len()]; + for _ in 0..1000 { + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; + let collector_id = datastore + .producer_endpoint_upsert_and_assign(&opctx, &producer) .await - .expect_err("producer creation fails") - }; - assert_eq!( - err, - Error::ObjectNotFound { - type_name: ResourceType::Oximeter, - lookup_type: LookupType::ById(collector_ids[0]) - } - ); + .expect("inserted producer") + .id; + let i = collector_ids + .iter() + .position(|id| *id == collector_id) + .expect("found collector position"); + seen_collector_counts[i] += 1; + } + eprintln!("saw collector counts: {seen_collector_counts:?}"); + assert_eq!(seen_collector_counts[0], 0); + for count in seen_collector_counts.into_iter().skip(1) { + assert_ne!(count, 0); + } - // We can still insert metric producers for the other collectors... + // Expunge the remaining collectors; trying to create a producer now + // should fail. for &collector_id in &collector_ids[1..] { - let mut producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_id, - ); datastore - .producer_endpoint_create(&opctx, &producer) - .await - .expect("created producer"); - - // ... and we can update them. - producer.port = 100.into(); - datastore - .producer_endpoint_create(&opctx, &producer) + .oximeter_expunge(&opctx, collector_id) .await - .expect("created producer"); + .expect("expunged collector"); } + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; + let err = datastore + .producer_endpoint_upsert_and_assign(&opctx, &producer) + .await + .expect_err("unexpected success - all oximeters expunged") + .to_string(); + assert!( + err.contains("no Oximeter instances available for assignment"), + "unexpected error: {err}" + ); // Cleanup db.cleanup().await.unwrap(); @@ -633,26 +709,35 @@ mod tests { .expect("inserted collector"); } - // Insert 250 metric producers assigned to each collector. - for &collector_id in &collector_ids { - for _ in 0..250 { - let producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_id, - ); - datastore - .producer_endpoint_create(&opctx, &producer) - .await - .expect("created producer"); - } + // Insert 1000 metric producers. + let mut seen_collector_counts = vec![0; collector_ids.len()]; + for _ in 0..1000 { + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; + let collector_id = datastore + .producer_endpoint_upsert_and_assign(&opctx, &producer) + .await + .expect("inserted producer") + .id; + let i = collector_ids + .iter() + .position(|id| *id == collector_id) + .expect("found collector position"); + seen_collector_counts[i] += 1; } + eprintln!("saw collector counts: {seen_collector_counts:?}"); + // Sanity check that we got at least one assignment to collector 0 (so + // our reassignment below actually does something). + assert!( + seen_collector_counts[0] > 0, + "expected more than 0 assignments to collector 0 (very unlucky?!)" + ); - // Delete one collector. + // Expunge one collector. datastore .oximeter_expunge(&opctx, collector_ids[0]) .await @@ -663,7 +748,10 @@ mod tests { .oximeter_reassign_all_producers(&opctx, collector_ids[0]) .await .expect("reassigned producers"); - assert_eq!(num_reassigned, CollectorReassignment::Complete(250)); + assert_eq!( + num_reassigned, + CollectorReassignment::Complete(seen_collector_counts[0]) + ); // Check the distribution of producers for each of the remaining // collectors. We don't know the exact count, so we'll check that: @@ -673,10 +761,10 @@ mod tests { // enough that most calculators give up and call it 0) // * All 1000 producers are assigned to one of the three collectors // - // to guard against "the reassignment query gave all 250 to exactly one - // of the remaining collectors", which is an easy failure mode for this - // kind of SQL query, where the query engine only evaluates the - // randomness once instead of once for each producer. + // to guard against "the reassignment query gave all of collector 0's + // producers to exactly one of the remaining collectors", which is an + // easy failure mode for this kind of SQL query, where the query engine + // only evaluates the randomness once instead of once for each producer. let mut producer_counts = [0; 4]; for i in 0..4 { producer_counts[i] = datastore @@ -690,9 +778,13 @@ mod tests { .len(); } assert_eq!(producer_counts[0], 0); // all reassigned - assert!(producer_counts[1] > 250); // gained at least one - assert!(producer_counts[2] > 250); // gained at least one - assert!(producer_counts[3] > 250); // gained at least one + + // each gained at least one + assert!(producer_counts[1] > seen_collector_counts[1]); + assert!(producer_counts[2] > seen_collector_counts[2]); + assert!(producer_counts[3] > seen_collector_counts[3]); + + // all producers are assigned assert_eq!(producer_counts[1..].iter().sum::(), 1000); // Cleanup @@ -723,23 +815,25 @@ mod tests { .expect("inserted collector"); } - // Insert 10 metric producers assigned to each collector. - for &collector_id in &collector_ids { - for _ in 0..10 { - let producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_id, - ); - datastore - .producer_endpoint_create(&opctx, &producer) - .await - .expect("created producer"); - } + // Insert 100 metric producers. + let mut seen_collector_counts = vec![0; collector_ids.len()]; + for _ in 0..100 { + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; + let collector_id = datastore + .producer_endpoint_upsert_and_assign(&opctx, &producer) + .await + .expect("inserted producer") + .id; + let i = collector_ids + .iter() + .position(|id| *id == collector_id) + .expect("found collector position"); + seen_collector_counts[i] += 1; } // Delete all four collectors. @@ -777,15 +871,18 @@ mod tests { .expect("inserted collector"); // Reassigning the original four collectors should now all succeed. - for &collector_id in &collector_ids { + for (i, &collector_id) in collector_ids.iter().enumerate() { let num_reassigned = datastore .oximeter_reassign_all_producers(&opctx, collector_id) .await .expect("reassigned producers"); - assert_eq!(num_reassigned, CollectorReassignment::Complete(10)); + assert_eq!( + num_reassigned, + CollectorReassignment::Complete(seen_collector_counts[i]) + ); } - // All 40 producers should be assigned to our new collector. + // All 100 producers should be assigned to our new collector. let nproducers = datastore .producers_list_by_oximeter_id( &opctx, @@ -795,7 +892,7 @@ mod tests { .await .expect("listed producers") .len(); - assert_eq!(nproducers, 40); + assert_eq!(nproducers, 100); // Cleanup db.cleanup().await.unwrap(); @@ -821,17 +918,14 @@ mod tests { .expect("failed to insert collector"); // Insert a producer - let producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_info.id, - ); + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; datastore - .producer_endpoint_create(&opctx, &producer) + .producer_endpoint_upsert_and_assign(&opctx, &producer) .await .expect("failed to insert producer"); @@ -845,7 +939,7 @@ mod tests { .await .expect("failed to list all producers"); assert_eq!(all_producers.len(), 1); - assert_eq!(all_producers[0].id(), producer.id()); + assert_eq!(all_producers[0].id(), producer.id); // Steal this producer so we have a database-precision timestamp and can // use full equality checks moving forward. diff --git a/nexus/db-queries/src/db/datastore/physical_disk.rs b/nexus/db-queries/src/db/datastore/physical_disk.rs index 6326f385ad..61852e454e 100644 --- a/nexus/db-queries/src/db/datastore/physical_disk.rs +++ b/nexus/db-queries/src/db/datastore/physical_disk.rs @@ -330,7 +330,7 @@ mod test { use dropshot::PaginationOrder; use nexus_db_model::Generation; use nexus_sled_agent_shared::inventory::{ - Baseboard, Inventory, InventoryDisk, SledRole, + Baseboard, Inventory, InventoryDisk, OmicronZonesConfig, SledRole, }; use nexus_test_utils::db::test_setup_database; use nexus_types::identity::Asset; @@ -696,6 +696,10 @@ mod test { sled_id: SledUuid::from_untyped_uuid(sled.id()), usable_hardware_threads: 10, usable_physical_ram: ByteCount::from(1024 * 1024), + omicron_zones: OmicronZonesConfig { + generation: OmicronZonesConfig::INITIAL_GENERATION, + zones: vec![], + }, disks, zpools: vec![], datasets: vec![], diff --git a/nexus/db-queries/src/db/datastore/project.rs b/nexus/db-queries/src/db/datastore/project.rs index 42ccca4ed6..58b7b315c1 100644 --- a/nexus/db-queries/src/db/datastore/project.rs +++ b/nexus/db-queries/src/db/datastore/project.rs @@ -26,7 +26,7 @@ use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use nexus_db_fixed_data::project::SERVICES_PROJECT; -use nexus_db_fixed_data::silo::INTERNAL_SILO_ID; +use nexus_types::silo::INTERNAL_SILO_ID; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::CreateResult; use omicron_common::api::external::DeleteResult; @@ -103,7 +103,7 @@ impl DataStore { debug!(opctx.log, "attempting to create built-in projects"); let (authz_silo,) = db::lookup::LookupPath::new(&opctx, self) - .silo_id(*INTERNAL_SILO_ID) + .silo_id(INTERNAL_SILO_ID) .lookup_for(authz::Action::CreateChild) .await?; diff --git a/nexus/db-queries/src/db/datastore/rack.rs b/nexus/db-queries/src/db/datastore/rack.rs index 8b7cf4804a..534854d2df 100644 --- a/nexus/db-queries/src/db/datastore/rack.rs +++ b/nexus/db-queries/src/db/datastore/rack.rs @@ -33,7 +33,6 @@ use diesel::prelude::*; use diesel::result::Error as DieselError; use diesel::upsert::excluded; use ipnetwork::IpNetwork; -use nexus_db_fixed_data::silo::INTERNAL_SILO_ID; use nexus_db_fixed_data::vpc_subnet::DNS_VPC_SUBNET; use nexus_db_fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; use nexus_db_fixed_data::vpc_subnet::NTP_VPC_SUBNET; @@ -57,6 +56,7 @@ use nexus_types::external_api::shared::IdentityType; use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::shared::SiloRole; use nexus_types::identity::Resource; +use nexus_types::silo::INTERNAL_SILO_ID; use omicron_common::api::external::AllowedSourceIps; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; @@ -983,7 +983,7 @@ impl DataStore { db::model::IpPoolResource { ip_pool_id: internal_pool_id, resource_type: db::model::IpPoolResourceType::Silo, - resource_id: *INTERNAL_SILO_ID, + resource_id: INTERNAL_SILO_ID, is_default: true, }, ) @@ -1007,6 +1007,7 @@ mod test { use crate::db::model::IpPoolRange; use crate::db::model::Sled; use async_bb8_diesel::AsyncSimpleConnection; + use internal_dns_types::names::DNS_ZONE; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; use nexus_db_model::{DnsGroup, Generation, InitialDnsGroup, SledUpdate}; use nexus_inventory::now_db_precision; @@ -1066,6 +1067,7 @@ mod test { internal_dns_version: *Generation::new(), external_dns_version: *Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: Utc::now(), creator: "test suite".to_string(), comment: "test suite".to_string(), @@ -1076,14 +1078,14 @@ mod test { service_ip_pool_ranges: vec![], internal_dns: InitialDnsGroup::new( DnsGroup::Internal, - internal_dns::DNS_ZONE, + DNS_ZONE, "test suite", "test suite", HashMap::new(), ), external_dns: InitialDnsGroup::new( DnsGroup::External, - internal_dns::DNS_ZONE, + DNS_ZONE, "test suite", "test suite", HashMap::new(), @@ -1103,7 +1105,7 @@ mod test { }, recovery_silo_fq_dns_name: format!( "test-silo.sys.{}", - internal_dns::DNS_ZONE + DNS_ZONE ), recovery_user_id: "test-user".parse().unwrap(), // empty string password @@ -1549,6 +1551,7 @@ mod test { internal_dns_version: *Generation::new(), external_dns_version: *Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: now_db_precision(), creator: "test suite".to_string(), comment: "test blueprint".to_string(), @@ -1780,7 +1783,7 @@ mod test { ]; let internal_dns = InitialDnsGroup::new( DnsGroup::Internal, - internal_dns::DNS_ZONE, + DNS_ZONE, "test suite", "initial test suite internal rev", HashMap::from([("nexus".to_string(), internal_records.clone())]), @@ -1810,6 +1813,7 @@ mod test { internal_dns_version: *Generation::new(), external_dns_version: *Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: now_db_precision(), creator: "test suite".to_string(), comment: "test blueprint".to_string(), @@ -1921,10 +1925,7 @@ mod test { .unwrap(); assert_eq!(dns_config_internal.generation, 1); assert_eq!(dns_config_internal.zones.len(), 1); - assert_eq!( - dns_config_internal.zones[0].zone_name, - internal_dns::DNS_ZONE - ); + assert_eq!(dns_config_internal.zones[0].zone_name, DNS_ZONE); assert_eq!( dns_config_internal.zones[0].records, HashMap::from([("nexus".to_string(), internal_records)]), @@ -2024,6 +2025,7 @@ mod test { internal_dns_version: *Generation::new(), external_dns_version: *Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: now_db_precision(), creator: "test suite".to_string(), comment: "test blueprint".to_string(), @@ -2167,6 +2169,7 @@ mod test { internal_dns_version: *Generation::new(), external_dns_version: *Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: now_db_precision(), creator: "test suite".to_string(), comment: "test blueprint".to_string(), diff --git a/nexus/db-queries/src/db/datastore/region_replacement.rs b/nexus/db-queries/src/db/datastore/region_replacement.rs index 62598d710e..b9b5a0827f 100644 --- a/nexus/db-queries/src/db/datastore/region_replacement.rs +++ b/nexus/db-queries/src/db/datastore/region_replacement.rs @@ -657,7 +657,7 @@ impl DataStore { /// Transition a RegionReplacement record from Completing to Complete, /// clearing the operating saga id. Also removes the `volume_repair` record - /// that is taking a "lock" on the Volume. + /// that is taking a lock on the Volume. pub async fn set_region_replacement_complete( &self, opctx: &OpContext, @@ -723,6 +723,75 @@ impl DataStore { }) } + /// Transition a RegionReplacement record from Requested to Complete, which + /// occurs when the associated volume is soft or hard deleted. Also removes + /// the `volume_repair` record that is taking a lock on the Volume. + pub async fn set_region_replacement_complete_from_requested( + &self, + opctx: &OpContext, + request: RegionReplacement, + ) -> Result<(), Error> { + type TxnError = TransactionError; + + assert_eq!( + request.replacement_state, + RegionReplacementState::Requested, + ); + + self.pool_connection_authorized(opctx) + .await? + .transaction_async(|conn| async move { + Self::volume_repair_delete_query( + request.volume_id, + request.id, + ) + .execute_async(&conn) + .await?; + + use db::schema::region_replacement::dsl; + + let result = diesel::update(dsl::region_replacement) + .filter(dsl::id.eq(request.id)) + .filter( + dsl::replacement_state.eq(RegionReplacementState::Requested), + ) + .filter(dsl::operating_saga_id.is_null()) + .set(( + dsl::replacement_state.eq(RegionReplacementState::Complete), + )) + .check_if_exists::(request.id) + .execute_and_check(&conn) + .await?; + + match result.status { + UpdateStatus::Updated => Ok(()), + + UpdateStatus::NotUpdatedButExists => { + let record = result.found; + + if record.replacement_state == RegionReplacementState::Complete { + Ok(()) + } else { + Err(TxnError::CustomError(Error::conflict(format!( + "region replacement {} set to {:?} (operating saga id {:?})", + request.id, + record.replacement_state, + record.operating_saga_id, + )))) + } + } + } + }) + .await + .map_err(|e| match e { + TxnError::CustomError(error) => error, + + TxnError::Database(error) => { + public_error_from_diesel(error, ErrorHandler::Server) + } + }) + } + /// Nexus has been notified by an Upstairs (or has otherwised determined) /// that a region replacement is done, so update the record. Filter on the /// following: diff --git a/nexus/db-queries/src/db/datastore/switch_port.rs b/nexus/db-queries/src/db/datastore/switch_port.rs index 59748aa4db..61335ccab4 100644 --- a/nexus/db-queries/src/db/datastore/switch_port.rs +++ b/nexus/db-queries/src/db/datastore/switch_port.rs @@ -1237,7 +1237,7 @@ async fn do_switch_port_settings_create( route.dst.into(), route.gw.into(), route.vid.map(Into::into), - route.local_pref.map(Into::into), + route.rib_priority.map(Into::into), )); } } diff --git a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs index 0e200f47bb..e838d38d37 100644 --- a/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs +++ b/nexus/db-queries/src/db/datastore/virtual_provisioning_collection.rs @@ -333,6 +333,7 @@ mod test { use nexus_db_model::SiloQuotasUpdate; use nexus_test_utils::db::test_setup_database; use nexus_types::external_api::params; + use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_test_utils::dev; use uuid::Uuid; @@ -380,7 +381,7 @@ mod test { opctx: &OpContext, ) -> TestData { let fleet_id = *nexus_db_fixed_data::FLEET_ID; - let silo_id = *nexus_db_fixed_data::silo::DEFAULT_SILO_ID; + let silo_id = DEFAULT_SILO_ID; let project_id = Uuid::new_v4(); let (authz_project, _project) = datastore diff --git a/nexus/db-queries/src/db/datastore/volume.rs b/nexus/db-queries/src/db/datastore/volume.rs index c643b86d24..3bd0ef41ed 100644 --- a/nexus/db-queries/src/db/datastore/volume.rs +++ b/nexus/db-queries/src/db/datastore/volume.rs @@ -1572,6 +1572,14 @@ impl DataStore { .optional() .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + + /// Return true if a volume was soft-deleted or hard-deleted + pub async fn volume_deleted(&self, volume_id: Uuid) -> Result { + match self.volume_get(volume_id).await? { + Some(v) => Ok(v.time_deleted.is_some()), + None => Ok(true), + } + } } #[derive(Default, Clone, Debug, Serialize, Deserialize)] @@ -2049,13 +2057,20 @@ impl DataStore { let old_volume = if let Some(old_volume) = maybe_old_volume { old_volume } else { - // Existing volume was deleted, so return here. We can't - // perform the region replacement now, and this will - // short-circuit the rest of the process. + // Existing volume was hard-deleted, so return here. We + // can't perform the region replacement now, and this + // will short-circuit the rest of the process. return Ok(VolumeReplaceResult::ExistingVolumeDeleted); }; + if old_volume.time_deleted.is_some() { + // Existing volume was soft-deleted, so return here for + // the same reason: the region replacement process + // should be short-circuited now. + return Ok(VolumeReplaceResult::ExistingVolumeDeleted); + } + let old_vcr: VolumeConstructionRequest = match serde_json::from_str(&old_volume.data()) { Ok(vcr) => vcr, @@ -2260,13 +2275,20 @@ impl DataStore { let old_volume = if let Some(old_volume) = maybe_old_volume { old_volume } else { - // Existing volume was deleted, so return here. We can't - // perform the region snapshot replacement now, and this + // Existing volume was hard-deleted, so return here. We + // can't perform the region replacement now, and this // will short-circuit the rest of the process. return Ok(VolumeReplaceResult::ExistingVolumeDeleted); }; + if old_volume.time_deleted.is_some() { + // Existing volume was soft-deleted, so return here for + // the same reason: the region replacement process + // should be short-circuited now. + return Ok(VolumeReplaceResult::ExistingVolumeDeleted); + } + let old_vcr: VolumeConstructionRequest = match serde_json::from_str(&old_volume.data()) { Ok(vcr) => vcr, diff --git a/nexus/db-queries/src/db/datastore/vpc.rs b/nexus/db-queries/src/db/datastore/vpc.rs index 27435fa3aa..473efb2575 100644 --- a/nexus/db-queries/src/db/datastore/vpc.rs +++ b/nexus/db-queries/src/db/datastore/vpc.rs @@ -50,7 +50,16 @@ use diesel::result::DatabaseErrorKind; use diesel::result::Error as DieselError; use futures::stream::{self, StreamExt}; use ipnetwork::IpNetwork; +use nexus_db_fixed_data::vpc::SERVICES_INTERNET_GATEWAY_DEFAULT_ROUTE_V4; +use nexus_db_fixed_data::vpc::SERVICES_INTERNET_GATEWAY_DEFAULT_ROUTE_V6; +use nexus_db_fixed_data::vpc::SERVICES_INTERNET_GATEWAY_ID; use nexus_db_fixed_data::vpc::SERVICES_VPC_ID; +use nexus_db_model::ExternalIp; +use nexus_db_model::InternetGateway; +use nexus_db_model::InternetGatewayIpAddress; +use nexus_db_model::InternetGatewayIpPool; +use nexus_db_model::IpPoolRange; +use nexus_db_model::NetworkInterfaceKind; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::SledFilter; use omicron_common::api::external::http_pagination::PaginatedBy; @@ -68,6 +77,7 @@ use omicron_common::api::external::RouteTarget; use omicron_common::api::external::RouterRouteKind as ExternalRouteKind; use omicron_common::api::external::UpdateResult; use omicron_common::api::external::Vni as ExternalVni; +use omicron_common::api::internal::shared::ResolvedVpcRoute; use omicron_common::api::internal::shared::RouterTarget; use oxnet::IpNet; use ref_cast::RefCast; @@ -85,8 +95,6 @@ impl DataStore { ) -> Result<(), Error> { use nexus_db_fixed_data::project::SERVICES_PROJECT_ID; use nexus_db_fixed_data::vpc::SERVICES_VPC; - use nexus_db_fixed_data::vpc::SERVICES_VPC_DEFAULT_V4_ROUTE_ID; - use nexus_db_fixed_data::vpc::SERVICES_VPC_DEFAULT_V6_ROUTE_ID; opctx.authorize(authz::Action::Modify, &authz::DATABASE).await?; @@ -121,6 +129,25 @@ impl DataStore { Err(e) => Err(e), }?; + let igw = db::model::InternetGateway::new( + *SERVICES_INTERNET_GATEWAY_ID, + authz_vpc.id(), + nexus_types::external_api::params::InternetGatewayCreate { + identity: IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: String::from("Default VPC gateway"), + }, + }, + ); + + match self.vpc_create_internet_gateway(&opctx, &authz_vpc, igw).await { + Ok(_) => Ok(()), + Err(e) => match e { + Error::ObjectAlreadyExists { .. } => Ok(()), + _ => Err(e), + }, + }?; + // Also add the system router and internet gateway route let system_router = db::lookup::LookupPath::new(opctx, self) @@ -147,38 +174,41 @@ impl DataStore { .map(|(authz_router, _)| authz_router)? }; - // Unwrap safety: these are known valid CIDR blocks. - let default_ips = [ - ( - "default-v4", - "0.0.0.0/0".parse().unwrap(), - *SERVICES_VPC_DEFAULT_V4_ROUTE_ID, - ), - ( - "default-v6", - "::/0".parse().unwrap(), - *SERVICES_VPC_DEFAULT_V6_ROUTE_ID, - ), - ]; + let default_v4 = RouterRoute::new( + *SERVICES_INTERNET_GATEWAY_DEFAULT_ROUTE_V4, + SERVICES_VPC.system_router_id, + ExternalRouteKind::Default, + nexus_types::external_api::params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "default-v4".parse().unwrap(), + description: String::from("Default IPv4 route"), + }, + target: RouteTarget::InternetGateway( + "default".parse().unwrap(), + ), + destination: RouteDestination::IpNet( + "0.0.0.0/0".parse().unwrap(), + ), + }, + ); - for (name, default, uuid) in default_ips { - let route = RouterRoute::new( - uuid, - SERVICES_VPC.system_router_id, - ExternalRouteKind::Default, - nexus_types::external_api::params::RouterRouteCreate { - identity: IdentityMetadataCreateParams { - name: name.parse().unwrap(), - description: - "Default internet gateway route for Oxide Services" - .to_string(), - }, - target: RouteTarget::InternetGateway( - "outbound".parse().unwrap(), - ), - destination: RouteDestination::IpNet(default), + let default_v6 = RouterRoute::new( + *SERVICES_INTERNET_GATEWAY_DEFAULT_ROUTE_V6, + SERVICES_VPC.system_router_id, + ExternalRouteKind::Default, + nexus_types::external_api::params::RouterRouteCreate { + identity: IdentityMetadataCreateParams { + name: "default-v6".parse().unwrap(), + description: String::from("Default IPv6 route"), }, - ); + target: RouteTarget::InternetGateway( + "default".parse().unwrap(), + ), + destination: RouteDestination::IpNet("::/0".parse().unwrap()), + }, + ); + + for route in [default_v4, default_v6] { self.router_create_route(opctx, &authz_router, route) .await .map(|_| ()) @@ -1082,6 +1112,137 @@ impl DataStore { .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) } + pub async fn internet_gateway_list( + &self, + opctx: &OpContext, + authz_vpc: &authz::Vpc, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::ListChildren, authz_vpc).await?; + + use db::schema::internet_gateway::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::internet_gateway, dsl::id, pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::internet_gateway, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::time_deleted.is_null()) + .filter(dsl::vpc_id.eq(authz_vpc.id())) + .select(InternetGateway::as_select()) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn internet_gateway_has_ip_pools( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + ) -> LookupResult { + opctx.authorize(authz::Action::ListChildren, authz_igw).await?; + + use db::schema::internet_gateway_ip_pool::dsl; + let result = dsl::internet_gateway_ip_pool + .filter(dsl::time_deleted.is_null()) + .filter(dsl::internet_gateway_id.eq(authz_igw.id())) + .select(InternetGatewayIpPool::as_select()) + .limit(1) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(!result.is_empty()) + } + + pub async fn internet_gateway_has_ip_addresses( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + ) -> LookupResult { + opctx.authorize(authz::Action::ListChildren, authz_igw).await?; + + use db::schema::internet_gateway_ip_address::dsl; + let result = dsl::internet_gateway_ip_address + .filter(dsl::time_deleted.is_null()) + .filter(dsl::internet_gateway_id.eq(authz_igw.id())) + .select(InternetGatewayIpAddress::as_select()) + .limit(1) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(!result.is_empty()) + } + + pub async fn internet_gateway_list_ip_pools( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::ListChildren, authz_igw).await?; + + use db::schema::internet_gateway_ip_pool::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::internet_gateway_ip_pool, dsl::id, pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::internet_gateway_ip_pool, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::time_deleted.is_null()) + .filter(dsl::internet_gateway_id.eq(authz_igw.id())) + .select(InternetGatewayIpPool::as_select()) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + pub async fn internet_gateway_list_ip_addresses( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + opctx.authorize(authz::Action::ListChildren, authz_igw).await?; + + use db::schema::internet_gateway_ip_address::dsl; + match pagparams { + PaginatedBy::Id(pagparams) => { + paginated(dsl::internet_gateway_ip_address, dsl::id, pagparams) + } + PaginatedBy::Name(pagparams) => paginated( + dsl::internet_gateway_ip_address, + dsl::name, + &pagparams.map_name(|n| Name::ref_cast(n)), + ), + } + .filter(dsl::time_deleted.is_null()) + .filter(dsl::internet_gateway_id.eq(authz_igw.id())) + .select(InternetGatewayIpAddress::as_select()) + .load_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + pub async fn vpc_create_router( &self, opctx: &OpContext, @@ -1118,6 +1279,45 @@ impl DataStore { )) } + pub async fn vpc_create_internet_gateway( + &self, + opctx: &OpContext, + authz_vpc: &authz::Vpc, + igw: InternetGateway, + ) -> CreateResult<(authz::InternetGateway, InternetGateway)> { + opctx.authorize(authz::Action::CreateChild, authz_vpc).await?; + + use db::schema::internet_gateway::dsl; + let name = igw.name().clone(); + let igw = diesel::insert_into(dsl::internet_gateway) + .values(igw) + .returning(InternetGateway::as_returning()) + .get_result_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::InternetGateway, + name.as_str(), + ), + ) + })?; + + // This is a named resource in router rules, so router resolution + // will change on add/delete. + self.vpc_increment_rpw_version(opctx, authz_vpc.id()).await?; + + Ok(( + authz::InternetGateway::new( + authz_vpc.clone(), + igw.id(), + LookupType::ById(igw.id()), + ), + igw, + )) + } + pub async fn vpc_delete_router( &self, opctx: &OpContext, @@ -1168,6 +1368,176 @@ impl DataStore { Ok(()) } + pub async fn vpc_delete_internet_gateway( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + vpc_id: Uuid, + cascade: bool, + ) -> DeleteResult { + let res = if cascade { + self.vpc_delete_internet_gateway_cascade(opctx, authz_igw).await + } else { + self.vpc_delete_internet_gateway_no_cascade(opctx, authz_igw).await + }; + + if res.is_ok() { + // This is a named resource in router rules, so router resolution + // will change on add/delete. + self.vpc_increment_rpw_version(opctx, vpc_id).await?; + } + + res + } + + pub async fn vpc_delete_internet_gateway_no_cascade( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_igw).await?; + let conn = self.pool_connection_authorized(opctx).await?; + let err = OptionalError::new(); + + #[derive(Debug)] + enum DeleteError { + IpPoolsExist, + IpAddressesExist, + } + + self.transaction_retry_wrapper("vpc_delete_internet_gateway_no_cascade") + .transaction(&conn, |conn| { + let err = err.clone(); + async move { + // Delete ip pool associations + use db::schema::internet_gateway_ip_pool::dsl as pool; + let count = pool::internet_gateway_ip_pool + .filter(pool::time_deleted.is_null()) + .filter(pool::internet_gateway_id.eq(authz_igw.id())) + .count() + .first_async::(&conn) + .await?; + if count > 0 { + return Err(err.bail(DeleteError::IpPoolsExist)); + } + + // Delete ip address associations + use db::schema::internet_gateway_ip_address::dsl as addr; + let count = addr::internet_gateway_ip_address + .filter(addr::time_deleted.is_null()) + .filter(addr::internet_gateway_id.eq(authz_igw.id())) + .count() + .first_async::(&conn) + .await?; + if count > 0 { + return Err(err.bail(DeleteError::IpAddressesExist)); + } + + use db::schema::internet_gateway::dsl; + let now = Utc::now(); + diesel::update(dsl::internet_gateway) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_igw.id())) + .set(dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + DeleteError::IpPoolsExist => Error::invalid_request("Ip pools referencing this gateway exist. To perform a cascading delete set the cascade option"), + DeleteError::IpAddressesExist => Error::invalid_request("Ip addresses referencing this gateway exist. To perform a cascading delete set the cascade option"), + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn vpc_delete_internet_gateway_cascade( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_igw).await?; + let conn = self.pool_connection_authorized(opctx).await?; + + self.transaction_retry_wrapper("vpc_delete_internet_gateway_cascade") + .transaction(&conn, |conn| { + async move { + use db::schema::internet_gateway::dsl as igw; + let igw_info = igw::internet_gateway + .filter(igw::time_deleted.is_null()) + .filter(igw::id.eq(authz_igw.id())) + .select(InternetGateway::as_select()) + .first_async(&conn) + .await?; + + use db::schema::internet_gateway::dsl; + let now = Utc::now(); + diesel::update(dsl::internet_gateway) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_igw.id())) + .set(dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + // Delete ip pool associations + use db::schema::internet_gateway_ip_pool::dsl as pool; + let now = Utc::now(); + diesel::update(pool::internet_gateway_ip_pool) + .filter(pool::time_deleted.is_null()) + .filter(pool::internet_gateway_id.eq(authz_igw.id())) + .set(pool::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + // Delete ip address associations + use db::schema::internet_gateway_ip_address::dsl as addr; + let now = Utc::now(); + diesel::update(addr::internet_gateway_ip_address) + .filter(addr::time_deleted.is_null()) + .filter(addr::internet_gateway_id.eq(authz_igw.id())) + .set(addr::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + // Delete routes targeting this igw + use db::schema::vpc_router::dsl as vr; + let vpc_routers = vr::vpc_router + .filter(vr::time_deleted.is_null()) + .filter(vr::vpc_id.eq(igw_info.vpc_id)) + .select(VpcRouter::as_select()) + .load_async(&conn) + .await? + .into_iter() + .map(|x| x.id()) + .collect::>(); + + use db::schema::router_route::dsl as rr; + let now = Utc::now(); + diesel::update(rr::router_route) + .filter(rr::time_deleted.is_null()) + .filter(rr::vpc_router_id.eq_any(vpc_routers)) + .filter( + rr::target + .eq(format!("inetgw:{}", igw_info.name())), + ) + .set(rr::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + pub async fn vpc_update_router( &self, opctx: &OpContext, @@ -1266,6 +1636,76 @@ impl DataStore { }) } + pub async fn internet_gateway_attach_ip_pool( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + igwip: InternetGatewayIpPool, + ) -> CreateResult { + use db::schema::internet_gateway_ip_pool::dsl; + opctx.authorize(authz::Action::CreateChild, authz_igw).await?; + + let igw_id = igwip.internet_gateway_id; + let name = igwip.name().clone(); + + InternetGateway::insert_resource( + igw_id, + diesel::insert_into(dsl::internet_gateway_ip_pool).values(igwip), + ) + .insert_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { + type_name: ResourceType::InternetGateway, + lookup_type: LookupType::ById(igw_id), + }, + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::InternetGatewayIpPool, + name.as_str(), + ), + ), + }) + } + + pub async fn internet_gateway_attach_ip_address( + &self, + opctx: &OpContext, + authz_igw: &authz::InternetGateway, + igwip: InternetGatewayIpAddress, + ) -> CreateResult { + use db::schema::internet_gateway_ip_address::dsl; + opctx.authorize(authz::Action::CreateChild, authz_igw).await?; + + let igw_id = igwip.internet_gateway_id; + let name = igwip.name().clone(); + + InternetGateway::insert_resource( + igw_id, + diesel::insert_into(dsl::internet_gateway_ip_address).values(igwip), + ) + .insert_and_get_result_async( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| match e { + AsyncInsertError::CollectionNotFound => Error::ObjectNotFound { + type_name: ResourceType::InternetGateway, + lookup_type: LookupType::ById(igw_id), + }, + AsyncInsertError::DatabaseError(e) => public_error_from_diesel( + e, + ErrorHandler::Conflict( + ResourceType::InternetGatewayIpAddress, + name.as_str(), + ), + ), + }) + } + pub async fn router_delete_route( &self, opctx: &OpContext, @@ -1290,6 +1730,276 @@ impl DataStore { Ok(()) } + pub async fn internet_gateway_detach_ip_pool( + &self, + opctx: &OpContext, + igw_name: String, + authz_igw_pool: &authz::InternetGatewayIpPool, + ip_pool_id: Uuid, + vpc_id: Uuid, + cascade: bool, + ) -> DeleteResult { + if cascade { + self.internet_gateway_detach_ip_pool_cascade(opctx, authz_igw_pool) + .await + } else { + self.internet_gateway_detach_ip_pool_no_cascade( + opctx, + igw_name, + authz_igw_pool, + ip_pool_id, + vpc_id, + ) + .await + } + } + + pub async fn internet_gateway_detach_ip_pool_no_cascade( + &self, + opctx: &OpContext, + igw_name: String, + authz_igw_pool: &authz::InternetGatewayIpPool, + ip_pool_id: Uuid, + vpc_id: Uuid, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_igw_pool).await?; + let conn = self.pool_connection_authorized(opctx).await?; + let err = OptionalError::new(); + #[derive(Debug)] + enum DeleteError { + DependentInstances, + } + + self.transaction_retry_wrapper("internet_gateway_detach_ip_pool_no_cascade") + .transaction(&conn, |conn| { + let err = err.clone(); + let igw_name = igw_name.clone(); + async move { + // determine if there are routes that target this igw + let this_target = format!("inetgw:{}", igw_name); + use db::schema::router_route::dsl as rr; + let count = rr::router_route + .filter(rr::time_deleted.is_null()) + .filter(rr::target.eq(this_target)) + .count() + .first_async::(&conn) + .await?; + + info!(self.log, "detach ip pool: applies to {} routes", count); + + if count > 0 { + // determine if there are instances that have IP + // addresses in the IP pool being removed. + + use db::schema::ip_pool_range::dsl as ipr; + let pr = ipr::ip_pool_range + .filter(ipr::time_deleted.is_null()) + .filter(ipr::ip_pool_id.eq(ip_pool_id)) + .select(IpPoolRange::as_select()) + .first_async::(&conn) + .await?; + info!(self.log, "POOL {pr:#?}"); + + use db::schema::instance_network_interface::dsl as ini; + let vpc_interfaces = ini::instance_network_interface + .filter(ini::time_deleted.is_null()) + .filter(ini::vpc_id.eq(vpc_id)) + .select(InstanceNetworkInterface::as_select()) + .load_async::(&conn) + .await?; + + info!(self.log, "detach ip pool: applies to {} interfaces", vpc_interfaces.len()); + + for ifx in &vpc_interfaces { + info!(self.log, "IFX {ifx:#?}"); + + use db::schema::external_ip::dsl as xip; + let ext_ips = xip::external_ip + .filter(xip::time_deleted.is_null()) + .filter(xip::parent_id.eq(ifx.instance_id)) + .select(ExternalIp::as_select()) + .load_async::(&conn) + .await?; + + info!(self.log, "EXT IP {ext_ips:#?}"); + + for ext_ip in &ext_ips { + if ext_ip.ip.ip() >= pr.first_address.ip() && + ext_ip.ip.ip() <= pr.last_address.ip() { + return Err(err.bail(DeleteError::DependentInstances)); + } + } + } + } + + use db::schema::internet_gateway_ip_pool::dsl; + let now = Utc::now(); + diesel::update(dsl::internet_gateway_ip_pool) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_igw_pool.id())) + .set(dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + DeleteError::DependentInstances => Error::invalid_request("VPC routes dependent on this IP pool. To perform a cascading delete set the cascade option"), + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + + pub async fn internet_gateway_detach_ip_pool_cascade( + &self, + opctx: &OpContext, + authz_pool: &authz::InternetGatewayIpPool, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_pool).await?; + use db::schema::internet_gateway_ip_pool::dsl; + let now = Utc::now(); + diesel::update(dsl::internet_gateway_ip_pool) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_pool.id())) + .set(dsl::time_deleted.eq(now)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server))?; + + Ok(()) + } + + pub async fn internet_gateway_detach_ip_address( + &self, + opctx: &OpContext, + igw_name: String, + authz_addr: &authz::InternetGatewayIpAddress, + addr: IpAddr, + vpc_id: Uuid, + cascade: bool, + ) -> DeleteResult { + if cascade { + self.internet_gateway_detach_ip_address_cascade(opctx, authz_addr) + .await + } else { + self.internet_gateway_detach_ip_address_no_cascade( + opctx, igw_name, authz_addr, addr, vpc_id, + ) + .await + } + } + + pub async fn internet_gateway_detach_ip_address_cascade( + &self, + opctx: &OpContext, + authz_addr: &authz::InternetGatewayIpAddress, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_addr).await?; + + use db::schema::internet_gateway_ip_address::dsl; + let now = Utc::now(); + diesel::update(dsl::internet_gateway_ip_address) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_addr.id())) + .set(dsl::time_deleted.eq(now)) + .execute_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByResource(authz_addr), + ) + })?; + Ok(()) + } + + pub async fn internet_gateway_detach_ip_address_no_cascade( + &self, + opctx: &OpContext, + igw_name: String, + authz_addr: &authz::InternetGatewayIpAddress, + addr: IpAddr, + vpc_id: Uuid, + ) -> DeleteResult { + opctx.authorize(authz::Action::Delete, authz_addr).await?; + let conn = self.pool_connection_authorized(opctx).await?; + let err = OptionalError::new(); + #[derive(Debug)] + enum DeleteError { + DependentInstances, + } + self.transaction_retry_wrapper("internet_gateway_detach_ip_address_no_cascade") + .transaction(&conn, |conn| { + let err = err.clone(); + let igw_name = igw_name.clone(); + async move { + // determine if there are routes that target this igw + let this_target = format!("inetgw:{}", igw_name); + use db::schema::router_route::dsl as rr; + let count = rr::router_route + .filter(rr::time_deleted.is_null()) + .filter(rr::target.eq(this_target)) + .count() + .first_async::(&conn) + .await?; + + if count > 0 { + use db::schema::instance_network_interface::dsl as ini; + let vpc_interfaces = ini::instance_network_interface + .filter(ini::time_deleted.is_null()) + .filter(ini::vpc_id.eq(vpc_id)) + .select(InstanceNetworkInterface::as_select()) + .load_async::(&conn) + .await?; + + for ifx in &vpc_interfaces { + + use db::schema::external_ip::dsl as xip; + let ext_ips = xip::external_ip + .filter(xip::time_deleted.is_null()) + .filter(xip::parent_id.eq(ifx.instance_id)) + .select(ExternalIp::as_select()) + .load_async::(&conn) + .await?; + + for ext_ip in &ext_ips { + if ext_ip.ip.ip() == addr { + return Err(err.bail(DeleteError::DependentInstances)); + } + } + } + } + + use db::schema::internet_gateway_ip_address::dsl; + let now = Utc::now(); + diesel::update(dsl::internet_gateway_ip_address) + .filter(dsl::time_deleted.is_null()) + .filter(dsl::id.eq(authz_addr.id())) + .set(dsl::time_deleted.eq(now)) + .execute_async(&conn) + .await?; + + Ok(()) + } + }) + .await + .map_err(|e| { + if let Some(err) = err.take() { + match err { + DeleteError::DependentInstances => Error::invalid_request("VPC routes dependent on this IP pool. To perform a cascading delete set the cascade option"), + } + } else { + public_error_from_diesel(e, ErrorHandler::Server) + } + }) + } + pub async fn router_update_route( &self, opctx: &OpContext, @@ -1582,9 +2292,9 @@ impl DataStore { }) } - /// Fetch all active custom routers (and their parent subnets) + /// Fetch all active custom routers (and their associated subnets) /// in a VPC. - pub async fn vpc_get_active_custom_routers( + pub async fn vpc_get_active_custom_routers_with_associated_subnets( &self, opctx: &OpContext, vpc_id: Uuid, @@ -1616,12 +2326,157 @@ impl DataStore { }) } + /// Fetch all custom routers in a VPC. + pub async fn vpc_get_custom_routers( + &self, + opctx: &OpContext, + vpc_id: Uuid, + ) -> ListResultVec { + use db::schema::vpc_router::dsl as router_dsl; + + router_dsl::vpc_router + .filter(router_dsl::time_deleted.is_null()) + .filter(router_dsl::vpc_id.eq(vpc_id)) + .select(VpcRouter::as_select()) + .load_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| { + public_error_from_diesel( + e, + ErrorHandler::NotFoundByLookup( + ResourceType::Vpc, + LookupType::ById(vpc_id), + ), + ) + }) + } + + // XXX: maybe wants to live in external IP? + /// Returns a (Ip, NicId) -> InetGwId map which identifies, for a given sled: + /// * a) which external IPs belong to any of its services/instances/probe NICs. + /// * b) whether each IP is linked to an Internet Gateway via its parent pool. + pub async fn vpc_resolve_sled_external_ips_to_gateways( + &self, + opctx: &OpContext, + sled_id: Uuid, + ) -> Result>>, Error> { + // TODO: give GW-bound addresses preferential treatment. + use db::schema::external_ip as eip; + use db::schema::external_ip::dsl as eip_dsl; + use db::schema::internet_gateway as igw; + use db::schema::internet_gateway::dsl as igw_dsl; + use db::schema::internet_gateway_ip_address as igw_ip; + use db::schema::internet_gateway_ip_address::dsl as igw_ip_dsl; + use db::schema::internet_gateway_ip_pool as igw_pool; + use db::schema::internet_gateway_ip_pool::dsl as igw_pool_dsl; + use db::schema::network_interface as ni; + use db::schema::vmm; + + // We don't know at first glance which VPC ID each IP addr has. + // VPC info is necessary to map back to the intended gateway. + // Goal is to join sled-specific + // (IP, IP pool ID, VPC ID) X (IGW, IP pool ID) + let conn = self.pool_connection_authorized(opctx).await?; + + let mut out = HashMap::new(); + + let instance_mappings = eip_dsl::external_ip + .inner_join( + vmm::table.on(vmm::instance_id + .nullable() + .eq(eip::parent_id) + .and(vmm::sled_id.eq(sled_id))), + ) + .inner_join( + ni::table.on(ni::parent_id.nullable().eq(eip::parent_id)), + ) + .inner_join( + igw_pool::table + .on(eip_dsl::ip_pool_id.eq(igw_pool_dsl::ip_pool_id)), + ) + .inner_join( + igw::table.on(igw_dsl::id + .eq(igw_pool_dsl::internet_gateway_id) + .and(igw_dsl::vpc_id.eq(ni::vpc_id))), + ) + .filter(eip::time_deleted.is_null()) + .filter(ni::time_deleted.is_null()) + .filter(ni::is_primary.eq(true)) + .filter(vmm::time_deleted.is_null()) + .filter(igw_dsl::time_deleted.is_null()) + .filter(igw_pool_dsl::time_deleted.is_null()) + .select((eip_dsl::ip, ni::id, igw_dsl::id)) + .load_async::<(IpNetwork, Uuid, Uuid)>(&*conn) + .await; + + match instance_mappings { + Ok(map) => { + for (ip, nic_id, inet_gw_id) in map { + let per_nic: &mut HashMap<_, _> = + out.entry(nic_id).or_default(); + let igw_list: &mut HashSet<_> = + per_nic.entry(ip.ip()).or_default(); + + igw_list.insert(inet_gw_id); + } + } + Err(e) => { + return Err(Error::non_resourcetype_not_found(&format!( + "unable to find IGW mappings for sled {sled_id}: {e}" + ))) + } + } + + // Map all individual IPs bound to IGWs to NICs in their VPC. + let indiv_ip_mappings = ni::table + .inner_join(igw::table.on(igw_dsl::vpc_id.eq(ni::vpc_id))) + .inner_join( + igw_ip::table + .on(igw_ip_dsl::internet_gateway_id.eq(igw_dsl::id)), + ) + .filter(ni::time_deleted.is_null()) + .filter(ni::kind.eq(NetworkInterfaceKind::Instance)) + .filter(igw_dsl::time_deleted.is_null()) + .filter(igw_ip_dsl::time_deleted.is_null()) + .select((igw_ip_dsl::address, ni::id, igw_dsl::id)) + .load_async::<(IpNetwork, Uuid, Uuid)>(&*conn) + .await; + + match indiv_ip_mappings { + Ok(map) => { + for (ip, nic_id, inet_gw_id) in map { + let per_nic: &mut HashMap<_, _> = + out.entry(nic_id).or_default(); + let igw_list: &mut HashSet<_> = + per_nic.entry(ip.ip()).or_default(); + + igw_list.insert(inet_gw_id); + } + } + Err(e) => { + return Err(Error::non_resourcetype_not_found(&format!( + "unable to find IGW mappings for sled {sled_id}: {e}" + ))) + } + } + + // TODO: service & probe EIP mappings. + // note that the current sled-agent design + // does not yet allow us to re-ensure the set of + // external IPs for non-instance entities. + // if we insert those here, we need to be sure that + // the mappings are ignored by sled-agent for new + // services/probes/etc. + + Ok(out) + } + /// Resolve all targets in a router into concrete details. pub async fn vpc_resolve_router_rules( &self, opctx: &OpContext, vpc_router_id: Uuid, - ) -> Result, Error> { + ) -> Result, Error> { // Get all rules in target router. opctx.check_complex_operations_allowed()?; @@ -1712,6 +2567,19 @@ impl DataStore { .collect::>() .await; + let inetgws = stream::iter(inetgw_names) + .filter_map(|name| async { + db::lookup::LookupPath::new(opctx, self) + .vpc_id(authz_vpc.id()) + .internet_gateway_name(Name::ref_cast(&name)) + .fetch() + .await + .ok() + .map(|(.., igw)| (name, igw)) + }) + .collect::>() + .await; + let instances = stream::iter(instance_names) .filter_map(|name| async { db::lookup::LookupPath::new(opctx, self) @@ -1737,13 +2605,11 @@ impl DataStore { .collect::>() .await; - // TODO: validate names of Internet Gateways. - // See the discussion in `resolve_firewall_rules_for_sled_agent` on // how we should resolve name misses in route resolution. // This method adopts the same strategy: a lookup failure corresponds // to a NO-OP rule. - let mut out = HashMap::new(); + let mut out = HashSet::new(); for rule in all_rules { // Some dests/targets (e.g., subnet) resolve to *several* specifiers // to handle both v4 and v6. The user-facing API will prevent severe @@ -1772,12 +2638,12 @@ impl DataStore { RouteDestination::Vpc(_) => (None, None), }; - let (v4_target, v6_target) = match rule.target.0 { + let (v4_target, v6_target) = match &rule.target.0 { RouteTarget::Ip(ip @ IpAddr::V4(_)) => { - (Some(RouterTarget::Ip(ip)), None) + (Some(RouterTarget::Ip(*ip)), None) } RouteTarget::Ip(ip @ IpAddr::V6(_)) => { - (None, Some(RouterTarget::Ip(ip))) + (None, Some(RouterTarget::Ip(*ip))) } RouteTarget::Subnet(n) => subnets .get(&n) @@ -1808,15 +2674,17 @@ impl DataStore { (Some(RouterTarget::Drop), Some(RouterTarget::Drop)) } - // TODO: Internet Gateways. - // The semantic here is 'name match => allow', - // as the other aspect they will control is SNAT - // IP allocation. Today, presence of this rule - // allows upstream regardless of name. - RouteTarget::InternetGateway(_n) => ( - Some(RouterTarget::InternetGateway), - Some(RouterTarget::InternetGateway), - ), + // Internet gateways tag matching packets with their ID, for + // NAT IP selection. + RouteTarget::InternetGateway(n) => inetgws + .get(&n) + .map(|igw| { + ( + Some(RouterTarget::InternetGateway(Some(igw.id()))), + Some(RouterTarget::InternetGateway(Some(igw.id()))), + ) + }) + .unwrap_or_default(), // TODO: VPC Peering. RouteTarget::Vpc(_) => (None, None), @@ -1829,11 +2697,11 @@ impl DataStore { // It would be really useful to raise collisions and // misses to users, somehow. if let (Some(dest), Some(target)) = (v4_dest, v4_target) { - out.insert(dest, target); + out.insert(ResolvedVpcRoute { dest, target }); } if let (Some(dest), Some(target)) = (v6_dest, v6_target) { - out.insert(dest, target); + out.insert(ResolvedVpcRoute { dest, target }); } } @@ -2733,7 +3601,9 @@ mod tests { // And each subnet generates a v4->v4 and v6->v6. for subnet in subnets { - assert!(resolved.iter().any(|(k, v)| { + assert!(resolved.iter().any(|x| { + let k = &x.dest; + let v = &x.target; *k == subnet.ipv4_block.0.into() && match v { RouterTarget::VpcSubnet(ip) => { @@ -2742,7 +3612,9 @@ mod tests { _ => false, } })); - assert!(resolved.iter().any(|(k, v)| { + assert!(resolved.iter().any(|x| { + let k = &x.dest; + let v = &x.target; *k == subnet.ipv6_block.0.into() && match v { RouterTarget::VpcSubnet(ip) => { @@ -2894,10 +3766,10 @@ mod tests { // Verify we now have a route pointing at this instance. assert_eq!(routes.len(), 3); - assert!(routes.iter().any(|(k, v)| (*k + assert!(routes.iter().any(|x| (x.dest == "192.168.0.0/16".parse::().unwrap()) - && match v { - RouterTarget::Ip(ip) => *ip == nic.ip.ip(), + && match x.target { + RouterTarget::Ip(ip) => ip == nic.ip.ip(), _ => false, })); diff --git a/nexus/db-queries/src/db/lookup.rs b/nexus/db-queries/src/db/lookup.rs index 0999694c54..bc1368820c 100644 --- a/nexus/db-queries/src/db/lookup.rs +++ b/nexus/db-queries/src/db/lookup.rs @@ -227,11 +227,32 @@ impl<'a> LookupPath<'a> { VpcRouter::PrimaryKey(Root { lookup_root: self }, id) } + /// Select a resource of type InternetGateway, identified by its id + pub fn internet_gateway_id(self, id: Uuid) -> InternetGateway<'a> { + InternetGateway::PrimaryKey(Root { lookup_root: self }, id) + } + /// Select a resource of type RouterRoute, identified by its id pub fn router_route_id(self, id: Uuid) -> RouterRoute<'a> { RouterRoute::PrimaryKey(Root { lookup_root: self }, id) } + /// Select a resource of type InternetGatewayIpPool, identified by its id + pub fn internet_gateway_ip_pool_id( + self, + id: Uuid, + ) -> InternetGatewayIpPool<'a> { + InternetGatewayIpPool::PrimaryKey(Root { lookup_root: self }, id) + } + + /// Select a resource of type InternetGatewayIpAddress, identified by its id + pub fn internet_gateway_ip_address_id( + self, + id: Uuid, + ) -> InternetGatewayIpAddress<'a> { + InternetGatewayIpAddress::PrimaryKey(Root { lookup_root: self }, id) + } + /// Select a resource of type FloatingIp, identified by its id pub fn floating_ip_id(self, id: Uuid) -> FloatingIp<'a> { FloatingIp::PrimaryKey(Root { lookup_root: self }, id) @@ -686,7 +707,7 @@ lookup_resource! { lookup_resource! { name = "Vpc", ancestors = [ "Silo", "Project" ], - children = [ "VpcRouter", "VpcSubnet" ], + children = [ "VpcRouter", "VpcSubnet", "InternetGateway" ], lookup_by_name = true, soft_deletes = true, primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] @@ -719,6 +740,33 @@ lookup_resource! { primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] } +lookup_resource! { + name = "InternetGateway", + ancestors = [ "Silo", "Project", "Vpc" ], + children = [ "InternetGatewayIpPool", "InternetGatewayIpAddress" ], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + +lookup_resource! { + name = "InternetGatewayIpPool", + ancestors = [ "Silo", "Project", "Vpc", "InternetGateway" ], + children = [ ], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + +lookup_resource! { + name = "InternetGatewayIpAddress", + ancestors = [ "Silo", "Project", "Vpc", "InternetGateway" ], + children = [ ], + lookup_by_name = true, + soft_deletes = true, + primary_key_columns = [ { column_name = "id", rust_type = Uuid } ] +} + lookup_resource! { name = "FloatingIp", ancestors = [ "Silo", "Project" ], diff --git a/nexus/db-queries/src/db/pool.rs b/nexus/db-queries/src/db/pool.rs index dccee6fa3f..ea669a419e 100644 --- a/nexus/db-queries/src/db/pool.rs +++ b/nexus/db-queries/src/db/pool.rs @@ -8,14 +8,13 @@ use super::Config as DbConfig; use crate::db::pool_connection::{DieselPgConnector, DieselPgConnectorArgs}; +use internal_dns_resolver::QorbResolver; +use internal_dns_types::names::ServiceName; use qorb::backend; use qorb::policy::Policy; use qorb::resolver::{AllBackends, Resolver}; -use qorb::resolvers::dns::{DnsResolver, DnsResolverConfig}; -use qorb::service; use slog::Logger; use std::collections::BTreeMap; -use std::net::SocketAddr; use std::sync::Arc; use tokio::sync::watch; @@ -54,19 +53,6 @@ impl Resolver for SingleHostResolver { } } -fn make_dns_resolver( - bootstrap_dns: Vec, -) -> qorb::resolver::BoxedResolver { - Box::new(DnsResolver::new( - service::Name(internal_dns::ServiceName::Cockroach.srv_name()), - bootstrap_dns, - DnsResolverConfig { - hardcoded_ttl: Some(tokio::time::Duration::MAX), - ..Default::default() - }, - )) -} - fn make_single_host_resolver( config: &DbConfig, ) -> qorb::resolver::BoxedResolver { @@ -95,11 +81,11 @@ impl Pool { /// /// Creating this pool does not necessarily wait for connections to become /// available, as backends may shift over time. - pub fn new(log: &Logger, bootstrap_dns: Vec) -> Self { + pub fn new(log: &Logger, resolver: &QorbResolver) -> Self { // Make sure diesel-dtrace's USDT probes are enabled. usdt::register_probes().expect("Failed to register USDT DTrace probes"); - let resolver = make_dns_resolver(bootstrap_dns); + let resolver = resolver.for_service(ServiceName::Cockroach); let connector = make_postgres_connector(log); let policy = Policy::default(); diff --git a/nexus/db-queries/src/db/queries/oximeter.rs b/nexus/db-queries/src/db/queries/oximeter.rs index 40f7a2b493..ab2d194c3e 100644 --- a/nexus/db-queries/src/db/queries/oximeter.rs +++ b/nexus/db-queries/src/db/queries/oximeter.rs @@ -4,10 +4,165 @@ //! Implementation of queries for Oximeter collectors and producers. +use crate::db::column_walker::AllColumnsOf; use crate::db::raw_query_builder::{QueryBuilder, TypedSqlQuery}; +use diesel::pg::Pg; use diesel::sql_types; +use ipnetwork::IpNetwork; +use nexus_db_model::{OximeterInfo, ProducerKind, ProducerKindEnum, SqlU16}; +use omicron_common::api::internal; use uuid::Uuid; +type AllColumnsOfOximeterInfo = + AllColumnsOf; +type SelectableSql = < + >::SelectExpression as diesel::Expression +>::SqlType; + +/// Upsert a metric producer. +/// +/// If the producer is being inserted for the first time, a random Oximeter will +/// be chosen from among all non-expunged entries in the `oximeter` table. +/// +/// If the producer is being updated, it will keep its existing Oximeter as long +/// as that Oximeter has not been expunged. If its previously-chosen Oximeter +/// has been expunged, its assignment will be changed to a random non-expunged +/// Oximeter. +/// +/// If this query succeeds but returns 0 rows inserted/updated, there are no +/// non-expunged `Oximeter` instances to choose. +/// +/// Returns the oximeter ID assigned to this producer (either the +/// randomly-chosen one, if newly inserted or updated-from-an-expunged, or the +/// previously-chosen, if updated and the existing assignment is still valid). +pub fn upsert_producer( + producer: &internal::nexus::ProducerEndpoint, +) -> TypedSqlQuery> { + let builder = QueryBuilder::new(); + + // Select the existing oximeter ID for this producer, if it exists and is + // not expunged. + let builder = builder + .sql( + r#" + WITH existing_oximeter AS ( + SELECT oximeter.id + FROM metric_producer INNER JOIN oximeter + ON (metric_producer.oximeter_id = oximeter.id) + WHERE + oximeter.time_expunged IS NULL + AND metric_producer.id = "#, + ) + .param() + .bind::(producer.id) + .sql("), "); + + // Choose a random non-expunged Oximeter instance to use if the previous + // clause did not find an existing, non-expunged Oximeter. + let builder = builder.sql( + r#" + random_oximeter AS ( + SELECT id FROM oximeter + WHERE time_expunged IS NULL + ORDER BY random() + LIMIT 1 + ), + "#, + ); + + // Combine the previous two queries. The `LEFT JOIN ... ON true` ensures we + // always get a row from this clause if there is _any_ non-expunged Oximeter + // available. + let builder = builder.sql( + r#" + chosen_oximeter AS ( + SELECT COALESCE(existing_oximeter.id, random_oximeter.id) AS oximeter_id + FROM random_oximeter LEFT JOIN existing_oximeter ON true + ), + "#, + ); + + // Build the INSERT for new producers... + let builder = builder.sql( + r#" + inserted_producer AS ( + INSERT INTO metric_producer ( + id, + time_created, + time_modified, + kind, + ip, + port, + interval, + oximeter_id + ) + "#, + ); + + // ... by querying our chosen oximeter ID and the values from `producer`. + let builder = builder + .sql("SELECT ") + .param() + .bind::(producer.id) + .sql(", now()") // time_created + .sql(", now()") // time_modified + .sql(", ") + .param() + .bind::(producer.kind.into()) + .sql(", ") + .param() + .bind::(producer.address.ip().into()) + .sql(", ") + .param() + .bind::(producer.address.port().into()) + .sql(", ") + .param() + .bind::(producer.interval.as_secs_f32()) + .sql(", oximeter_id FROM chosen_oximeter"); + + // If the producer already exists, update everything except id/time_created. + // This will keep the existing `oximeter_id` if we got a non-NULL value from + // the first clause in our CTE (selecting the existing oximeter id if it's + // not expunged), or reassign to our randomly-chosen one (the second clause + // above) if our current assignment is expunged. + let builder = builder.sql( + r#" + ON CONFLICT (id) + DO UPDATE SET + time_modified = now(), + kind = excluded.kind, + ip = excluded.ip, + port = excluded.port, + interval = excluded.interval, + oximeter_id = excluded.oximeter_id + "#, + ); + + // ... and return this producer's assigned collector ID. + let builder = builder.sql( + r#" + RETURNING oximeter_id + ) + "#, + ); + + // Finally, join the oximeter ID from our inserted or updated producer with + // the `oximeter` table to get all of its information. + let builder = builder + .sql("SELECT ") + .sql(AllColumnsOfOximeterInfo::with_prefix("oximeter")) + .sql( + r#" + FROM oximeter + INNER JOIN inserted_producer + ON (oximeter.id = inserted_producer.oximeter_id) + WHERE oximeter.time_expunged IS NULL + "#, + ); + + builder.query() +} + /// For a given Oximeter instance (which is presumably no longer running), /// reassign any producers assigned to it to a different Oximeter. Each /// assignment is randomly chosen from among the non-expunged Oximeter instances @@ -70,13 +225,32 @@ mod test { use crate::db::raw_query_builder::expectorate_query_contents; use nexus_test_utils::db::test_setup_database; use omicron_test_utils::dev; + use std::time::Duration; use uuid::Uuid; - // This test is a bit of a "change detector", but it's here to help with - // debugging too. If you change this query, it can be useful to see exactly + // These tests are a bit of a "change detector", but it's here to help with + // debugging too. If you change these query, it can be useful to see exactly // how the output SQL has been altered. #[tokio::test] - async fn expectorate_query() { + async fn expectorate_query_upsert_producer() { + let producer = internal::nexus::ProducerEndpoint { + id: Uuid::nil(), + kind: ProducerKind::SledAgent.into(), + address: "[::1]:0".parse().unwrap(), + interval: Duration::from_secs(30), + }; + + let query = upsert_producer(&producer); + + expectorate_query_contents( + &query, + "tests/output/oximeter_upsert_producer.sql", + ) + .await; + } + + #[tokio::test] + async fn expectorate_query_reassign_producers() { let oximeter_id = Uuid::nil(); let query = reassign_producers_query(oximeter_id); @@ -88,10 +262,36 @@ mod test { .await; } - // Explain the SQL query to ensure that it creates a valid SQL string. + // Explain the SQL queries to ensure that they create valid SQL strings. + #[tokio::test] + async fn explainable_upsert_producer() { + let logctx = dev::test_setup_log("explainable_upsert_producer"); + let log = logctx.log.new(o!()); + let mut db = test_setup_database(&log).await; + let cfg = crate::db::Config { url: db.pg_config().clone() }; + let pool = crate::db::Pool::new_single_host(&logctx.log, &cfg); + let conn = pool.claim().await.unwrap(); + + let producer = internal::nexus::ProducerEndpoint { + id: Uuid::nil(), + kind: ProducerKind::SledAgent.into(), + address: "[::1]:0".parse().unwrap(), + interval: Duration::from_secs(30), + }; + + let query = upsert_producer(&producer); + let _ = query + .explain_async(&conn) + .await + .expect("Failed to explain query - is it valid SQL?"); + + db.cleanup().await.unwrap(); + logctx.cleanup_successful(); + } + #[tokio::test] - async fn explainable() { - let logctx = dev::test_setup_log("explainable"); + async fn explainable_reassign_producers() { + let logctx = dev::test_setup_log("explainable_reassign_producers"); let log = logctx.log.new(o!()); let mut db = test_setup_database(&log).await; let cfg = crate::db::Config { url: db.pg_config().clone() }; diff --git a/nexus/db-queries/src/policy_test/resource_builder.rs b/nexus/db-queries/src/policy_test/resource_builder.rs index 3d09b2ab2d..304b970377 100644 --- a/nexus/db-queries/src/policy_test/resource_builder.rs +++ b/nexus/db-queries/src/policy_test/resource_builder.rs @@ -254,6 +254,9 @@ impl_dyn_authorized_resource_for_resource!(authz::IdentityProvider); impl_dyn_authorized_resource_for_resource!(authz::Image); impl_dyn_authorized_resource_for_resource!(authz::Instance); impl_dyn_authorized_resource_for_resource!(authz::InstanceNetworkInterface); +impl_dyn_authorized_resource_for_resource!(authz::InternetGateway); +impl_dyn_authorized_resource_for_resource!(authz::InternetGatewayIpAddress); +impl_dyn_authorized_resource_for_resource!(authz::InternetGatewayIpPool); impl_dyn_authorized_resource_for_resource!(authz::LoopbackAddress); impl_dyn_authorized_resource_for_resource!(authz::Rack); impl_dyn_authorized_resource_for_resource!(authz::PhysicalDisk); diff --git a/nexus/db-queries/src/policy_test/resources.rs b/nexus/db-queries/src/policy_test/resources.rs index 478fa169ff..1a5ac77f53 100644 --- a/nexus/db-queries/src/policy_test/resources.rs +++ b/nexus/db-queries/src/policy_test/resources.rs @@ -318,7 +318,7 @@ async fn make_project( builder.new_resource(vpc1.clone()); // Test a resource nested two levels below Project builder.new_resource(authz::VpcSubnet::new( - vpc1, + vpc1.clone(), Uuid::new_v4(), LookupType::ByName(format!("{}-subnet1", vpc1_name)), )); @@ -342,6 +342,28 @@ async fn make_project( Uuid::new_v4(), LookupType::ByName(floating_ip_name), )); + + let igw_name = format!("{project_name}-igw1"); + let igw = authz::InternetGateway::new( + vpc1.clone(), + Uuid::new_v4(), + LookupType::ByName(igw_name.clone()), + ); + builder.new_resource(igw.clone()); + + let igw_ip_pool_name = format!("{igw_name}-pool1"); + builder.new_resource(authz::InternetGatewayIpPool::new( + igw.clone(), + Uuid::new_v4(), + LookupType::ByName(igw_ip_pool_name), + )); + + let igw_ip_address_name = format!("{igw_name}-address1"); + builder.new_resource(authz::InternetGatewayIpAddress::new( + igw.clone(), + Uuid::new_v4(), + LookupType::ByName(igw_ip_address_name), + )); } /// Returns the set of authz classes exempted from the coverage test @@ -378,6 +400,11 @@ pub fn exempted_authz_classes() -> BTreeSet { // to this list, modify `make_resources()` to test it instead. This // should be pretty straightforward in most cases. Adding a new // class to this list makes it harder to catch security flaws! + // + // NOTE: in order to add a resource to the aforementioned tests, you + // need to call the macro `impl_dyn_authorized_resource_for_resource!` + // for the type you are implementing the test for. See + // resource_builder.rs for examples. authz::IpPool::get_polar_class(), authz::VpcRouter::get_polar_class(), authz::RouterRoute::get_polar_class(), diff --git a/nexus/db-queries/tests/output/authz-roles.out b/nexus/db-queries/tests/output/authz-roles.out index 41a1ded3b4..36403fa700 100644 --- a/nexus/db-queries/tests/output/authz-roles.out +++ b/nexus/db-queries/tests/output/authz-roles.out @@ -404,6 +404,48 @@ resource: FloatingIp "silo1-proj1-fip1" silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: InternetGateway "silo1-proj1-igw1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: InternetGatewayIpPool "silo1-proj1-igw1-pool1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: InternetGatewayIpAddress "silo1-proj1-igw1-address1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-proj1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Project "silo1-proj2" USER Q R LC RP M MP CC D @@ -530,6 +572,48 @@ resource: FloatingIp "silo1-proj2-fip1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: InternetGateway "silo1-proj2-igw1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: InternetGatewayIpPool "silo1-proj2-igw1-pool1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: InternetGatewayIpAddress "silo1-proj2-igw1-address1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-collaborator ✘ ✔ ✔ ✔ ✔ ✔ ✔ ✔ + silo1-viewer ✘ ✔ ✔ ✔ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Silo "silo2" USER Q R LC RP M MP CC D @@ -824,6 +908,48 @@ resource: FloatingIp "silo2-proj1-fip1" silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ unauthenticated ! ! ! ! ! ! ! ! +resource: InternetGateway "silo2-proj1-igw1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: InternetGatewayIpPool "silo2-proj1-igw1-pool1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + +resource: InternetGatewayIpAddress "silo2-proj1-igw1-address1" + + USER Q R LC RP M MP CC D + fleet-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + fleet-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-admin ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-collaborator ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + silo1-proj1-viewer ✘ ✘ ✘ ✘ ✘ ✘ ✘ ✘ + unauthenticated ! ! ! ! ! ! ! ! + resource: Rack id "c037e882-8b6d-c8b5-bef4-97e848eb0a50" USER Q R LC RP M MP CC D diff --git a/nexus/db-queries/tests/output/oximeter_upsert_producer.sql b/nexus/db-queries/tests/output/oximeter_upsert_producer.sql new file mode 100644 index 0000000000..4ef2b4082f --- /dev/null +++ b/nexus/db-queries/tests/output/oximeter_upsert_producer.sql @@ -0,0 +1,52 @@ +WITH + existing_oximeter + AS ( + SELECT + oximeter.id + FROM + metric_producer INNER JOIN oximeter ON metric_producer.oximeter_id = oximeter.id + WHERE + oximeter.time_expunged IS NULL AND metric_producer.id = $1 + ), + random_oximeter + AS (SELECT id FROM oximeter WHERE time_expunged IS NULL ORDER BY random() LIMIT 1), + chosen_oximeter + AS ( + SELECT + COALESCE(existing_oximeter.id, random_oximeter.id) AS oximeter_id + FROM + random_oximeter LEFT JOIN existing_oximeter ON true + ), + inserted_producer + AS ( + INSERT + INTO + metric_producer (id, time_created, time_modified, kind, ip, port, "interval", oximeter_id) + SELECT + $2, now(), now(), $3, $4, $5, $6, oximeter_id + FROM + chosen_oximeter + ON CONFLICT + (id) + DO + UPDATE SET + time_modified = now(), + kind = excluded.kind, + ip = excluded.ip, + port = excluded.port, + "interval" = excluded.interval, + oximeter_id = excluded.oximeter_id + RETURNING + oximeter_id + ) +SELECT + oximeter.id, + oximeter.time_created, + oximeter.time_modified, + oximeter.time_expunged, + oximeter.ip, + oximeter.port +FROM + oximeter INNER JOIN inserted_producer ON oximeter.id = inserted_producer.oximeter_id +WHERE + oximeter.time_expunged IS NULL diff --git a/nexus/examples/config-second.toml b/nexus/examples/config-second.toml index f5d0b240da..a955766554 100644 --- a/nexus/examples/config-second.toml +++ b/nexus/examples/config-second.toml @@ -129,7 +129,7 @@ blueprints.period_secs_collect_crdb_node_ids = 180 sync_service_zone_nat.period_secs = 30 switch_port_settings_manager.period_secs = 30 region_replacement.period_secs = 30 -region_replacement_driver.period_secs = 10 +region_replacement_driver.period_secs = 30 # How frequently to query the status of active instances. instance_watcher.period_secs = 30 # How frequently to schedule new instance update sagas. diff --git a/nexus/examples/config.toml b/nexus/examples/config.toml index c653acb823..ce3dfe5751 100644 --- a/nexus/examples/config.toml +++ b/nexus/examples/config.toml @@ -115,7 +115,7 @@ blueprints.period_secs_collect_crdb_node_ids = 180 sync_service_zone_nat.period_secs = 30 switch_port_settings_manager.period_secs = 30 region_replacement.period_secs = 30 -region_replacement_driver.period_secs = 10 +region_replacement_driver.period_secs = 30 # How frequently to query the status of active instances. instance_watcher.period_secs = 30 # How frequently to schedule new instance update sagas. diff --git a/nexus/external-api/output/nexus_tags.txt b/nexus/external-api/output/nexus_tags.txt index cd6385f589..2d64679ca0 100644 --- a/nexus/external-api/output/nexus_tags.txt +++ b/nexus/external-api/output/nexus_tags.txt @@ -232,6 +232,16 @@ ping GET /v1/ping API operations found with tag "vpcs" OPERATION ID METHOD URL PATH +internet_gateway_create POST /v1/internet-gateways +internet_gateway_delete DELETE /v1/internet-gateways/{gateway} +internet_gateway_ip_address_create POST /v1/internet-gateway-ip-addresses +internet_gateway_ip_address_delete DELETE /v1/internet-gateway-ip-addresses/{address} +internet_gateway_ip_address_list GET /v1/internet-gateway-ip-addresses +internet_gateway_ip_pool_create POST /v1/internet-gateway-ip-pools +internet_gateway_ip_pool_delete DELETE /v1/internet-gateway-ip-pools/{pool} +internet_gateway_ip_pool_list GET /v1/internet-gateway-ip-pools +internet_gateway_list GET /v1/internet-gateways +internet_gateway_view GET /v1/internet-gateways/{gateway} vpc_create POST /v1/vpcs vpc_delete DELETE /v1/vpcs/{vpc} vpc_firewall_rules_update PUT /v1/vpc-firewall-rules diff --git a/nexus/external-api/src/lib.rs b/nexus/external-api/src/lib.rs index d41a23346e..b05366e0c5 100644 --- a/nexus/external-api/src/lib.rs +++ b/nexus/external-api/src/lib.rs @@ -23,7 +23,7 @@ use omicron_common::api::external::{ use openapi_manager_types::ValidationContext; use openapiv3::OpenAPI; -pub const API_VERSION: &str = "20241009.0"; +pub const API_VERSION: &str = "20241204.0"; // API ENDPOINT FUNCTION NAMING CONVENTIONS // @@ -2208,6 +2208,135 @@ pub trait NexusExternalApi { router_params: TypedBody, ) -> Result, HttpError>; + // Internet gateways + + /// List internet gateways + #[endpoint { + method = GET, + path = "/v1/internet-gateways", + tags = ["vpcs"], + }] + async fn internet_gateway_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError>; + + /// Fetch internet gateway + #[endpoint { + method = GET, + path = "/v1/internet-gateways/{gateway}", + tags = ["vpcs"], + }] + async fn internet_gateway_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError>; + + /// Create VPC internet gateway + #[endpoint { + method = POST, + path = "/v1/internet-gateways", + tags = ["vpcs"], + }] + async fn internet_gateway_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError>; + + /// Delete internet gateway + #[endpoint { + method = DELETE, + path = "/v1/internet-gateways/{gateway}", + tags = ["vpcs"], + }] + async fn internet_gateway_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// List IP pools attached to internet gateway + #[endpoint { + method = GET, + path = "/v1/internet-gateway-ip-pools", + tags = ["vpcs"], + }] + async fn internet_gateway_ip_pool_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result< + HttpResponseOk>, + HttpError, + >; + + /// Attach IP pool to internet gateway + #[endpoint { + method = POST, + path = "/v1/internet-gateway-ip-pools", + tags = ["vpcs"], + }] + async fn internet_gateway_ip_pool_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError>; + + /// Detach IP pool from internet gateway + #[endpoint { + method = DELETE, + path = "/v1/internet-gateway-ip-pools/{pool}", + tags = ["vpcs"], + }] + async fn internet_gateway_ip_pool_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + + /// List IP addresses attached to internet gateway + #[endpoint { + method = GET, + path = "/v1/internet-gateway-ip-addresses", + tags = ["vpcs"], + }] + async fn internet_gateway_ip_address_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result< + HttpResponseOk>, + HttpError, + >; + + /// Attach IP address to internet gateway + #[endpoint { + method = POST, + path = "/v1/internet-gateway-ip-addresses", + tags = ["vpcs"], + }] + async fn internet_gateway_ip_address_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError>; + + /// Detach IP address from internet gateway + #[endpoint { + method = DELETE, + path = "/v1/internet-gateway-ip-addresses/{address}", + tags = ["vpcs"], + }] + async fn internet_gateway_ip_address_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result; + // Racks /// List racks @@ -3044,6 +3173,6 @@ pub fn validate_api(spec: &OpenAPI, mut cx: ValidationContext<'_>) { pub type IpPoolRangePaginationParams = PaginationParams; -/// Type used to paginate request to list timeseries schema. +/// Type used to paginate request to list timeseries schema pub type TimeseriesSchemaPaginationParams = PaginationParams; diff --git a/nexus/inventory/Cargo.toml b/nexus/inventory/Cargo.toml index 3057617c67..b5ca91283e 100644 --- a/nexus/inventory/Cargo.toml +++ b/nexus/inventory/Cargo.toml @@ -11,6 +11,9 @@ workspace = true anyhow.workspace = true base64.workspace = true chrono.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true +clickhouse-admin-types.workspace = true futures.workspace = true gateway-client.workspace = true gateway-messages.workspace = true diff --git a/nexus/inventory/src/builder.rs b/nexus/inventory/src/builder.rs index 6e2d8ba28d..00c4b7045d 100644 --- a/nexus/inventory/src/builder.rs +++ b/nexus/inventory/src/builder.rs @@ -11,18 +11,17 @@ use anyhow::anyhow; use chrono::DateTime; use chrono::Utc; +use clickhouse_admin_types::ClickhouseKeeperClusterMembership; use gateway_client::types::SpComponentCaboose; use gateway_client::types::SpState; use gateway_client::types::SpType; use nexus_sled_agent_shared::inventory::Baseboard; use nexus_sled_agent_shared::inventory::Inventory; -use nexus_sled_agent_shared::inventory::OmicronZonesConfig; use nexus_types::inventory::BaseboardId; use nexus_types::inventory::Caboose; use nexus_types::inventory::CabooseFound; use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::Collection; -use nexus_types::inventory::OmicronZonesFound; use nexus_types::inventory::RotPage; use nexus_types::inventory::RotPageFound; use nexus_types::inventory::RotPageWhich; @@ -91,7 +90,9 @@ pub struct CollectionBuilder { rot_pages_found: BTreeMap, RotPageFound>>, sleds: BTreeMap, - omicron_zones: BTreeMap, + clickhouse_keeper_cluster_membership: + BTreeSet, + // We just generate one UUID for each collection. id_rng: TypedUuidRng, } @@ -118,7 +119,7 @@ impl CollectionBuilder { cabooses_found: BTreeMap::new(), rot_pages_found: BTreeMap::new(), sleds: BTreeMap::new(), - omicron_zones: BTreeMap::new(), + clickhouse_keeper_cluster_membership: BTreeSet::new(), id_rng: TypedUuidRng::from_entropy(), } } @@ -127,8 +128,8 @@ impl CollectionBuilder { pub fn build(mut self) -> Collection { // This is not strictly necessary. But for testing, it's helpful for // things to be in sorted order. - for v in self.omicron_zones.values_mut() { - v.zones.zones.sort_by(|a, b| a.id.cmp(&b.id)); + for v in self.sleds.values_mut() { + v.omicron_zones.zones.sort_by(|a, b| a.id.cmp(&b.id)); } Collection { @@ -145,10 +146,8 @@ impl CollectionBuilder { cabooses_found: self.cabooses_found, rot_pages_found: self.rot_pages_found, sled_agents: self.sleds, - omicron_zones: self.omicron_zones, - // Currently unused - // See: https://github.com/oxidecomputer/omicron/issues/6578 - clickhouse_keeper_cluster_membership: BTreeMap::new(), + clickhouse_keeper_cluster_membership: self + .clickhouse_keeper_cluster_membership, } } @@ -512,6 +511,7 @@ impl CollectionBuilder { reservoir_size: inventory.reservoir_size, time_collected, sled_id, + omicron_zones: inventory.omicron_zones, disks: inventory.disks.into_iter().map(|d| d.into()).collect(), zpools: inventory .zpools @@ -536,30 +536,13 @@ impl CollectionBuilder { } } - /// Record information about Omicron zones found on a sled - pub fn found_sled_omicron_zones( + /// Record information about Keeper cluster membership learned from the + /// clickhouse-admin service running in the keeper zones. + pub fn found_clickhouse_keeper_cluster_membership( &mut self, - source: &str, - sled_id: SledUuid, - zones: OmicronZonesConfig, - ) -> Result<(), anyhow::Error> { - if let Some(previous) = self.omicron_zones.get(&sled_id) { - Err(anyhow!( - "sled {sled_id} omicron zones: reported previously: {:?}", - previous - )) - } else { - self.omicron_zones.insert( - sled_id, - OmicronZonesFound { - time_collected: now_db_precision(), - source: source.to_string(), - sled_id, - zones, - }, - ); - Ok(()) - } + membership: ClickhouseKeeperClusterMembership, + ) { + self.clickhouse_keeper_cluster_membership.insert(membership); } } @@ -619,6 +602,7 @@ mod test { assert!(collection.rots.is_empty()); assert!(collection.cabooses_found.is_empty()); assert!(collection.rot_pages_found.is_empty()); + assert!(collection.clickhouse_keeper_cluster_membership.is_empty()); } // Simple test of a single, fairly typical collection that contains just diff --git a/nexus/inventory/src/collector.rs b/nexus/inventory/src/collector.rs index 6fb2959d42..8637765a48 100644 --- a/nexus/inventory/src/collector.rs +++ b/nexus/inventory/src/collector.rs @@ -16,8 +16,8 @@ use nexus_types::inventory::Collection; use nexus_types::inventory::RotPage; use nexus_types::inventory::RotPageWhich; use slog::o; +use slog::Logger; use slog::{debug, error}; -use std::sync::Arc; use std::time::Duration; use strum::IntoEnumIterator; @@ -27,7 +27,8 @@ const SLED_AGENT_TIMEOUT: Duration = Duration::from_secs(60); /// Collect all inventory data from an Oxide system pub struct Collector<'a> { log: slog::Logger, - mgs_clients: Vec>, + mgs_clients: Vec, + keeper_admin_clients: Vec, sled_agent_lister: &'a (dyn SledAgentEnumerator + Send + Sync), in_progress: CollectionBuilder, } @@ -35,13 +36,15 @@ pub struct Collector<'a> { impl<'a> Collector<'a> { pub fn new( creator: &str, - mgs_clients: &[Arc], + mgs_clients: Vec, + keeper_admin_clients: Vec, sled_agent_lister: &'a (dyn SledAgentEnumerator + Send + Sync), log: slog::Logger, ) -> Self { Collector { log, - mgs_clients: mgs_clients.to_vec(), + mgs_clients, + keeper_admin_clients, sled_agent_lister, in_progress: CollectionBuilder::new(creator), } @@ -66,6 +69,7 @@ impl<'a> Collector<'a> { self.collect_all_mgs().await; self.collect_all_sled_agents().await; + self.collect_all_keepers().await; debug!(&self.log, "finished collection"); @@ -74,14 +78,18 @@ impl<'a> Collector<'a> { /// Collect inventory from all MGS instances async fn collect_all_mgs(&mut self) { - let clients = self.mgs_clients.clone(); - for client in &clients { - self.collect_one_mgs(&client).await; + for client in &self.mgs_clients { + Self::collect_one_mgs(client, &self.log, &mut self.in_progress) + .await; } } - async fn collect_one_mgs(&mut self, client: &gateway_client::Client) { - debug!(&self.log, "begin collection from MGS"; + async fn collect_one_mgs( + client: &gateway_client::Client, + log: &Logger, + in_progress: &mut CollectionBuilder, + ) { + debug!(log, "begin collection from MGS"; "mgs_url" => client.baseurl() ); @@ -103,7 +111,7 @@ impl<'a> Collector<'a> { // being able to identify this particular condition. let sps = match ignition_result { Err(error) => { - self.in_progress.found_error(InventoryError::from(error)); + in_progress.found_error(InventoryError::from(error)); return; } @@ -139,14 +147,14 @@ impl<'a> Collector<'a> { }); let sp_state = match result { Err(error) => { - self.in_progress.found_error(InventoryError::from(error)); + in_progress.found_error(InventoryError::from(error)); continue; } Ok(response) => response.into_inner(), }; // Record the state that we found. - let Some(baseboard_id) = self.in_progress.found_sp_state( + let Some(baseboard_id) = in_progress.found_sp_state( client.baseurl(), sp.type_, sp.slot, @@ -162,8 +170,7 @@ impl<'a> Collector<'a> { // get here for the first MGS client. Assuming that one succeeds, // the other(s) will skip this loop. for which in CabooseWhich::iter() { - if self.in_progress.found_caboose_already(&baseboard_id, which) - { + if in_progress.found_caboose_already(&baseboard_id, which) { continue; } @@ -191,20 +198,19 @@ impl<'a> Collector<'a> { }); let caboose = match result { Err(error) => { - self.in_progress - .found_error(InventoryError::from(error)); + in_progress.found_error(InventoryError::from(error)); continue; } Ok(response) => response.into_inner(), }; - if let Err(error) = self.in_progress.found_caboose( + if let Err(error) = in_progress.found_caboose( &baseboard_id, which, client.baseurl(), caboose, ) { error!( - &self.log, + log, "error reporting caboose: {:?} {:?} {:?}: {:#}", baseboard_id, which, @@ -219,8 +225,7 @@ impl<'a> Collector<'a> { // get here for the first MGS client. Assuming that one succeeds, // the other(s) will skip this loop. for which in RotPageWhich::iter() { - if self.in_progress.found_rot_page_already(&baseboard_id, which) - { + if in_progress.found_rot_page_already(&baseboard_id, which) { continue; } @@ -270,20 +275,19 @@ impl<'a> Collector<'a> { let page = match result { Err(error) => { - self.in_progress - .found_error(InventoryError::from(error)); + in_progress.found_error(InventoryError::from(error)); continue; } Ok(data_base64) => RotPage { data_base64 }, }; - if let Err(error) = self.in_progress.found_rot_page( + if let Err(error) = in_progress.found_rot_page( &baseboard_id, which, client.baseurl(), page, ) { error!( - &self.log, + log, "error reporting rot page: {:?} {:?} {:?}: {:#}", baseboard_id, which, @@ -312,11 +316,11 @@ impl<'a> Collector<'a> { .timeout(SLED_AGENT_TIMEOUT) .build() .unwrap(); - let client = Arc::new(sled_agent_client::Client::new_with_client( + let client = sled_agent_client::Client::new_with_client( &url, reqwest_client, log, - )); + ); if let Err(error) = self.collect_one_sled_agent(&client).await { error!( @@ -349,23 +353,41 @@ impl<'a> Collector<'a> { } }; - let sled_id = inventory.sled_id; - self.in_progress.found_sled_inventory(&sled_agent_url, inventory)?; + self.in_progress.found_sled_inventory(&sled_agent_url, inventory) + } + + /// Collect inventory from about keepers from all `ClickhouseAdminKeeper` + /// clients + async fn collect_all_keepers(&mut self) { + for client in &self.keeper_admin_clients { + Self::collect_one_keeper(&client, &self.log, &mut self.in_progress) + .await; + } + } - let maybe_config = - client.omicron_zones_get().await.with_context(|| { - format!("Sled Agent {:?}: omicron zones", &sled_agent_url) - }); - match maybe_config { + /// Collect inventory about one keeper from one `ClickhouseAdminKeeper` + async fn collect_one_keeper( + client: &clickhouse_admin_keeper_client::Client, + log: &slog::Logger, + in_progress: &mut CollectionBuilder, + ) { + debug!(log, "begin collection from clickhouse-admin-keeper"; + "keeper_admin_url" => client.baseurl() + ); + + let res = client.keeper_cluster_membership().await.with_context(|| { + format!("Clickhouse Keeper {:?}: inventory", &client.baseurl()) + }); + + match res { Err(error) => { - self.in_progress.found_error(InventoryError::from(error)); - Ok(()) + in_progress.found_error(InventoryError::from(error)); + } + Ok(membership) => { + in_progress.found_clickhouse_keeper_cluster_membership( + membership.into_inner(), + ); } - Ok(zones) => self.in_progress.found_sled_omicron_zones( - &sled_agent_url, - sled_id, - zones.into_inner(), - ), } } } @@ -388,7 +410,6 @@ mod test { use std::fmt::Write; use std::net::Ipv6Addr; use std::net::SocketAddrV6; - use std::sync::Arc; fn dump_collection(collection: &Collection) -> String { // Construct a stable, human-readable summary of the Collection @@ -484,24 +505,21 @@ mod test { write!(&mut s, " baseboard {:?}\n", sled_info.baseboard_id) .unwrap(); - if let Some(found_zones) = collection.omicron_zones.get(sled_id) { - assert_eq!(*sled_id, found_zones.sled_id); + write!( + &mut s, + " zone generation: {:?}\n", + sled_info.omicron_zones.generation + ) + .unwrap(); + write!(&mut s, " zones found:\n").unwrap(); + for zone in &sled_info.omicron_zones.zones { write!( &mut s, - " zone generation: {:?}\n", - found_zones.zones.generation + " zone {} type {}\n", + zone.id, + zone.zone_type.kind().report_str(), ) .unwrap(); - write!(&mut s, " zones found:\n").unwrap(); - for zone in &found_zones.zones.zones { - write!( - &mut s, - " zone {} type {}\n", - zone.id, - zone.zone_type.kind().report_str(), - ) - .unwrap(); - } } } @@ -594,12 +612,15 @@ mod test { let sled1_url = format!("http://{}/", sled1.http_server.local_addr()); let sled2_url = format!("http://{}/", sled2.http_server.local_addr()); let mgs_url = format!("http://{}/", gwtestctx.client.bind_address); - let mgs_client = - Arc::new(gateway_client::Client::new(&mgs_url, log.clone())); + let mgs_client = gateway_client::Client::new(&mgs_url, log.clone()); let sled_enum = StaticSledAgentEnumerator::new([sled1_url, sled2_url]); + // We don't have any mocks for this, and it's unclear how much value + // there would be in providing them at this juncture. + let keeper_clients = Vec::new(); let collector = Collector::new( "test-suite", - &[mgs_client], + vec![mgs_client], + keeper_clients, &sled_enum, log.clone(), ); @@ -651,13 +672,20 @@ mod test { .into_iter() .map(|g| { let url = format!("http://{}/", g.client.bind_address); - let client = gateway_client::Client::new(&url, log.clone()); - Arc::new(client) + gateway_client::Client::new(&url, log.clone()) }) .collect::>(); let sled_enum = StaticSledAgentEnumerator::new([sled1_url, sled2_url]); - let collector = - Collector::new("test-suite", &mgs_clients, &sled_enum, log.clone()); + // We don't have any mocks for this, and it's unclear how much value + // there would be in providing them at this juncture. + let keeper_clients = Vec::new(); + let collector = Collector::new( + "test-suite", + mgs_clients, + keeper_clients, + &sled_enum, + log.clone(), + ); let collection = collector .collect_all() .await @@ -686,19 +714,25 @@ mod test { let log = &gwtestctx.logctx.log; let real_client = { let url = format!("http://{}/", gwtestctx.client.bind_address); - let client = gateway_client::Client::new(&url, log.clone()); - Arc::new(client) + gateway_client::Client::new(&url, log.clone()) }; let bad_client = { // This IP range is guaranteed by RFC 6666 to discard traffic. let url = "http://[100::1]:12345"; - let client = gateway_client::Client::new(url, log.clone()); - Arc::new(client) + gateway_client::Client::new(url, log.clone()) }; - let mgs_clients = &[bad_client, real_client]; + let mgs_clients = vec![bad_client, real_client]; let sled_enum = StaticSledAgentEnumerator::empty(); - let collector = - Collector::new("test-suite", mgs_clients, &sled_enum, log.clone()); + // We don't have any mocks for this, and it's unclear how much value + // there would be in providing them at this juncture. + let keeper_clients = Vec::new(); + let collector = Collector::new( + "test-suite", + mgs_clients, + keeper_clients, + &sled_enum, + log.clone(), + ); let collection = collector .collect_all() .await @@ -730,13 +764,16 @@ mod test { let sled1_url = format!("http://{}/", sled1.http_server.local_addr()); let sledbogus_url = String::from("http://[100::1]:45678"); let mgs_url = format!("http://{}/", gwtestctx.client.bind_address); - let mgs_client = - Arc::new(gateway_client::Client::new(&mgs_url, log.clone())); + let mgs_client = gateway_client::Client::new(&mgs_url, log.clone()); let sled_enum = StaticSledAgentEnumerator::new([sled1_url, sledbogus_url]); + // We don't have any mocks for this, and it's unclear how much value + // there would be in providing them at this juncture. + let keeper_clients = Vec::new(); let collector = Collector::new( "test-suite", - &[mgs_client], + vec![mgs_client], + keeper_clients, &sled_enum, log.clone(), ); diff --git a/nexus/inventory/src/examples.rs b/nexus/inventory/src/examples.rs index 6aed9a48fb..8279465475 100644 --- a/nexus/inventory/src/examples.rs +++ b/nexus/inventory/src/examples.rs @@ -5,6 +5,8 @@ //! Example collections used for testing use crate::CollectionBuilder; +use clickhouse_admin_types::ClickhouseKeeperClusterMembership; +use clickhouse_admin_types::KeeperId; use gateway_client::types::PowerState; use gateway_client::types::RotSlot; use gateway_client::types::RotState; @@ -23,6 +25,7 @@ use nexus_types::inventory::CabooseWhich; use nexus_types::inventory::RotPage; use nexus_types::inventory::RotPageWhich; use omicron_common::api::external::ByteCount; +use omicron_common::api::external::Generation; use omicron_common::disk::DiskVariant; use omicron_uuid_kinds::SledUuid; use std::sync::Arc; @@ -274,6 +277,24 @@ pub fn representative() -> Representative { // We deliberately provide no RoT pages for sled2. + // Report a representative set of Omicron zones, used in the sled-agent + // constructors below. + // + // We've hand-selected a minimal set of files to cover each type of zone. + // These files were constructed by: + // + // (1) copying the "omicron zones" ledgers from the sleds in a working + // Omicron deployment + // (2) pretty-printing each one with `json --in-place --file FILENAME` + // (3) adjusting the format slightly with + // `jq '{ generation: .omicron_generation, zones: .zones }'` + let sled14_data = include_str!("../example-data/madrid-sled14.json"); + let sled16_data = include_str!("../example-data/madrid-sled16.json"); + let sled17_data = include_str!("../example-data/madrid-sled17.json"); + let sled14: OmicronZonesConfig = serde_json::from_str(sled14_data).unwrap(); + let sled16: OmicronZonesConfig = serde_json::from_str(sled16_data).unwrap(); + let sled17: OmicronZonesConfig = serde_json::from_str(sled17_data).unwrap(); + // Report some sled agents. // // This first one will match "sled1_bb"'s baseboard information. @@ -366,6 +387,7 @@ pub fn representative() -> Representative { revision: 0, }, SledRole::Gimlet, + sled14, disks, zpools, datasets, @@ -393,6 +415,7 @@ pub fn representative() -> Representative { revision: 0, }, SledRole::Scrimlet, + sled16, vec![], vec![], vec![], @@ -415,6 +438,7 @@ pub fn representative() -> Representative { model: String::from("fellofftruck"), }, SledRole::Gimlet, + sled17, vec![], vec![], vec![], @@ -435,6 +459,12 @@ pub fn representative() -> Representative { sled_agent_id_unknown, Baseboard::Unknown, SledRole::Gimlet, + // We only have omicron zones for three sleds so use empty zone + // info here. + OmicronZonesConfig { + generation: Generation::new(), + zones: Vec::new(), + }, vec![], vec![], vec![], @@ -442,35 +472,13 @@ pub fn representative() -> Representative { ) .unwrap(); - // Report a representative set of Omicron zones. - // - // We've hand-selected a minimal set of files to cover each type of zone. - // These files were constructed by: - // - // (1) copying the "omicron zones" ledgers from the sleds in a working - // Omicron deployment - // (2) pretty-printing each one with `json --in-place --file FILENAME` - // (3) adjusting the format slightly with - // `jq '{ generation: .omicron_generation, zones: .zones }'` - let sled14_data = include_str!("../example-data/madrid-sled14.json"); - let sled16_data = include_str!("../example-data/madrid-sled16.json"); - let sled17_data = include_str!("../example-data/madrid-sled17.json"); - let sled14: OmicronZonesConfig = serde_json::from_str(sled14_data).unwrap(); - let sled16: OmicronZonesConfig = serde_json::from_str(sled16_data).unwrap(); - let sled17: OmicronZonesConfig = serde_json::from_str(sled17_data).unwrap(); - - let sled14_id = "7612d745-d978-41c8-8ee0-84564debe1d2".parse().unwrap(); - builder - .found_sled_omicron_zones("fake sled 14 agent", sled14_id, sled14) - .unwrap(); - let sled16_id = "af56cb43-3422-4f76-85bf-3f229db5f39c".parse().unwrap(); - builder - .found_sled_omicron_zones("fake sled 15 agent", sled16_id, sled16) - .unwrap(); - let sled17_id = "6eb2a0d9-285d-4e03-afa1-090e4656314b".parse().unwrap(); - builder - .found_sled_omicron_zones("fake sled 15 agent", sled17_id, sled17) - .unwrap(); + builder.found_clickhouse_keeper_cluster_membership( + ClickhouseKeeperClusterMembership { + queried_keeper: KeeperId(1), + leader_committed_log_index: 1000, + raft_config: [KeeperId(1)].into_iter().collect(), + }, + ); Representative { builder, @@ -548,6 +556,7 @@ pub fn sled_agent( sled_id: SledUuid, baseboard: Baseboard, sled_role: SledRole, + omicron_zones: OmicronZonesConfig, disks: Vec, zpools: Vec, datasets: Vec, @@ -560,6 +569,7 @@ pub fn sled_agent( sled_id, usable_hardware_threads: 10, usable_physical_ram: ByteCount::from(1024 * 1024), + omicron_zones, disks, zpools, datasets, diff --git a/nexus/metrics-producer-gc/src/lib.rs b/nexus/metrics-producer-gc/src/lib.rs index 4ed8f1bbb5..407af2fdbd 100644 --- a/nexus/metrics-producer-gc/src/lib.rs +++ b/nexus/metrics-producer-gc/src/lib.rs @@ -239,22 +239,19 @@ mod tests { assert!(pruned.failures.is_empty()); // Insert a producer. - let producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_info.id, - ); + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; datastore - .producer_endpoint_create(&opctx, &producer) + .producer_endpoint_upsert_and_assign(&opctx, &producer) .await .expect("failed to insert producer"); let producer_time_modified = - read_time_modified(&datastore, producer.id()).await; + read_time_modified(&datastore, producer.id).await; // GC'ing expired producers with an expiration time older than our // producer's `time_modified` should not prune anything. @@ -278,7 +275,7 @@ mod tests { .await .expect("failed to prune expired producers"); let expected_success = - [producer.id()].into_iter().collect::>(); + [producer.id].into_iter().collect::>(); assert_eq!(pruned.successes, expected_success); assert!(pruned.failures.is_empty()); @@ -321,22 +318,19 @@ mod tests { .expect("failed to insert collector"); // Insert a producer. - let producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_info.id, - ); + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; datastore - .producer_endpoint_create(&opctx, &producer) + .producer_endpoint_upsert_and_assign(&opctx, &producer) .await .expect("failed to insert producer"); let producer_time_modified = - read_time_modified(&datastore, producer.id()).await; + read_time_modified(&datastore, producer.id).await; // GC'ing expired producers with an expiration time _newer_ than our // producer's `time_modified` should prune our one producer and notify @@ -344,7 +338,7 @@ mod tests { collector.expect( Expectation::matching(request::method_path( "DELETE", - format!("/producers/{}", producer.id()), + format!("/producers/{}", producer.id), )) .respond_with(status_code(204)), ); @@ -357,7 +351,7 @@ mod tests { .await .expect("failed to prune expired producers"); let expected_success = - [producer.id()].into_iter().collect::>(); + [producer.id].into_iter().collect::>(); assert_eq!(pruned.successes, expected_success); assert!(pruned.failures.is_empty()); diff --git a/nexus/reconfigurator/execution/Cargo.toml b/nexus/reconfigurator/execution/Cargo.toml index bb3c7ad2b9..21d861ef51 100644 --- a/nexus/reconfigurator/execution/Cargo.toml +++ b/nexus/reconfigurator/execution/Cargo.toml @@ -11,11 +11,15 @@ omicron-rpaths.workspace = true [dependencies] anyhow.workspace = true +camino.workspace = true +clickhouse-admin-keeper-client.workspace = true +clickhouse-admin-server-client.workspace = true +clickhouse-admin-types.workspace = true cockroach-admin-client.workspace = true -dns-service-client.workspace = true chrono.workspace = true futures.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true newtype-uuid.workspace = true nexus-config.workspace = true nexus-db-model.workspace = true diff --git a/nexus/reconfigurator/execution/src/clickhouse.rs b/nexus/reconfigurator/execution/src/clickhouse.rs new file mode 100644 index 0000000000..e623952ed9 --- /dev/null +++ b/nexus/reconfigurator/execution/src/clickhouse.rs @@ -0,0 +1,471 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Deployment of Clickhouse keeper and server nodes via clickhouse-admin running in +//! deployed clickhouse zones. + +use anyhow::anyhow; +use camino::Utf8PathBuf; +use clickhouse_admin_keeper_client::Client as ClickhouseKeeperClient; +use clickhouse_admin_server_client::Client as ClickhouseServerClient; +use clickhouse_admin_types::ClickhouseHost; +use clickhouse_admin_types::KeeperConfigurableSettings; +use clickhouse_admin_types::KeeperSettings; +use clickhouse_admin_types::RaftServerSettings; +use clickhouse_admin_types::ServerConfigurableSettings; +use clickhouse_admin_types::ServerSettings; +use futures::future::Either; +use futures::stream::FuturesUnordered; +use futures::stream::StreamExt; +use nexus_db_queries::context::OpContext; +use nexus_types::deployment::BlueprintZonesConfig; +use nexus_types::deployment::ClickhouseClusterConfig; +use omicron_common::address::CLICKHOUSE_ADMIN_PORT; +use omicron_uuid_kinds::OmicronZoneUuid; +use omicron_uuid_kinds::SledUuid; +use slog::error; +use slog::info; +use slog::warn; +use std::collections::BTreeMap; +use std::net::Ipv6Addr; +use std::net::SocketAddr; +use std::net::SocketAddrV6; +use std::str::FromStr; + +const CLICKHOUSE_SERVER_CONFIG_DIR: &str = + "/opt/oxide/clickhouse_server/config.d"; +const CLICKHOUSE_KEEPER_CONFIG_DIR: &str = "/opt/oxide/clickhouse_keeper"; +const CLICKHOUSE_DATA_DIR: &str = "/data"; + +pub(crate) async fn deploy_nodes( + opctx: &OpContext, + zones: &BTreeMap, + clickhouse_cluster_config: &ClickhouseClusterConfig, +) -> Result<(), Vec> { + let keeper_configs = match keeper_configs(zones, clickhouse_cluster_config) + { + Ok(keeper_configs) => keeper_configs, + Err(e) => { + // We can't proceed if we fail to generate configs. + // Let's be noisy about it. + error!( + opctx.log, + "failed to generate clickhouse keeper configs: {e}" + ); + return Err(vec![e]); + } + }; + + let keeper_hosts: Vec<_> = keeper_configs + .iter() + .map(|s| ClickhouseHost::Ipv6(s.settings.listen_addr)) + .collect(); + + let server_configs = + match server_configs(zones, clickhouse_cluster_config, keeper_hosts) { + Ok(server_configs) => server_configs, + Err(e) => { + // We can't proceed if we fail to generate configs. + // Let's be noisy about it. + error!( + opctx.log, + "Failed to generate clickhouse server configs: {e}" + ); + return Err(vec![e]); + } + }; + + let mut errors = vec![]; + let log = opctx.log.clone(); + + // Inform each clickhouse-admin server in a keeper zone or server zone about + // its node's configuration + let mut futs = FuturesUnordered::new(); + for config in keeper_configs { + let admin_addr = SocketAddr::V6(SocketAddrV6::new( + config.settings.listen_addr, + CLICKHOUSE_ADMIN_PORT, + 0, + 0, + )); + let admin_url = format!("http://{admin_addr}"); + let log = log.new(slog::o!("admin_url" => admin_url.clone())); + futs.push(Either::Left(async move { + let client = ClickhouseKeeperClient::new(&admin_url, log.clone()); + client.generate_config(&config).await.map(|_| ()).map_err(|e| { + anyhow!( + concat!( + "failed to send config for clickhouse keeper ", + "with id {} to clickhouse-admin-keeper; admin_url = {}", + "error = {}" + ), + config.settings.id, + admin_url, + e + ) + }) + })); + } + for config in server_configs { + let admin_addr = SocketAddr::V6(SocketAddrV6::new( + config.settings.listen_addr, + CLICKHOUSE_ADMIN_PORT, + 0, + 0, + )); + let admin_url = format!("http://{admin_addr}"); + let log = opctx.log.new(slog::o!("admin_url" => admin_url.clone())); + futs.push(Either::Right(async move { + let client = ClickhouseServerClient::new(&admin_url, log.clone()); + client.generate_config(&config).await.map(|_| ()).map_err(|e| { + anyhow!( + concat!( + "failed to send config for clickhouse server ", + "with id {} to clickhouse-admin-server; admin_url = {}", + "error = {}" + ), + config.settings.id, + admin_url, + e + ) + }) + })); + } + + while let Some(res) = futs.next().await { + if let Err(e) = res { + warn!(log, "{e}"); + errors.push(e); + } + } + + if !errors.is_empty() { + return Err(errors); + } + + info!( + opctx.log, + "Successfully deployed all clickhouse server and keeper configs" + ); + + Ok(()) +} + +fn server_configs( + zones: &BTreeMap, + clickhouse_cluster_config: &ClickhouseClusterConfig, + keepers: Vec, +) -> Result, anyhow::Error> { + let server_ips: BTreeMap = zones + .values() + .flat_map(|zones_config| { + zones_config + .zones + .iter() + .filter(|zone_config| { + clickhouse_cluster_config + .servers + .contains_key(&zone_config.id) + }) + .map(|zone_config| { + (zone_config.id, zone_config.underlay_address) + }) + }) + .collect(); + + let mut remote_servers = + Vec::with_capacity(clickhouse_cluster_config.servers.len()); + + for (zone_id, server_id) in &clickhouse_cluster_config.servers { + remote_servers.push(ClickhouseHost::Ipv6( + *server_ips.get(zone_id).ok_or_else(|| { + anyhow!( + "Failed to retrieve zone {} for server id {}", + zone_id, + server_id + ) + })?, + )); + } + + let mut server_configs = + Vec::with_capacity(clickhouse_cluster_config.servers.len()); + + for (zone_id, server_id) in &clickhouse_cluster_config.servers { + server_configs.push(ServerConfigurableSettings { + generation: clickhouse_cluster_config.generation, + settings: ServerSettings { + config_dir: Utf8PathBuf::from_str(CLICKHOUSE_SERVER_CONFIG_DIR) + .unwrap(), + id: *server_id, + datastore_path: Utf8PathBuf::from_str(CLICKHOUSE_DATA_DIR) + .unwrap(), + // SAFETY: We already successfully performed the same lookup to compute + // `remote_servers` above. + listen_addr: *server_ips.get(zone_id).unwrap(), + keepers: keepers.clone(), + remote_servers: remote_servers.clone(), + }, + }); + } + + Ok(server_configs) +} + +fn keeper_configs( + zones: &BTreeMap, + clickhouse_cluster_config: &ClickhouseClusterConfig, +) -> Result, anyhow::Error> { + let keeper_ips: BTreeMap = zones + .values() + .flat_map(|zones_config| { + zones_config + .zones + .iter() + .filter(|zone_config| { + clickhouse_cluster_config + .keepers + .contains_key(&zone_config.id) + }) + .map(|zone_config| { + (zone_config.id, zone_config.underlay_address) + }) + }) + .collect(); + + let mut raft_servers = + Vec::with_capacity(clickhouse_cluster_config.keepers.len()); + + for (zone_id, keeper_id) in &clickhouse_cluster_config.keepers { + raft_servers.push(RaftServerSettings { + id: *keeper_id, + host: ClickhouseHost::Ipv6(*keeper_ips.get(zone_id).ok_or_else( + || { + anyhow!( + "Failed to retrieve zone {} for keeper id {}", + zone_id, + keeper_id + ) + }, + )?), + }); + } + + let mut keeper_configs = + Vec::with_capacity(clickhouse_cluster_config.keepers.len()); + + for (zone_id, keeper_id) in &clickhouse_cluster_config.keepers { + keeper_configs.push(KeeperConfigurableSettings { + generation: clickhouse_cluster_config.generation, + settings: KeeperSettings { + config_dir: Utf8PathBuf::from_str(CLICKHOUSE_KEEPER_CONFIG_DIR) + .unwrap(), + id: *keeper_id, + raft_servers: raft_servers.clone(), + datastore_path: Utf8PathBuf::from_str(CLICKHOUSE_DATA_DIR) + .unwrap(), + // SAFETY: We already successfully performed the same lookup to compute + // `raft_servers` above. + listen_addr: *keeper_ips.get(zone_id).unwrap(), + }, + }); + } + + Ok(keeper_configs) +} + +#[cfg(test)] +mod test { + use super::*; + use clickhouse_admin_types::ClickhouseHost; + use clickhouse_admin_types::KeeperId; + use clickhouse_admin_types::ServerId; + use nexus_sled_agent_shared::inventory::OmicronZoneDataset; + use nexus_types::deployment::blueprint_zone_type; + use nexus_types::deployment::BlueprintZoneConfig; + use nexus_types::deployment::BlueprintZoneDisposition; + use nexus_types::deployment::BlueprintZoneType; + use nexus_types::inventory::ZpoolName; + use omicron_common::api::external::Generation; + use omicron_uuid_kinds::ZpoolUuid; + use std::collections::BTreeSet; + + fn test_data( + ) -> (BTreeMap, ClickhouseClusterConfig) + { + let num_keepers = 3u64; + let num_servers = 2u64; + + let mut zones = BTreeMap::new(); + let mut config = ClickhouseClusterConfig::new("test".to_string()); + + for keeper_id in 1..=num_keepers { + let sled_id = SledUuid::new_v4(); + let zone_id = OmicronZoneUuid::new_v4(); + let zone_config = BlueprintZonesConfig { + generation: Generation::new(), + zones: vec![BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::InService, + id: zone_id, + underlay_address: Ipv6Addr::new( + 0, + 0, + 0, + 0, + 0, + 0, + 0, + keeper_id as u16, + ), + filesystem_pool: None, + zone_type: BlueprintZoneType::ClickhouseKeeper( + blueprint_zone_type::ClickhouseKeeper { + address: SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + dataset: OmicronZoneDataset { + pool_name: ZpoolName::new_external( + ZpoolUuid::new_v4(), + ), + }, + }, + ), + }], + }; + zones.insert(sled_id, zone_config); + config.keepers.insert(zone_id, keeper_id.into()); + } + + for server_id in 1..=num_servers { + let sled_id = SledUuid::new_v4(); + let zone_id = OmicronZoneUuid::new_v4(); + let zone_config = BlueprintZonesConfig { + generation: Generation::new(), + zones: vec![BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::InService, + id: zone_id, + underlay_address: Ipv6Addr::new( + 0, + 0, + 0, + 0, + 0, + 0, + 0, + server_id as u16 + 10, + ), + filesystem_pool: None, + zone_type: BlueprintZoneType::ClickhouseServer( + blueprint_zone_type::ClickhouseServer { + address: SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ), + dataset: OmicronZoneDataset { + pool_name: ZpoolName::new_external( + ZpoolUuid::new_v4(), + ), + }, + }, + ), + }], + }; + zones.insert(sled_id, zone_config); + config.servers.insert(zone_id, server_id.into()); + } + + (zones, config) + } + + #[test] + fn test_generate_config_settings() { + let (zones, clickhouse_cluster_config) = test_data(); + + // Generate our keeper settings to send to keepers + let keeper_settings = + keeper_configs(&zones, &clickhouse_cluster_config) + .expect("generated keeper settings"); + + // Are the keeper settings what we expect + assert_eq!(keeper_settings.len(), 3); + let expected_keeper_ids: BTreeSet<_> = + [1u64, 2, 3].into_iter().map(KeeperId::from).collect(); + let mut keeper_ids = BTreeSet::new(); + let mut keeper_ips_last_octet_as_keeper_id = BTreeSet::new(); + for k in &keeper_settings { + assert_eq!(k.settings.raft_servers.len(), 3); + for rs in &k.settings.raft_servers { + keeper_ids.insert(rs.id); + let ClickhouseHost::Ipv6(ip) = rs.host else { + panic!("bad host"); + }; + keeper_ips_last_octet_as_keeper_id + .insert(KeeperId(u64::from(*ip.octets().last().unwrap()))); + } + } + assert_eq!(keeper_ids, expected_keeper_ids); + assert_eq!(keeper_ids, keeper_ips_last_octet_as_keeper_id); + + let keeper_hosts: Vec<_> = keeper_settings + .iter() + .map(|s| ClickhouseHost::Ipv6(s.settings.listen_addr)) + .collect(); + + // Generate our server settings to send to clickhouse servers + let server_settings = + server_configs(&zones, &clickhouse_cluster_config, keeper_hosts) + .expect("generated server settings"); + + // Are our server settings what we expect + assert_eq!(server_settings.len(), 2); + let expected_server_ids: BTreeSet<_> = + [1u64, 2].into_iter().map(ServerId::from).collect(); + let mut server_ids = BTreeSet::new(); + let mut server_ips_last_octet = BTreeSet::new(); + let expected_server_ips_last_octet: BTreeSet = + [11u8, 12].into_iter().collect(); + for s in server_settings { + assert_eq!(s.settings.keepers.len(), 3); + assert_eq!(s.settings.remote_servers.len(), 2); + server_ids.insert(s.settings.id); + + server_ips_last_octet + .insert(*s.settings.listen_addr.octets().last().unwrap()); + + // Are all our keeper ips correct? + let mut keeper_ips_last_octet_as_keeper_id = BTreeSet::new(); + for host in &s.settings.keepers { + let ClickhouseHost::Ipv6(ip) = host else { + panic!("bad host"); + }; + keeper_ips_last_octet_as_keeper_id + .insert(KeeperId(u64::from(*ip.octets().last().unwrap()))); + } + assert_eq!(keeper_ips_last_octet_as_keeper_id, expected_keeper_ids); + + // Are all our remote server ips correct? + let mut remote_server_last_octets = BTreeSet::new(); + for host in &s.settings.remote_servers { + let ClickhouseHost::Ipv6(ip) = host else { + panic!("bad host"); + }; + remote_server_last_octets.insert(*ip.octets().last().unwrap()); + } + assert_eq!( + remote_server_last_octets, + expected_server_ips_last_octet + ); + } + // Are all our server ids correct + assert_eq!(server_ids, expected_server_ids); + + // Are all our server listen ips correct? + assert_eq!(server_ips_last_octet, expected_server_ips_last_octet); + } +} diff --git a/nexus/reconfigurator/execution/src/cockroachdb.rs b/nexus/reconfigurator/execution/src/cockroachdb.rs index 33b8176df6..9656428c2d 100644 --- a/nexus/reconfigurator/execution/src/cockroachdb.rs +++ b/nexus/reconfigurator/execution/src/cockroachdb.rs @@ -33,7 +33,7 @@ pub(crate) async fn ensure_settings( #[cfg(test)] mod test { use super::*; - use crate::overridables::Overridables; + use crate::test_utils::overridables_for_test; use crate::test_utils::realize_blueprint_and_expect; use nexus_db_queries::authn; use nexus_db_queries::authz; @@ -98,7 +98,7 @@ mod test { ) .await; // Execute the initial blueprint. - let overrides = Overridables::for_test(cptestctx); + let overrides = overridables_for_test(cptestctx); _ = realize_blueprint_and_expect( &opctx, datastore, resolver, &blueprint, &overrides, ) diff --git a/nexus/reconfigurator/execution/src/datasets.rs b/nexus/reconfigurator/execution/src/datasets.rs index 2f84378a13..3b5cfeb564 100644 --- a/nexus/reconfigurator/execution/src/datasets.rs +++ b/nexus/reconfigurator/execution/src/datasets.rs @@ -118,7 +118,7 @@ pub(crate) async fn ensure_dataset_records_exist( mod tests { use super::*; use nexus_db_model::Zpool; - use nexus_reconfigurator_planning::example::example; + use nexus_reconfigurator_planning::example::ExampleSystemBuilder; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; use nexus_test_utils_macros::nexus_test; use nexus_types::deployment::blueprint_zone_type; @@ -149,7 +149,9 @@ mod tests { let opctx = &opctx; // Use the standard example system. - let (collection, _, blueprint) = example(&opctx.log, TEST_NAME, 5); + let (example, blueprint) = + ExampleSystemBuilder::new(&opctx.log, TEST_NAME).nsleds(5).build(); + let collection = example.collection; // Record the sleds and zpools. crate::tests::insert_sled_records(datastore, &blueprint).await; @@ -221,7 +223,7 @@ mod tests { // Create another zpool on one of the sleds, so we can add new // zones that use it. let new_zpool_id = ZpoolUuid::new_v4(); - for &sled_id in collection.omicron_zones.keys().take(1) { + for &sled_id in collection.sled_agents.keys().take(1) { let zpool = Zpool::new( new_zpool_id.into_untyped_uuid(), sled_id.into_untyped_uuid(), diff --git a/nexus/reconfigurator/execution/src/dns.rs b/nexus/reconfigurator/execution/src/dns.rs index 8cc0ed96cc..d6f4cdb5de 100644 --- a/nexus/reconfigurator/execution/src/dns.rs +++ b/nexus/reconfigurator/execution/src/dns.rs @@ -4,35 +4,27 @@ //! Propagates DNS changes in a given blueprint -use crate::overridables::Overridables; use crate::Sled; -use dns_service_client::DnsDiff; -use internal_dns::DnsConfigBuilder; -use internal_dns::ServiceName; +use internal_dns_types::diff::DnsDiff; use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::Discoverability; use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder; -use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_db_queries::db::DataStore; -use nexus_types::deployment::blueprint_zone_type; +use nexus_types::deployment::execution::blueprint_external_dns_config; +use nexus_types::deployment::execution::blueprint_internal_dns_config; +use nexus_types::deployment::execution::Overridables; use nexus_types::deployment::Blueprint; -use nexus_types::deployment::BlueprintZoneFilter; -use nexus_types::deployment::BlueprintZoneType; use nexus_types::identity::Resource; use nexus_types::internal_api::params::DnsConfigParams; use nexus_types::internal_api::params::DnsConfigZone; -use nexus_types::internal_api::params::DnsRecord; use omicron_common::api::external::Error; use omicron_common::api::external::Generation; use omicron_common::api::external::InternalContext; -use omicron_common::api::external::Name; use omicron_common::bail_unless; use omicron_uuid_kinds::SledUuid; use slog::{debug, info, o}; use std::collections::BTreeMap; -use std::collections::HashMap; -use std::net::IpAddr; pub(crate) async fn deploy_dns( opctx: &OpContext, @@ -60,7 +52,10 @@ pub(crate) async fn deploy_dns( // Next, construct the DNS config represented by the blueprint. let internal_dns_zone_blueprint = - blueprint_internal_dns_config(blueprint, sleds_by_id, overrides)?; + blueprint_internal_dns_config(blueprint, sleds_by_id, overrides) + .map_err(|e| Error::InternalError { + internal_message: e.to_string(), + })?; let silos = datastore .silo_list_all_batched(opctx, Discoverability::All) .await @@ -245,156 +240,6 @@ pub(crate) async fn deploy_dns_one( datastore.dns_update_from_version(opctx, update, generation).await } -/// Returns the expected contents of internal DNS based on the given blueprint -pub fn blueprint_internal_dns_config( - blueprint: &Blueprint, - sleds_by_id: &BTreeMap, - overrides: &Overridables, -) -> Result { - // The DNS names configured here should match what RSS configures for the - // same zones. It's tricky to have RSS share the same code because it uses - // Sled Agent's _internal_ `OmicronZoneConfig` (and friends), whereas we're - // using `sled-agent-client`'s version of that type. However, the - // DnsConfigBuilder's interface is high-level enough that it handles most of - // the details. - let mut dns_builder = DnsConfigBuilder::new(); - - 'all_zones: for (_, zone) in - blueprint.all_omicron_zones(BlueprintZoneFilter::ShouldBeInInternalDns) - { - let (service_name, port) = match &zone.zone_type { - BlueprintZoneType::BoundaryNtp( - blueprint_zone_type::BoundaryNtp { address, .. }, - ) => (ServiceName::BoundaryNtp, address.port()), - BlueprintZoneType::InternalNtp( - blueprint_zone_type::InternalNtp { address, .. }, - ) => (ServiceName::InternalNtp, address.port()), - BlueprintZoneType::Clickhouse( - blueprint_zone_type::Clickhouse { address, .. }, - ) - | BlueprintZoneType::ClickhouseServer( - blueprint_zone_type::ClickhouseServer { address, .. }, - ) => { - // Add the HTTP and native TCP interfaces for ClickHouse data - // replicas. This adds the zone itself, so we need to continue - // back up to the loop over all the Omicron zones, rather than - // falling through to call `host_zone_with_one_backend()`. - let http_service = if matches!( - &zone.zone_type, - BlueprintZoneType::Clickhouse(_) - ) { - ServiceName::Clickhouse - } else { - ServiceName::ClickhouseServer - }; - dns_builder - .host_zone_clickhouse( - zone.id, - zone.underlay_address, - http_service, - address.port(), - ) - .map_err(|e| Error::InternalError { - internal_message: e.to_string(), - })?; - continue 'all_zones; - } - BlueprintZoneType::ClickhouseKeeper( - blueprint_zone_type::ClickhouseKeeper { address, .. }, - ) => (ServiceName::ClickhouseKeeper, address.port()), - BlueprintZoneType::CockroachDb( - blueprint_zone_type::CockroachDb { address, .. }, - ) => (ServiceName::Cockroach, address.port()), - BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { - internal_address, - .. - }) => (ServiceName::Nexus, internal_address.port()), - BlueprintZoneType::Crucible(blueprint_zone_type::Crucible { - address, - .. - }) => (ServiceName::Crucible(zone.id), address.port()), - BlueprintZoneType::CruciblePantry( - blueprint_zone_type::CruciblePantry { address }, - ) => (ServiceName::CruciblePantry, address.port()), - BlueprintZoneType::Oximeter(blueprint_zone_type::Oximeter { - address, - }) => (ServiceName::Oximeter, address.port()), - BlueprintZoneType::ExternalDns( - blueprint_zone_type::ExternalDns { http_address, .. }, - ) => (ServiceName::ExternalDns, http_address.port()), - BlueprintZoneType::InternalDns( - blueprint_zone_type::InternalDns { http_address, .. }, - ) => (ServiceName::InternalDns, http_address.port()), - }; - dns_builder - .host_zone_with_one_backend( - zone.id, - zone.underlay_address, - service_name, - port, - ) - .map_err(|e| Error::InternalError { - internal_message: e.to_string(), - })?; - } - - let scrimlets = sleds_by_id.values().filter(|sled| sled.is_scrimlet); - for scrimlet in scrimlets { - let sled_subnet = scrimlet.subnet(); - let switch_zone_ip = overrides.switch_zone_ip(scrimlet.id, sled_subnet); - dns_builder - .host_zone_switch( - scrimlet.id, - switch_zone_ip, - overrides.dendrite_port(scrimlet.id), - overrides.mgs_port(scrimlet.id), - overrides.mgd_port(scrimlet.id), - ) - .map_err(|e| Error::InternalError { - internal_message: e.to_string(), - })?; - } - - Ok(dns_builder.build_zone()) -} - -pub fn blueprint_external_dns_config( - blueprint: &Blueprint, - silos: &[Name], - external_dns_zone_name: String, -) -> DnsConfigZone { - let nexus_external_ips = blueprint_nexus_external_ips(blueprint); - - let dns_records: Vec = nexus_external_ips - .into_iter() - .map(|addr| match addr { - IpAddr::V4(addr) => DnsRecord::A(addr), - IpAddr::V6(addr) => DnsRecord::Aaaa(addr), - }) - .collect(); - - let records = silos - .into_iter() - // We do not generate a DNS name for the "default" Silo. - // - // We use the name here rather than the id. It shouldn't really matter - // since every system will have this silo and so no other Silo could - // have this name. But callers (particularly the test suite and - // reconfigurator-cli) specify silos by name, not id, so if we used the - // id here then they'd have to apply this filter themselves (and this - // abstraction, such as it is, would be leakier). - .filter_map(|silo_name| { - (silo_name != DEFAULT_SILO.name()) - .then(|| (silo_dns_name(&silo_name), dns_records.clone())) - }) - .collect::>>(); - - DnsConfigZone { - zone_name: external_dns_zone_name, - records: records.clone(), - } -} - fn dns_compute_update( log: &slog::Logger, dns_group: DnsGroup, @@ -453,44 +298,18 @@ fn dns_compute_update( Ok(Some(update)) } -/// Returns the (relative) DNS name for this Silo's API and console endpoints -/// _within_ the external DNS zone (i.e., without that zone's suffix) -/// -/// This specific naming scheme is determined under RFD 357. -pub fn silo_dns_name(name: &omicron_common::api::external::Name) -> String { - // RFD 4 constrains resource names (including Silo names) to DNS-safe - // strings, which is why it's safe to directly put the name of the - // resource into the DNS name rather than doing any kind of escaping. - format!("{}.sys", name) -} - -/// Return the Nexus external addresses according to the given blueprint -pub fn blueprint_nexus_external_ips(blueprint: &Blueprint) -> Vec { - blueprint - .all_omicron_zones(BlueprintZoneFilter::ShouldBeExternallyReachable) - .filter_map(|(_, z)| match z.zone_type { - BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { - external_ip, - .. - }) => Some(external_ip.ip), - _ => None, - }) - .collect() -} - #[cfg(test)] mod test { use super::*; - use crate::overridables::Overridables; + use crate::test_utils::overridables_for_test; use crate::test_utils::realize_blueprint_and_expect; use crate::Sled; - use dns_service_client::DnsDiff; - use internal_dns::config::Host; - use internal_dns::config::Zone; - use internal_dns::names::BOUNDARY_NTP_DNS_NAME; - use internal_dns::resolver::Resolver; - use internal_dns::ServiceName; - use internal_dns::DNS_ZONE; + use internal_dns_resolver::Resolver; + use internal_dns_types::config::Host; + use internal_dns_types::config::Zone; + use internal_dns_types::names::ServiceName; + use internal_dns_types::names::BOUNDARY_NTP_DNS_NAME; + use internal_dns_types::names::DNS_ZONE; use nexus_db_model::DnsGroup; use nexus_db_model::Silo; use nexus_db_queries::authn; @@ -501,18 +320,22 @@ mod test { use nexus_inventory::CollectionBuilder; use nexus_reconfigurator_planning::blueprint_builder::BlueprintBuilder; use nexus_reconfigurator_planning::blueprint_builder::EnsureMultiple; - use nexus_reconfigurator_planning::example::example; + use nexus_reconfigurator_planning::example::ExampleSystemBuilder; use nexus_reconfigurator_preparation::PlanningInputFromDb; use nexus_sled_agent_shared::inventory::OmicronZoneConfig; use nexus_sled_agent_shared::inventory::OmicronZoneType; + use nexus_sled_agent_shared::inventory::SledRole; use nexus_sled_agent_shared::inventory::ZoneKind; use nexus_test_utils::resource_helpers::create_silo; use nexus_test_utils::resource_helpers::DiskTestBuilder; use nexus_test_utils_macros::nexus_test; + use nexus_types::deployment::blueprint_zone_type; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintTarget; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; + use nexus_types::deployment::BlueprintZoneFilter; + use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::BlueprintZonesConfig; use nexus_types::deployment::CockroachDbClusterVersion; use nexus_types::deployment::CockroachDbPreserveDowngrade; @@ -529,6 +352,7 @@ mod test { use nexus_types::internal_api::params::DnsConfigZone; use nexus_types::internal_api::params::DnsRecord; use nexus_types::internal_api::params::Srv; + use nexus_types::silo::silo_dns_name; use omicron_common::address::get_sled_address; use omicron_common::address::get_switch_zone_address; use omicron_common::address::IpRange; @@ -539,8 +363,10 @@ mod test { use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::policy::BOUNDARY_NTP_REDUNDANCY; use omicron_common::policy::COCKROACHDB_REDUNDANCY; + use omicron_common::policy::CRUCIBLE_PANTRY_REDUNDANCY; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_common::policy::NEXUS_REDUNDANCY; + use omicron_common::policy::OXIMETER_REDUNDANCY; use omicron_common::zpool_name::ZpoolName; use omicron_test_utils::dev::test_setup_log; use omicron_uuid_kinds::ExternalIpUuid; @@ -809,13 +635,12 @@ mod test { // Also assume any sled in the collection is active. let mut sled_state = BTreeMap::new(); - for (sled_id, zones_config) in collection.omicron_zones { + for (sled_id, sa) in collection.sled_agents { blueprint_zones.insert( sled_id, BlueprintZonesConfig { - generation: zones_config.zones.generation, - zones: zones_config - .zones + generation: sa.omicron_zones.generation, + zones: sa.omicron_zones .zones .into_iter() .map(|config| -> BlueprintZoneConfig { @@ -849,6 +674,7 @@ mod test { internal_dns_version: initial_dns_generation, external_dns_version: Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: now_db_precision(), creator: "test-suite".to_string(), comment: "test blueprint".to_string(), @@ -887,15 +713,13 @@ mod test { .zip(possible_sled_subnets) .enumerate() .map(|(i, (sled_id, subnet))| { - let sled_info = Sled { - id: *sled_id, - sled_agent_address: get_sled_address(Ipv6Subnet::new( - subnet.network(), - )), + let sled_info = Sled::new( + *sled_id, + get_sled_address(Ipv6Subnet::new(subnet.network())), // The first two of these (arbitrarily) will be marked // Scrimlets. - is_scrimlet: i < 2, - }; + if i < 2 { SledRole::Scrimlet } else { SledRole::Gimlet }, + ); (*sled_id, sled_info) }) .collect(); @@ -952,7 +776,7 @@ mod test { let mut switch_sleds_by_ip: BTreeMap<_, _> = sleds_by_id .iter() .filter_map(|(sled_id, sled)| { - if sled.is_scrimlet { + if sled.is_scrimlet() { let sled_subnet = sleds_by_id.get(sled_id).unwrap().subnet(); let switch_zone_ip = get_switch_zone_address(sled_subnet); @@ -1136,7 +960,8 @@ mod test { async fn test_blueprint_external_dns_basic() { static TEST_NAME: &str = "test_blueprint_external_dns_basic"; let logctx = test_setup_log(TEST_NAME); - let (_, _, mut blueprint) = example(&logctx.log, TEST_NAME, 5); + let (_, mut blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).nsleds(5).build(); blueprint.internal_dns_version = Generation::new(); blueprint.external_dns_version = Generation::new(); @@ -1477,7 +1302,7 @@ mod test { .await; // Now, execute the initial blueprint. - let overrides = Overridables::for_test(cptestctx); + let overrides = overridables_for_test(cptestctx); _ = realize_blueprint_and_expect( &opctx, datastore, resolver, &blueprint, &overrides, ) @@ -1547,9 +1372,11 @@ mod test { target_boundary_ntp_zone_count: BOUNDARY_NTP_REDUNDANCY, target_nexus_zone_count: NEXUS_REDUNDANCY, target_internal_dns_zone_count: INTERNAL_DNS_REDUNDANCY, + target_oximeter_zone_count: OXIMETER_REDUNDANCY, target_cockroachdb_zone_count: COCKROACHDB_REDUNDANCY, target_cockroachdb_cluster_version: CockroachDbClusterVersion::POLICY, + target_crucible_pantry_zone_count: CRUCIBLE_PANTRY_REDUNDANCY, log, } .build() @@ -1641,8 +1468,8 @@ mod test { let (new_name, &[DnsRecord::Aaaa(_)]) = new_records[0] else { panic!("did not find expected AAAA record for new Nexus zone"); }; - let new_zone_host = internal_dns::config::Host::for_zone( - internal_dns::config::Zone::Other(new_zone_id), + let new_zone_host = internal_dns_types::config::Host::for_zone( + internal_dns_types::config::Zone::Other(new_zone_id), ); assert!(new_zone_host.fqdn().starts_with(new_name)); diff --git a/nexus/reconfigurator/execution/src/lib.rs b/nexus/reconfigurator/execution/src/lib.rs index bd0c23fcf5..e160ddc9a0 100644 --- a/nexus/reconfigurator/execution/src/lib.rs +++ b/nexus/reconfigurator/execution/src/lib.rs @@ -7,7 +7,7 @@ //! See `nexus_reconfigurator_planning` crate-level docs for background. use anyhow::{anyhow, Context}; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_types::deployment::execution::*; @@ -16,67 +16,28 @@ use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::SledFilter; use nexus_types::external_api::views::SledState; use nexus_types::identity::Asset; -use omicron_common::address::Ipv6Subnet; -use omicron_common::address::SLED_PREFIX; use omicron_physical_disks::DeployDisksDone; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; -use overridables::Overridables; use slog::info; use slog_error_chain::InlineErrorChain; use std::collections::BTreeMap; -use std::net::SocketAddrV6; use std::sync::Arc; use tokio::sync::mpsc; use update_engine::merge_anyhow_list; +mod clickhouse; mod cockroachdb; mod datasets; mod dns; mod omicron_physical_disks; mod omicron_zones; -mod overridables; mod sagas; mod sled_state; #[cfg(test)] mod test_utils; -pub use dns::blueprint_external_dns_config; -pub use dns::blueprint_internal_dns_config; -pub use dns::blueprint_nexus_external_ips; -pub use dns::silo_dns_name; - -pub struct Sled { - id: SledUuid, - sled_agent_address: SocketAddrV6, - is_scrimlet: bool, -} - -impl Sled { - pub fn new( - id: SledUuid, - sled_agent_address: SocketAddrV6, - is_scrimlet: bool, - ) -> Sled { - Sled { id, sled_agent_address, is_scrimlet } - } - - pub(crate) fn subnet(&self) -> Ipv6Subnet { - Ipv6Subnet::::new(*self.sled_agent_address.ip()) - } -} - -impl From for Sled { - fn from(value: nexus_db_model::Sled) -> Self { - Sled { - id: SledUuid::from_untyped_uuid(value.id()), - sled_agent_address: value.address(), - is_scrimlet: value.is_scrimlet(), - } - } -} - /// The result of calling [`realize_blueprint`] or /// [`realize_blueprint_with_overrides`]. #[derive(Debug)] @@ -206,6 +167,12 @@ pub async fn realize_blueprint_with_overrides( deploy_disks_done, ); + register_deploy_clickhouse_cluster_nodes_step( + &engine.for_component(ExecutionComponent::Clickhouse), + &opctx, + blueprint, + ); + let reassign_saga_output = register_reassign_sagas_step( &engine.for_component(ExecutionComponent::OmicronZones), &opctx, @@ -279,7 +246,7 @@ fn register_sled_list_step<'a>( .map(|db_sled| { ( SledUuid::from_untyped_uuid(db_sled.id()), - Sled::from(db_sled), + db_sled.into(), ) }) .collect(); @@ -519,6 +486,34 @@ fn register_decommission_expunged_disks_step<'a>( .register(); } +fn register_deploy_clickhouse_cluster_nodes_step<'a>( + registrar: &ComponentRegistrar<'_, 'a>, + opctx: &'a OpContext, + blueprint: &'a Blueprint, +) { + registrar + .new_step( + ExecutionStepId::Ensure, + "Deploy clickhouse cluster nodes", + move |_cx| async move { + if let Some(clickhouse_cluster_config) = + &blueprint.clickhouse_cluster_config + { + clickhouse::deploy_nodes( + &opctx, + &blueprint.blueprint_zones, + &clickhouse_cluster_config, + ) + .await + .map_err(merge_anyhow_list)?; + } + + StepSuccess::new(()).into() + }, + ) + .register(); +} + #[derive(Debug)] struct ReassignSagaOutput { needs_saga_recovery: bool, diff --git a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs index d94bbe2e27..895b7a2d9d 100644 --- a/nexus/reconfigurator/execution/src/omicron_physical_disks.rs +++ b/nexus/reconfigurator/execution/src/omicron_physical_disks.rs @@ -44,7 +44,7 @@ pub(crate) async fn deploy_disks( let client = nexus_networking::sled_client_from_address( sled_id.into_untyped_uuid(), - db_sled.sled_agent_address, + db_sled.sled_agent_address(), &log, ); let result = @@ -143,6 +143,7 @@ mod test { use nexus_db_model::Zpool; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; + use nexus_sled_agent_shared::inventory::SledRole; use nexus_test_utils::SLED_AGENT_UUID; use nexus_test_utils_macros::nexus_test; use nexus_types::deployment::{ @@ -187,6 +188,7 @@ mod test { internal_dns_version: Generation::new(), external_dns_version: Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: chrono::Utc::now(), creator: "test".to_string(), comment: "test blueprint".to_string(), @@ -216,11 +218,7 @@ mod test { let SocketAddr::V6(addr) = server.addr() else { panic!("Expected Ipv6 address. Got {}", server.addr()); }; - let sled = Sled { - id: sled_id, - sled_agent_address: addr, - is_scrimlet: false, - }; + let sled = Sled::new(sled_id, addr, SledRole::Gimlet); (sled_id, sled) }) .collect(); diff --git a/nexus/reconfigurator/execution/src/omicron_zones.rs b/nexus/reconfigurator/execution/src/omicron_zones.rs index 3e8ff84a0d..b594c5599b 100644 --- a/nexus/reconfigurator/execution/src/omicron_zones.rs +++ b/nexus/reconfigurator/execution/src/omicron_zones.rs @@ -12,9 +12,10 @@ use cockroach_admin_client::types::NodeDecommission; use cockroach_admin_client::types::NodeId; use futures::stream; use futures::StreamExt; -use internal_dns::resolver::Resolver; -use internal_dns::ServiceName; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::datastore::CollectorReassignment; use nexus_db_queries::db::DataStore; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneDisposition; @@ -61,7 +62,7 @@ pub(crate) async fn deploy_zones( let client = nexus_networking::sled_client_from_address( sled_id.into_untyped_uuid(), - db_sled.sled_agent_address, + db_sled.sled_agent_address(), &opctx.log, ); let omicron_zones = config @@ -132,6 +133,9 @@ pub(crate) async fn clean_up_expunged_zones( ) .await, ), + BlueprintZoneType::Oximeter(_) => Some( + oximeter_cleanup(opctx, datastore, config.id, &log).await, + ), // Zones that may or may not need cleanup work - we haven't // gotten to these yet! @@ -143,8 +147,7 @@ pub(crate) async fn clean_up_expunged_zones( | BlueprintZoneType::CruciblePantry(_) | BlueprintZoneType::ExternalDns(_) | BlueprintZoneType::InternalDns(_) - | BlueprintZoneType::InternalNtp(_) - | BlueprintZoneType::Oximeter(_) => { + | BlueprintZoneType::InternalNtp(_) => { warn!( log, "unsupported zone type for expungement cleanup; \ @@ -178,6 +181,42 @@ pub(crate) async fn clean_up_expunged_zones( } } +async fn oximeter_cleanup( + opctx: &OpContext, + datastore: &DataStore, + zone_id: OmicronZoneUuid, + log: &Logger, +) -> anyhow::Result<()> { + // Record that this Oximeter instance is gone. + datastore + .oximeter_expunge(opctx, zone_id.into_untyped_uuid()) + .await + .context("failed to mark Oximeter instance deleted")?; + + // Reassign any producers it was collecting to other Oximeter instances. + match datastore + .oximeter_reassign_all_producers(opctx, zone_id.into_untyped_uuid()) + .await + .context("failed to reassign metric producers")? + { + CollectorReassignment::Complete(n) => { + info!( + log, + "successfully reassigned {n} metric producers \ + to new Oximeter collectors" + ); + } + CollectorReassignment::NoCollectorsAvailable => { + warn!( + log, + "metric producers need reassignment, but there are no \ + available Oximeter collectors" + ); + } + } + Ok(()) +} + // Helper trait that is implemented by `Resolver`, but allows unit tests to // inject a fake resolver that points to a mock server when calling // `decommission_cockroachdb_node()`. @@ -311,7 +350,7 @@ mod test { use httptest::responders::{json_encoded, status_code}; use httptest::Expectation; use nexus_sled_agent_shared::inventory::{ - OmicronZoneDataset, OmicronZonesConfig, + OmicronZoneDataset, OmicronZonesConfig, SledRole, }; use nexus_test_utils_macros::nexus_test; use nexus_types::deployment::{ @@ -351,6 +390,7 @@ mod test { internal_dns_version: Generation::new(), external_dns_version: Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: chrono::Utc::now(), creator: "test".to_string(), comment: "test blueprint".to_string(), @@ -380,11 +420,7 @@ mod test { let SocketAddr::V6(addr) = server.addr() else { panic!("Expected Ipv6 address. Got {}", server.addr()); }; - let sled = Sled { - id: sled_id, - sled_agent_address: addr, - is_scrimlet: false, - }; + let sled = Sled::new(sled_id, addr, SledRole::Gimlet); (sled_id, sled) }) .collect(); diff --git a/nexus/reconfigurator/execution/src/test_utils.rs b/nexus/reconfigurator/execution/src/test_utils.rs index 3f623af6c4..4e7521dce1 100644 --- a/nexus/reconfigurator/execution/src/test_utils.rs +++ b/nexus/reconfigurator/execution/src/test_utils.rs @@ -4,13 +4,18 @@ //! Test utilities for reconfigurator execution. -use internal_dns::resolver::Resolver; +use std::net::Ipv6Addr; + +use internal_dns_resolver::Resolver; use nexus_db_queries::{context::OpContext, db::DataStore}; -use nexus_types::deployment::{execution::EventBuffer, Blueprint}; +use nexus_types::deployment::{ + execution::{EventBuffer, Overridables}, + Blueprint, +}; use omicron_uuid_kinds::OmicronZoneUuid; use update_engine::TerminalKind; -use crate::{overridables::Overridables, RealizeBlueprintOutput}; +use crate::RealizeBlueprintOutput; pub(crate) async fn realize_blueprint_and_expect( opctx: &OpContext, @@ -63,3 +68,37 @@ pub(crate) async fn realize_blueprint_and_expect( (output, buffer) } + +/// Generates a set of overrides describing the simulated test environment. +pub fn overridables_for_test( + cptestctx: &nexus_test_utils::ControlPlaneTestContext< + omicron_nexus::Server, + >, +) -> Overridables { + use omicron_common::api::external::SwitchLocation; + + let mut overrides = Overridables::default(); + let scrimlets = [ + (nexus_test_utils::SLED_AGENT_UUID, SwitchLocation::Switch0), + (nexus_test_utils::SLED_AGENT2_UUID, SwitchLocation::Switch1), + ]; + for (id_str, switch_location) in scrimlets { + let sled_id = id_str.parse().unwrap(); + let ip = Ipv6Addr::LOCALHOST; + let mgs_port = cptestctx + .gateway + .get(&switch_location) + .unwrap() + .client + .bind_address + .port(); + let dendrite_port = + cptestctx.dendrite.get(&switch_location).unwrap().port; + let mgd_port = cptestctx.mgd.get(&switch_location).unwrap().port; + overrides.override_switch_zone_ip(sled_id, ip); + overrides.override_dendrite_port(sled_id, dendrite_port); + overrides.override_mgs_port(sled_id, mgs_port); + overrides.override_mgd_port(sled_id, mgd_port); + } + overrides +} diff --git a/nexus/reconfigurator/planning/Cargo.toml b/nexus/reconfigurator/planning/Cargo.toml index 42a89c35df..9607e26394 100644 --- a/nexus/reconfigurator/planning/Cargo.toml +++ b/nexus/reconfigurator/planning/Cargo.toml @@ -13,7 +13,7 @@ chrono.workspace = true debug-ignore.workspace = true gateway-client.workspace = true indexmap.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true ipnet.workspace = true nexus-config.workspace = true nexus-inventory.workspace = true diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs index 35512a50f3..3302768a56 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/builder.rs @@ -8,6 +8,7 @@ use crate::ip_allocator::IpAllocator; use crate::planner::zone_needs_expungement; use crate::planner::ZoneExpungeReason; use anyhow::anyhow; +use clickhouse_admin_types::OXIMETER_CLUSTER; use ipnet::IpAdd; use nexus_inventory::now_db_precision; use nexus_sled_agent_shared::inventory::OmicronZoneDataset; @@ -21,6 +22,7 @@ use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::BlueprintZoneType; use nexus_types::deployment::BlueprintZonesConfig; +use nexus_types::deployment::ClickhouseClusterConfig; use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::deployment::DiskFilter; use nexus_types::deployment::OmicronZoneExternalFloatingAddr; @@ -37,6 +39,7 @@ use nexus_types::inventory::Collection; use omicron_common::address::get_sled_address; use omicron_common::address::get_switch_zone_address; use omicron_common::address::ReservedRackSubnet; +use omicron_common::address::CLICKHOUSE_HTTP_PORT; use omicron_common::address::CP_SERVICES_RESERVED_ADDRESSES; use omicron_common::address::DNS_HTTP_PORT; use omicron_common::address::DNS_PORT; @@ -75,6 +78,7 @@ use thiserror::Error; use typed_rng::TypedUuidRng; use typed_rng::UuidRng; +use super::clickhouse::ClickhouseAllocator; use super::external_networking::BuilderExternalNetworking; use super::external_networking::ExternalNetworkingChoice; use super::external_networking::ExternalSnatNetworkingChoice; @@ -175,6 +179,9 @@ impl fmt::Display for Operation { ZoneExpungeReason::SledExpunged => { "sled policy is expunged" } + ZoneExpungeReason::ClickhouseClusterDisabled => { + "clickhouse cluster disabled via policy" + } }; write!( f, @@ -216,6 +223,7 @@ pub struct BlueprintBuilder<'a> { sled_ip_allocators: BTreeMap, external_networking: OnceCell>, internal_dns_subnets: OnceCell, + clickhouse_allocator: Option, // These fields will become part of the final blueprint. See the // corresponding fields in `Blueprint`. @@ -278,6 +286,7 @@ impl<'a> BlueprintBuilder<'a> { .copied() .map(|sled_id| (sled_id, SledState::Active)) .collect(); + Blueprint { id: rng.blueprint_rng.next(), blueprint_zones, @@ -289,6 +298,7 @@ impl<'a> BlueprintBuilder<'a> { cockroachdb_fingerprint: String::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, + clickhouse_cluster_config: None, time_created: now_db_precision(), creator: creator.to_owned(), comment: format!("starting blueprint with {num_sleds} empty sleds"), @@ -334,6 +344,43 @@ impl<'a> BlueprintBuilder<'a> { || commissioned_sled_ids.contains(sled_id) }); + // If we have the clickhouse cluster setup enabled via policy and we + // don't yet have a `ClickhouseClusterConfiguration`, then we must create + // one and feed it to our `ClickhouseAllocator`. + let clickhouse_allocator = if input.clickhouse_cluster_enabled() { + let parent_config = parent_blueprint + .clickhouse_cluster_config + .clone() + .unwrap_or_else(|| { + info!( + log, + concat!( + "Clickhouse cluster enabled by policy: ", + "generating initial 'ClickhouseClusterConfig' ", + "and 'ClickhouseAllocator'" + ) + ); + ClickhouseClusterConfig::new(OXIMETER_CLUSTER.to_string()) + }); + Some(ClickhouseAllocator::new( + log.clone(), + parent_config, + inventory.latest_clickhouse_keeper_membership(), + )) + } else { + if parent_blueprint.clickhouse_cluster_config.is_some() { + info!( + log, + concat!( + "clickhouse cluster disabled via policy ", + "discarding existing 'ClickhouseAllocator' and ", + "the resulting generated 'ClickhouseClusterConfig" + ) + ); + } + None + }; + Ok(BlueprintBuilder { log, parent_blueprint, @@ -347,6 +394,7 @@ impl<'a> BlueprintBuilder<'a> { sled_state, cockroachdb_setting_preserve_downgrade: parent_blueprint .cockroachdb_setting_preserve_downgrade, + clickhouse_allocator, creator: creator.to_owned(), operations: Vec::new(), comments: Vec::new(), @@ -427,6 +475,18 @@ impl<'a> BlueprintBuilder<'a> { let blueprint_disks = self .disks .into_disks_map(self.input.all_sled_ids(SledFilter::InService)); + + // If we have an allocator, use it to generate a new config. If an error + // is returned then log it and carry over the parent_config. + let clickhouse_cluster_config = self.clickhouse_allocator.map(|a| { + match a.plan(&(&blueprint_zones).into()) { + Ok(config) => config, + Err(e) => { + error!(self.log, "clickhouse allocator error: {e}"); + a.parent_config().clone() + } + } + }); Blueprint { id: self.rng.blueprint_rng.next(), blueprint_zones, @@ -442,6 +502,7 @@ impl<'a> BlueprintBuilder<'a> { .clone(), cockroachdb_setting_preserve_downgrade: self .cockroachdb_setting_preserve_downgrade, + clickhouse_cluster_config, time_created: now_db_precision(), creator: self.creator, comment: self @@ -501,6 +562,13 @@ impl<'a> BlueprintBuilder<'a> { "sled_id" => sled_id.to_string(), )); + // If there are any `ClickhouseServer` or `ClickhouseKeeper` zones that + // are not expunged and we no longer have a `ClickhousePolicy` which + // indicates replicated clickhouse clusters should be running, we need + // to expunge all such zones. + let clickhouse_cluster_enabled = + self.input.clickhouse_cluster_enabled(); + // Do any zones need to be marked expunged? let mut zones_to_expunge = BTreeMap::new(); @@ -512,9 +580,11 @@ impl<'a> BlueprintBuilder<'a> { "zone_id" => zone_id.to_string() )); - let Some(reason) = - zone_needs_expungement(sled_details, zone_config) - else { + let Some(reason) = zone_needs_expungement( + sled_details, + zone_config, + clickhouse_cluster_enabled, + ) else { continue; }; @@ -553,6 +623,13 @@ impl<'a> BlueprintBuilder<'a> { "expunged sled with non-expunged zone found" ); } + ZoneExpungeReason::ClickhouseClusterDisabled => { + info!( + &log, + "clickhouse cluster disabled via policy, \ + expunging related zone" + ); + } } zones_to_expunge.insert(zone_id, reason); @@ -583,6 +660,7 @@ impl<'a> BlueprintBuilder<'a> { let mut count_disk_expunged = 0; let mut count_sled_decommissioned = 0; let mut count_sled_expunged = 0; + let mut count_clickhouse_cluster_disabled = 0; for reason in zones_to_expunge.values() { match reason { ZoneExpungeReason::DiskExpunged => count_disk_expunged += 1, @@ -590,12 +668,19 @@ impl<'a> BlueprintBuilder<'a> { count_sled_decommissioned += 1; } ZoneExpungeReason::SledExpunged => count_sled_expunged += 1, + ZoneExpungeReason::ClickhouseClusterDisabled => { + count_clickhouse_cluster_disabled += 1 + } }; } let count_and_reason = [ (count_disk_expunged, ZoneExpungeReason::DiskExpunged), (count_sled_decommissioned, ZoneExpungeReason::SledDecommissioned), (count_sled_expunged, ZoneExpungeReason::SledExpunged), + ( + count_clickhouse_cluster_disabled, + ZoneExpungeReason::ClickhouseClusterDisabled, + ), ]; for (count, reason) in count_and_reason { if count > 0 { @@ -1062,6 +1147,97 @@ impl<'a> BlueprintBuilder<'a> { Ok(EnsureMultiple::Changed { added: num_nexus_to_add, removed: 0 }) } + pub fn sled_ensure_zone_multiple_oximeter( + &mut self, + sled_id: SledUuid, + desired_zone_count: usize, + ) -> Result { + // How many Oximeter zones do we need to add? + let oximeter_count = + self.sled_num_running_zones_of_kind(sled_id, ZoneKind::Oximeter); + let num_oximeter_to_add = + match desired_zone_count.checked_sub(oximeter_count) { + Some(0) => return Ok(EnsureMultiple::NotNeeded), + Some(n) => n, + None => { + return Err(Error::Planner(anyhow!( + "removing an Oximeter zone not yet supported \ + (sled {sled_id} has {oximeter_count}; \ + planner wants {desired_zone_count})" + ))); + } + }; + + for _ in 0..num_oximeter_to_add { + let oximeter_id = self.rng.zone_rng.next(); + let ip = self.sled_alloc_ip(sled_id)?; + let port = omicron_common::address::OXIMETER_PORT; + let address = SocketAddrV6::new(ip, port, 0, 0); + let zone_type = + BlueprintZoneType::Oximeter(blueprint_zone_type::Oximeter { + address, + }); + let filesystem_pool = + self.sled_select_zpool(sled_id, zone_type.kind())?; + + let zone = BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::InService, + id: oximeter_id, + underlay_address: ip, + filesystem_pool: Some(filesystem_pool), + zone_type, + }; + self.sled_add_zone(sled_id, zone)?; + } + + Ok(EnsureMultiple::Changed { added: num_oximeter_to_add, removed: 0 }) + } + + pub fn sled_ensure_zone_multiple_crucible_pantry( + &mut self, + sled_id: SledUuid, + desired_zone_count: usize, + ) -> Result { + // How many zones do we need to add? + let pantry_count = self + .sled_num_running_zones_of_kind(sled_id, ZoneKind::CruciblePantry); + let num_pantry_to_add = + match desired_zone_count.checked_sub(pantry_count) { + Some(0) => return Ok(EnsureMultiple::NotNeeded), + Some(n) => n, + None => { + return Err(Error::Planner(anyhow!( + "removing a Crucible pantry zone not yet supported \ + (sled {sled_id} has {pantry_count}; \ + planner wants {desired_zone_count})" + ))); + } + }; + + for _ in 0..num_pantry_to_add { + let pantry_id = self.rng.zone_rng.next(); + let ip = self.sled_alloc_ip(sled_id)?; + let port = omicron_common::address::CRUCIBLE_PANTRY_PORT; + let address = SocketAddrV6::new(ip, port, 0, 0); + let zone_type = BlueprintZoneType::CruciblePantry( + blueprint_zone_type::CruciblePantry { address }, + ); + let filesystem_pool = + self.sled_select_zpool(sled_id, zone_type.kind())?; + + let zone = BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::InService, + id: pantry_id, + underlay_address: ip, + filesystem_pool: Some(filesystem_pool), + zone_type, + }; + self.sled_add_zone(sled_id, zone)?; + } + + Ok(EnsureMultiple::Changed { added: num_pantry_to_add, removed: 0 }) + } + pub fn cockroachdb_preserve_downgrade( &mut self, version: CockroachDbPreserveDowngrade, @@ -1118,6 +1294,169 @@ impl<'a> BlueprintBuilder<'a> { Ok(EnsureMultiple::Changed { added: num_crdb_to_add, removed: 0 }) } + fn sled_add_zone_clickhouse( + &mut self, + sled_id: SledUuid, + ) -> Result { + let id = self.rng.zone_rng.next(); + let underlay_address = self.sled_alloc_ip(sled_id)?; + let address = + SocketAddrV6::new(underlay_address, CLICKHOUSE_HTTP_PORT, 0, 0); + let pool_name = + self.sled_select_zpool(sled_id, ZoneKind::Clickhouse)?; + let zone_type = + BlueprintZoneType::Clickhouse(blueprint_zone_type::Clickhouse { + address, + dataset: OmicronZoneDataset { pool_name: pool_name.clone() }, + }); + + let zone = BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::InService, + id, + underlay_address, + filesystem_pool: Some(pool_name), + zone_type, + }; + self.sled_add_zone(sled_id, zone)?; + Ok(Ensure::Added) + } + + pub fn sled_ensure_zone_multiple_clickhouse( + &mut self, + sled_id: SledUuid, + desired_zone_count: usize, + ) -> Result { + // How many single-node ClickHouse zones do we want to add? + let count = + self.sled_num_running_zones_of_kind(sled_id, ZoneKind::Clickhouse); + let to_add = match desired_zone_count.checked_sub(count) { + Some(0) => return Ok(EnsureMultiple::NotNeeded), + Some(n) => n, + None => { + return Err(Error::Planner(anyhow!( + "removing a single-node ClickHouse zone not yet supported \ + (sled {sled_id} has {count}; \ + planner wants {desired_zone_count})" + ))); + } + }; + for _ in 0..to_add { + self.sled_add_zone_clickhouse(sled_id)?; + } + Ok(EnsureMultiple::Changed { added: to_add, removed: 0 }) + } + + pub fn sled_ensure_zone_multiple_clickhouse_server( + &mut self, + sled_id: SledUuid, + desired_zone_count: usize, + ) -> Result { + // How many clickhouse server zones do we want to add? + let clickhouse_server_count = self.sled_num_running_zones_of_kind( + sled_id, + ZoneKind::ClickhouseServer, + ); + let num_clickhouse_servers_to_add = + match desired_zone_count.checked_sub(clickhouse_server_count) { + Some(0) => return Ok(EnsureMultiple::NotNeeded), + Some(n) => n, + None => { + return Err(Error::Planner(anyhow!( + "removing a ClickhouseServer zone not yet supported \ + (sled {sled_id} has {clickhouse_server_count}; \ + planner wants {desired_zone_count})" + ))); + } + }; + for _ in 0..num_clickhouse_servers_to_add { + let zone_id = self.rng.zone_rng.next(); + let underlay_ip = self.sled_alloc_ip(sled_id)?; + let pool_name = + self.sled_select_zpool(sled_id, ZoneKind::ClickhouseServer)?; + let address = + SocketAddrV6::new(underlay_ip, CLICKHOUSE_HTTP_PORT, 0, 0); + let zone_type = BlueprintZoneType::ClickhouseServer( + blueprint_zone_type::ClickhouseServer { + address, + dataset: OmicronZoneDataset { + pool_name: pool_name.clone(), + }, + }, + ); + let filesystem_pool = pool_name; + + let zone = BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::InService, + id: zone_id, + underlay_address: underlay_ip, + filesystem_pool: Some(filesystem_pool), + zone_type, + }; + self.sled_add_zone(sled_id, zone)?; + } + + Ok(EnsureMultiple::Changed { + added: num_clickhouse_servers_to_add, + removed: 0, + }) + } + + pub fn sled_ensure_zone_multiple_clickhouse_keeper( + &mut self, + sled_id: SledUuid, + desired_zone_count: usize, + ) -> Result { + // How many clickhouse keeper zones do we want to add? + let clickhouse_keeper_count = self.sled_num_running_zones_of_kind( + sled_id, + ZoneKind::ClickhouseKeeper, + ); + let num_clickhouse_keepers_to_add = + match desired_zone_count.checked_sub(clickhouse_keeper_count) { + Some(0) => return Ok(EnsureMultiple::NotNeeded), + Some(n) => n, + None => { + return Err(Error::Planner(anyhow!( + "removing a ClickhouseKeeper zone not yet supported \ + (sled {sled_id} has {clickhouse_keeper_count}; \ + planner wants {desired_zone_count})" + ))); + } + }; + + for _ in 0..num_clickhouse_keepers_to_add { + let zone_id = self.rng.zone_rng.next(); + let underlay_ip = self.sled_alloc_ip(sled_id)?; + let pool_name = + self.sled_select_zpool(sled_id, ZoneKind::ClickhouseKeeper)?; + let port = omicron_common::address::CLICKHOUSE_KEEPER_TCP_PORT; + let address = SocketAddrV6::new(underlay_ip, port, 0, 0); + let zone_type = BlueprintZoneType::ClickhouseKeeper( + blueprint_zone_type::ClickhouseKeeper { + address, + dataset: OmicronZoneDataset { + pool_name: pool_name.clone(), + }, + }, + ); + let filesystem_pool = pool_name; + + let zone = BlueprintZoneConfig { + disposition: BlueprintZoneDisposition::InService, + id: zone_id, + underlay_address: underlay_ip, + filesystem_pool: Some(filesystem_pool), + zone_type, + }; + self.sled_add_zone(sled_id, zone)?; + } + + Ok(EnsureMultiple::Changed { + added: num_clickhouse_keepers_to_add, + removed: 0, + }) + } + pub fn sled_promote_internal_ntp_to_boundary_ntp( &mut self, sled_id: SledUuid, @@ -1405,12 +1744,11 @@ impl<'a> BlueprintBuilder<'a> { /// ordinarily only come from RSS. /// /// TODO-cleanup: Remove when external DNS addresses are in the policy. - #[cfg(test)] - #[track_caller] - pub fn add_external_dns_ip(&mut self, addr: IpAddr) { - self.external_networking() - .expect("failed to initialize external networking allocator") - .add_external_dns_ip(addr); + pub(crate) fn add_external_dns_ip( + &mut self, + addr: IpAddr, + ) -> Result<(), Error> { + self.external_networking()?.add_external_dns_ip(addr) } } @@ -1687,7 +2025,7 @@ impl<'a> BlueprintDisksBuilder<'a> { pub mod test { use super::*; use crate::example::example; - use crate::example::ExampleSystem; + use crate::example::ExampleSystemBuilder; use crate::system::SledBuilder; use expectorate::assert_contents; use nexus_inventory::CollectionBuilder; @@ -1701,8 +2039,6 @@ pub mod test { use std::collections::BTreeSet; use std::mem; - pub const DEFAULT_N_SLEDS: usize = 3; - /// Checks various conditions that should be true for all blueprints #[track_caller] pub fn verify_blueprint(blueprint: &Blueprint) { @@ -1792,7 +2128,7 @@ pub mod test { static TEST_NAME: &str = "blueprint_builder_test_initial"; let logctx = test_setup_log(TEST_NAME); let (collection, input, blueprint_initial) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + example(&logctx.log, TEST_NAME); verify_blueprint(&blueprint_initial); let diff = blueprint_initial.diff_since_collection(&collection); @@ -1827,14 +2163,13 @@ pub mod test { fn test_basic() { static TEST_NAME: &str = "blueprint_builder_test_basic"; let logctx = test_setup_log(TEST_NAME); - let mut example = - ExampleSystem::new(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); - let blueprint1 = &example.blueprint; - verify_blueprint(blueprint1); + let (mut example, blueprint1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).build(); + verify_blueprint(&blueprint1); let mut builder = BlueprintBuilder::new_based_on( &logctx.log, - blueprint1, + &blueprint1, &example.input, &example.collection, "test_basic", @@ -1986,7 +2321,7 @@ pub mod test { "blueprint_builder_test_prune_decommissioned_sleds"; let logctx = test_setup_log(TEST_NAME); let (collection, input, mut blueprint1) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + example(&logctx.log, TEST_NAME); verify_blueprint(&blueprint1); // Mark one sled as having a desired state of decommissioned. @@ -2071,23 +2406,16 @@ pub mod test { fn test_add_physical_disks() { static TEST_NAME: &str = "blueprint_builder_test_add_physical_disks"; let logctx = test_setup_log(TEST_NAME); - let (collection, input, _) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); - let input = { - // Clear out the external networking records from `input`, since - // we're building an empty blueprint. - let mut builder = input.into_builder(); - *builder.network_resources_mut() = - OmicronZoneNetworkResources::new(); - builder.build() - }; - // Start with an empty blueprint (sleds with no zones). - let parent = BlueprintBuilder::build_empty_with_sleds_seeded( - input.all_sled_ids(SledFilter::Commissioned), - "test", - TEST_NAME, - ); + // Start with an empty system (sleds with no zones). However, we leave + // the disks around so that `sled_ensure_disks` can add them. + let (example, parent) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .create_zones(false) + .create_disks_in_blueprint(false) + .build(); + let collection = example.collection; + let input = example.input; { // We start empty, and can add a disk @@ -2101,7 +2429,17 @@ pub mod test { .expect("failed to create builder"); assert!(builder.disks.changed_disks.is_empty()); - assert!(builder.disks.parent_disks.is_empty()); + // In the parent_disks map, we expect entries to be present for + // each sled, but not have any disks in them. + for (sled_id, disks) in builder.disks.parent_disks { + assert_eq!( + disks.disks, + Vec::new(), + "for sled {}, expected no disks present in parent, \ + but found some", + sled_id + ); + } for (sled_id, sled_resources) in input.all_sled_resources(SledFilter::InService) @@ -2110,12 +2448,25 @@ pub mod test { builder .sled_ensure_disks(sled_id, &sled_resources) .unwrap(), - EnsureMultiple::Changed { added: 10, removed: 0 }, + EnsureMultiple::Changed { + added: usize::from(SledBuilder::DEFAULT_NPOOLS), + removed: 0 + }, ); } assert!(!builder.disks.changed_disks.is_empty()); - assert!(builder.disks.parent_disks.is_empty()); + // In the parent_disks map, we expect entries to be present for + // each sled, but not have any disks in them. + for (sled_id, disks) in builder.disks.parent_disks { + assert_eq!( + disks.disks, + Vec::new(), + "for sled {}, expected no disks present in parent, \ + but found some", + sled_id + ); + } } logctx.cleanup_successful(); @@ -2127,8 +2478,7 @@ pub mod test { static TEST_NAME: &str = "blueprint_builder_test_zone_filesystem_zpool_colocated"; let logctx = test_setup_log(TEST_NAME); - let (_, _, blueprint) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + let (_, _, blueprint) = example(&logctx.log, TEST_NAME); for (_, zone_config) in &blueprint.blueprint_zones { for zone in &zone_config.zones { @@ -2152,22 +2502,13 @@ pub mod test { "blueprint_builder_test_add_nexus_with_no_existing_nexus_zones"; let logctx = test_setup_log(TEST_NAME); - // Discard the example blueprint and start with an empty one. - let (collection, input, _) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); - let input = { - // Clear out the external networking records from `input`, since - // we're building an empty blueprint. - let mut builder = input.into_builder(); - *builder.network_resources_mut() = - OmicronZoneNetworkResources::new(); - builder.build() - }; - let parent = BlueprintBuilder::build_empty_with_sleds_seeded( - input.all_sled_ids(SledFilter::Commissioned), - "test", - TEST_NAME, - ); + // Start with an empty system (sleds with no zones). + let (example, parent) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .create_zones(false) + .build(); + let collection = example.collection; + let input = example.input; // Adding a new Nexus zone currently requires copying settings from an // existing Nexus zone. `parent` has no zones, so we should fail if we @@ -2184,7 +2525,7 @@ pub mod test { let err = builder .sled_ensure_zone_multiple_nexus( collection - .omicron_zones + .sled_agents .keys() .next() .copied() @@ -2206,17 +2547,17 @@ pub mod test { static TEST_NAME: &str = "blueprint_builder_test_add_nexus_error_cases"; let logctx = test_setup_log(TEST_NAME); let (mut collection, mut input, mut parent) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + example(&logctx.log, TEST_NAME); // Remove the Nexus zone from one of the sleds so that // `sled_ensure_zone_nexus` can attempt to add a Nexus zone to // `sled_id`. let sled_id = { let mut selected_sled_id = None; - for (sled_id, zones) in &mut collection.omicron_zones { - let nzones_before_retain = zones.zones.zones.len(); - zones.zones.zones.retain(|z| !z.zone_type.is_nexus()); - if zones.zones.zones.len() < nzones_before_retain { + for (sled_id, sa) in &mut collection.sled_agents { + let nzones_before_retain = sa.omicron_zones.zones.len(); + sa.omicron_zones.zones.retain(|z| !z.zone_type.is_nexus()); + if sa.omicron_zones.zones.len() < nzones_before_retain { selected_sled_id = Some(*sled_id); // Also remove this zone from the blueprint. let mut removed_nexus = None; @@ -2361,8 +2702,7 @@ pub mod test { "blueprint_builder_test_invalid_parent_blueprint_\ two_zones_with_same_external_ip"; let logctx = test_setup_log(TEST_NAME); - let (collection, input, mut parent) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + let (collection, input, mut parent) = example(&logctx.log, TEST_NAME); // We should fail if the parent blueprint claims to contain two // zones with the same external IP. Skim through the zones, copy the @@ -2420,8 +2760,7 @@ pub mod test { "blueprint_builder_test_invalid_parent_blueprint_\ two_nexus_zones_with_same_nic_ip"; let logctx = test_setup_log(TEST_NAME); - let (collection, input, mut parent) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + let (collection, input, mut parent) = example(&logctx.log, TEST_NAME); // We should fail if the parent blueprint claims to contain two // Nexus zones with the same NIC IP. Skim through the zones, copy @@ -2479,8 +2818,7 @@ pub mod test { "blueprint_builder_test_invalid_parent_blueprint_\ two_zones_with_same_vnic_mac"; let logctx = test_setup_log(TEST_NAME); - let (collection, input, mut parent) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + let (collection, input, mut parent) = example(&logctx.log, TEST_NAME); // We should fail if the parent blueprint claims to contain two // zones with the same service vNIC MAC address. Skim through the @@ -2537,22 +2875,13 @@ pub mod test { static TEST_NAME: &str = "blueprint_builder_test_ensure_cockroachdb"; let logctx = test_setup_log(TEST_NAME); - // Discard the example blueprint and start with an empty one. - let (collection, input, _) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); - let input = { - // Clear out the external networking records from `input`, since - // we're building an empty blueprint. - let mut builder = input.into_builder(); - *builder.network_resources_mut() = - OmicronZoneNetworkResources::new(); - builder.build() - }; - let parent = BlueprintBuilder::build_empty_with_sleds_seeded( - input.all_sled_ids(SledFilter::Commissioned), - "test", - TEST_NAME, - ); + // Start with an empty system (sleds with no zones). + let (example, parent) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .create_zones(false) + .build(); + let collection = example.collection; + let input = example.input; // Pick an arbitrary sled. let (target_sled_id, sled_resources) = input diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs b/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs index 4071346632..257ab50bd1 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs @@ -5,12 +5,11 @@ //! A mechanism for allocating clickhouse keeper and server nodes for clustered //! clickhouse setups during blueprint planning -use clickhouse_admin_types::KeeperId; +use clickhouse_admin_types::{ClickhouseKeeperClusterMembership, KeeperId}; use nexus_types::deployment::{ Blueprint, BlueprintZoneFilter, BlueprintZoneType, BlueprintZonesConfig, ClickhouseClusterConfig, }; -use nexus_types::inventory::ClickhouseKeeperClusterMembership; use omicron_uuid_kinds::{OmicronZoneUuid, SledUuid}; use slog::{error, Logger}; use std::collections::{BTreeMap, BTreeSet}; @@ -22,9 +21,9 @@ use thiserror::Error; // Will be removed once the planner starts using this code // See: https://github.com/oxidecomputer/omicron/issues/6577 #[allow(unused)] -struct ClickhouseZonesThatShouldBeRunning { - keepers: BTreeSet, - servers: BTreeSet, +pub struct ClickhouseZonesThatShouldBeRunning { + pub keepers: BTreeSet, + pub servers: BTreeSet, } impl From<&BTreeMap> @@ -64,7 +63,6 @@ impl From<&BTreeMap> #[allow(unused)] pub struct ClickhouseAllocator { log: Logger, - active_clickhouse_zones: ClickhouseZonesThatShouldBeRunning, parent_config: ClickhouseClusterConfig, // The latest clickhouse cluster membership from inventory inventory: Option, @@ -77,10 +75,6 @@ pub struct ClickhouseAllocator { #[allow(unused)] #[derive(Debug, Error)] pub enum KeeperAllocationError { - #[error("a clickhouse cluster configuration has not been created")] - NoConfig, - #[error("failed to retrieve clickhouse keeper membership from inventory")] - NoInventory, #[error("cannot add more than one keeper at a time: {added_keepers:?}")] BadMembershipChange { added_keepers: BTreeSet }, } @@ -91,13 +85,11 @@ pub enum KeeperAllocationError { impl ClickhouseAllocator { pub fn new( log: Logger, - zones_by_sled_id: &BTreeMap, clickhouse_cluster_config: ClickhouseClusterConfig, inventory: Option, ) -> ClickhouseAllocator { ClickhouseAllocator { log, - active_clickhouse_zones: zones_by_sled_id.into(), parent_config: clickhouse_cluster_config, inventory, } @@ -107,6 +99,7 @@ impl ClickhouseAllocator { /// on the parent blueprint and inventory pub fn plan( &self, + active_clickhouse_zones: &ClickhouseZonesThatShouldBeRunning, ) -> Result { let mut new_config = self.parent_config.clone(); @@ -122,10 +115,10 @@ impl ClickhouseAllocator { // First, remove the clickhouse servers that are no longer in service new_config.servers.retain(|zone_id, _| { - self.active_clickhouse_zones.servers.contains(zone_id) + active_clickhouse_zones.servers.contains(zone_id) }); // Next, add any new clickhouse servers - for zone_id in &self.active_clickhouse_zones.servers { + for zone_id in &active_clickhouse_zones.servers { if !new_config.servers.contains_key(zone_id) { // Allocate a new `ServerId` and map it to the server zone new_config.max_used_server_id += 1.into(); @@ -136,16 +129,37 @@ impl ClickhouseAllocator { } // Now we need to configure the keepers. We can only add or remove - // one keeper at a time. + // one keeper at a time during a reconfiguration. // // We need to see if we have any keeper inventory so we can compare it // with our current configuration and see if any changes are required. // If we fail to retrieve any inventory for keepers in the current // collection than we must not modify our keeper config, as we don't // know whether a configuration is ongoing or not. + // + // There is an exception to this rule: on *new* clusters that have + // keeper zones deployed but do not have any keepers running we can + // create a full cluster configuration unconditionally. We can add + // more than one keeper because this is the initial configuration and + // not a "reconfiguration" that only allows adding or removing one + // node at a time. Furthermore, we have to start at last one keeper + // unconditionally in this case because we cannot retrieve keeper + // inventory if there are no keepers running. Without retrieving + // inventory, we cannot make further progress. let current_keepers: BTreeSet<_> = self.parent_config.keepers.values().cloned().collect(); let Some(inventory_membership) = &self.inventory else { + // Are we a new cluster ? + if new_config.max_used_keeper_id == 0.into() { + // Generate our initial configuration + for zone_id in &active_clickhouse_zones.keepers { + // Allocate a new `KeeperId` and map it to the zone_id + new_config.max_used_keeper_id += 1.into(); + new_config + .keepers + .insert(*zone_id, new_config.max_used_keeper_id); + } + } return bump_gen_if_necessary(new_config); }; @@ -219,7 +233,7 @@ impl ClickhouseAllocator { // Let's ensure that this zone has not been expunged yet. If it has that means // that adding the keeper will never succeed. - if !self.active_clickhouse_zones.keepers.contains(added_zone_id) { + if !active_clickhouse_zones.keepers.contains(added_zone_id) { // The zone has been expunged, so we must remove it from our configuration. new_config.keepers.remove(added_zone_id); @@ -245,7 +259,7 @@ impl ClickhouseAllocator { // We remove first, because the zones are already gone and therefore // don't help our quorum. for (zone_id, _) in &self.parent_config.keepers { - if !self.active_clickhouse_zones.keepers.contains(&zone_id) { + if !active_clickhouse_zones.keepers.contains(&zone_id) { // Remove the keeper for the first expunged zone we see. // Remember, we only do one keeper membership change at time. new_config.keepers.remove(zone_id); @@ -254,9 +268,9 @@ impl ClickhouseAllocator { } // Do we need to add any nodes to in service zones that don't have them - for zone_id in &self.active_clickhouse_zones.keepers { + for zone_id in &active_clickhouse_zones.keepers { if !new_config.keepers.contains_key(zone_id) { - // Allocate a new `KeeperId` and map it to the server zone + // Allocate a new `KeeperId` and map it to the keeper zone new_config.max_used_keeper_id += 1.into(); new_config .keepers @@ -268,6 +282,10 @@ impl ClickhouseAllocator { // We possibly added or removed clickhouse servers, but not keepers. bump_gen_if_necessary(new_config) } + + pub fn parent_config(&self) -> &ClickhouseClusterConfig { + &self.parent_config + } } #[cfg(test)] @@ -363,13 +381,12 @@ pub mod test { let mut allocator = ClickhouseAllocator { log: logctx.log.clone(), - active_clickhouse_zones, parent_config: parent_config.clone(), inventory, }; // Our clickhouse cluster config should not have changed - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); // Note that we cannot directly check equality here and // in a bunch of the test cases below, because we bump the @@ -380,7 +397,7 @@ pub mod test { // Running again without changing the inventory should be idempotent allocator.parent_config = new_config; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config, allocator.parent_config); logctx.cleanup_successful(); @@ -412,13 +429,12 @@ pub mod test { // allocator should allocate one more keeper. let mut allocator = ClickhouseAllocator { log: logctx.log.clone(), - active_clickhouse_zones, parent_config: parent_config.clone(), inventory, }; // Did our new config change as we expect? - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.generation, Generation::from_u32(2)); assert_eq!(new_config.generation, parent_config.generation.next()); assert_eq!(new_config.max_used_keeper_id, 4.into()); @@ -440,14 +456,14 @@ pub mod test { // itself does not modify the allocator and a new one is created by the // `BlueprintBuilder` on each planning round. allocator.parent_config = new_config; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config, allocator.parent_config); // Now let's update our inventory to reflect the new keeper. This should // trigger the planner to add a 5th keeper. allocator.inventory.as_mut().unwrap().raft_config.insert(4.into()); allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.generation, Generation::from_u32(3)); assert_eq!( new_config.generation, @@ -473,7 +489,7 @@ pub mod test { // inventory raft config. We should end up with the same config. allocator.parent_config = new_config; allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert!(!new_config.needs_generation_bump(&allocator.parent_config)); // Now let's modify the inventory to reflect that the 5th keeper node @@ -483,7 +499,7 @@ pub mod test { // our keeper zones have a keeper that is part of the cluster. allocator.inventory.as_mut().unwrap().raft_config.insert(5.into()); allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert!(!new_config.needs_generation_bump(&allocator.parent_config)); logctx.cleanup_successful(); @@ -497,7 +513,7 @@ pub mod test { let (n_keeper_zones, n_server_zones, n_keepers, n_servers) = (5, 2, 5, 2); - let (active_clickhouse_zones, parent_config) = initial_config( + let (mut active_clickhouse_zones, parent_config) = initial_config( n_keeper_zones, n_server_zones, n_keepers, @@ -512,23 +528,22 @@ pub mod test { let mut allocator = ClickhouseAllocator { log: logctx.log.clone(), - active_clickhouse_zones, parent_config: parent_config.clone(), inventory, }; // Our clickhouse cluster config should not have changed // We have 5 keepers and 5 zones and all of them are in the inventory - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert!(!new_config.needs_generation_bump(&parent_config)); // Now expunge 2 of the 5 keeper zones by removing them from the // in-service zones - allocator.active_clickhouse_zones.keepers.pop_first(); - allocator.active_clickhouse_zones.keepers.pop_first(); + active_clickhouse_zones.keepers.pop_first(); + active_clickhouse_zones.keepers.pop_first(); // Running the planner should remove one of the keepers from the new config - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.generation, Generation::from_u32(2)); assert_eq!( new_config.generation, @@ -554,14 +569,14 @@ pub mod test { // since the inventory hasn't reflected the change allocator.parent_config = new_config; allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert!(!new_config.needs_generation_bump(&allocator.parent_config)); // Reflecting the new config in inventory should remove another keeper allocator.inventory.as_mut().unwrap().raft_config = new_config.keepers.values().cloned().collect(); allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.generation, Generation::from_u32(3)); assert_eq!( @@ -588,7 +603,7 @@ pub mod test { // change, because the inventory doesn't reflect the removed keeper allocator.parent_config = new_config; allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&&active_clickhouse_zones).unwrap(); assert!(!new_config.needs_generation_bump(&allocator.parent_config)); // Reflecting the keeper removal in inventory should also result in no @@ -597,7 +612,7 @@ pub mod test { new_config.keepers.values().cloned().collect(); allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; allocator.parent_config = new_config; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert!(!new_config.needs_generation_bump(&allocator.parent_config)); logctx.cleanup_successful(); @@ -612,7 +627,7 @@ pub mod test { let (n_keeper_zones, n_server_zones, n_keepers, n_servers) = (5, 2, 4, 2); - let (active_clickhouse_zones, parent_config) = initial_config( + let (mut active_clickhouse_zones, parent_config) = initial_config( n_keeper_zones, n_server_zones, n_keepers, @@ -627,14 +642,13 @@ pub mod test { let mut allocator = ClickhouseAllocator { log: logctx.log.clone(), - active_clickhouse_zones, parent_config, inventory, }; // First run the planner to add a 5th keeper to our config assert_eq!(allocator.parent_config.keepers.len(), 4); - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.keepers.len(), 5); // Pick one of the keepers currently in our inventory, find the zone @@ -654,7 +668,7 @@ pub mod test { .find(|(_, &keeper_id)| keeper_id == keeper_to_expunge) .map(|(zone_id, _)| *zone_id) .unwrap(); - allocator.active_clickhouse_zones.keepers.remove(&zone_to_expunge); + active_clickhouse_zones.keepers.remove(&zone_to_expunge); // Bump the inventory commit index so we guarantee we perform the keeper // checks @@ -666,7 +680,7 @@ pub mod test { // Run the plan. Our configuration should stay the same because we can // only add or remove one keeper node from the cluster at a time and we // are already in the process of adding a node. - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert!(!new_config.needs_generation_bump(&allocator.parent_config)); // Now we change the inventory to reflect the addition of the node to @@ -675,7 +689,7 @@ pub mod test { allocator.parent_config = new_config; allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; allocator.inventory.as_mut().unwrap().raft_config.insert(5.into()); - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.keepers.len(), 4); // Let's make sure that the right keeper was expunged. @@ -694,8 +708,8 @@ pub mod test { .raft_config .remove(&keeper_to_expunge); let new_zone_id = OmicronZoneUuid::new_v4(); - allocator.active_clickhouse_zones.keepers.insert(new_zone_id); - let new_config = allocator.plan().unwrap(); + active_clickhouse_zones.keepers.insert(new_zone_id); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.keepers.len(), 5); assert_eq!(*new_config.keepers.get(&new_zone_id).unwrap(), KeeperId(6)); assert_eq!(new_config.max_used_keeper_id, 6.into()); @@ -712,7 +726,7 @@ pub mod test { let (n_keeper_zones, n_server_zones, n_keepers, n_servers) = (5, 2, 4, 2); - let (active_clickhouse_zones, parent_config) = initial_config( + let (mut active_clickhouse_zones, parent_config) = initial_config( n_keeper_zones, n_server_zones, n_keepers, @@ -727,14 +741,13 @@ pub mod test { let mut allocator = ClickhouseAllocator { log: logctx.log.clone(), - active_clickhouse_zones, parent_config, inventory, }; // First run the planner to add a 5th keeper to our config assert_eq!(allocator.parent_config.keepers.len(), 4); - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.keepers.len(), 5); // Find the zone for our new keeper and expunge it before it is @@ -749,12 +762,12 @@ pub mod test { .map(|(zone_id, _)| *zone_id) .unwrap(); allocator.parent_config = new_config; - allocator.active_clickhouse_zones.keepers.remove(&zone_to_expunge); + active_clickhouse_zones.keepers.remove(&zone_to_expunge); // Bump the inventory commit index so we guarantee we perform the keeper // checks allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.keepers.len(), 4); assert!(!new_config.keepers.contains_key(&zone_to_expunge)); @@ -770,7 +783,7 @@ pub mod test { let (n_keeper_zones, n_server_zones, n_keepers, n_servers) = (3, 5, 3, 2); - let (active_clickhouse_zones, parent_config) = initial_config( + let (mut active_clickhouse_zones, parent_config) = initial_config( n_keeper_zones, n_server_zones, n_keepers, @@ -785,7 +798,6 @@ pub mod test { let mut allocator = ClickhouseAllocator { log: logctx.log.clone(), - active_clickhouse_zones, parent_config, inventory, }; @@ -793,12 +805,12 @@ pub mod test { let zone_to_expunge = *allocator.parent_config.servers.keys().next().unwrap(); - allocator.active_clickhouse_zones.servers.remove(&zone_to_expunge); + active_clickhouse_zones.servers.remove(&zone_to_expunge); // After running the planner we should see 4 servers: // Start with 2, expunge 1, add 3 to reach the number of zones we have. assert_eq!(allocator.parent_config.servers.len(), 2); - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.servers.len(), 4); assert_eq!(new_config.max_used_server_id, 5.into()); assert_eq!(new_config.generation, Generation::from_u32(2)); @@ -810,11 +822,11 @@ pub mod test { // We can add a new keeper and server at the same time let new_keeper_zone = OmicronZoneUuid::new_v4(); let new_server_id = OmicronZoneUuid::new_v4(); - allocator.active_clickhouse_zones.keepers.insert(new_keeper_zone); - allocator.active_clickhouse_zones.servers.insert(new_server_id); + active_clickhouse_zones.keepers.insert(new_keeper_zone); + active_clickhouse_zones.servers.insert(new_server_id); allocator.parent_config = new_config; allocator.inventory.as_mut().unwrap().leader_committed_log_index += 1; - let new_config = allocator.plan().unwrap(); + let new_config = allocator.plan(&active_clickhouse_zones).unwrap(); assert_eq!(new_config.generation, Generation::from_u32(3)); assert_eq!(new_config.max_used_server_id, 6.into()); assert_eq!(new_config.max_used_keeper_id, 4.into()); @@ -850,7 +862,6 @@ pub mod test { let allocator = ClickhouseAllocator { log: logctx.log.clone(), - active_clickhouse_zones, parent_config, inventory, }; @@ -858,7 +869,7 @@ pub mod test { // We expect to get an error back. This can be used by higher level // software to trigger alerts, etc... In practice the `BlueprintBuilder` // should not change it's config when it receives an error. - assert!(allocator.plan().is_err()); + assert!(allocator.plan(&active_clickhouse_zones).is_err()); logctx.cleanup_successful(); } diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/external_networking.rs b/nexus/reconfigurator/planning/src/blueprint_builder/external_networking.rs index 9cab099fcb..a5c5dd7864 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/external_networking.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/external_networking.rs @@ -3,6 +3,7 @@ // file, You can obtain one at https://mozilla.org/MPL/2.0/. use super::Error; +use anyhow::anyhow; use anyhow::bail; use debug_ignore::DebugIgnore; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; @@ -371,12 +372,18 @@ impl<'a> BuilderExternalNetworking<'a> { /// which could otherwise only be added via RSS. /// /// TODO-cleanup: Remove when external DNS addresses are in the policy. - #[cfg(test)] - pub fn add_external_dns_ip(&mut self, addr: IpAddr) { - assert!( - self.available_external_dns_ips.insert(addr), - "duplicate external DNS IP address" - ); + pub(crate) fn add_external_dns_ip( + &mut self, + addr: IpAddr, + ) -> Result<(), Error> { + if self.available_external_dns_ips.contains(&addr) { + return Err(Error::Planner(anyhow!( + "external DNS IP address already in use: {addr}" + ))); + } + + self.available_external_dns_ips.insert(addr); + Ok(()) } } diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/internal_dns.rs b/nexus/reconfigurator/planning/src/blueprint_builder/internal_dns.rs index 08ff02e997..56fa39374d 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/internal_dns.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/internal_dns.rs @@ -100,7 +100,7 @@ impl DnsSubnetAllocator { pub mod test { use super::*; use crate::blueprint_builder::test::verify_blueprint; - use crate::example::ExampleSystem; + use crate::example::ExampleSystemBuilder; use nexus_types::deployment::BlueprintZoneFilter; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_test_utils::dev::test_setup_log; @@ -115,9 +115,10 @@ pub mod test { assert!(INTERNAL_DNS_REDUNDANCY > 1); // Use our example system to create a blueprint and input. - let mut example = - ExampleSystem::new(&logctx.log, TEST_NAME, INTERNAL_DNS_REDUNDANCY); - let blueprint1 = &mut example.blueprint; + let (example, mut blueprint1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nsleds(INTERNAL_DNS_REDUNDANCY) + .build(); // `ExampleSystem` adds an internal DNS server to every sled. Manually // prune out all but the first of them to give us space to add more. @@ -127,7 +128,7 @@ pub mod test { let npruned = blueprint1.blueprint_zones.len() - 1; assert!(npruned > 0); - verify_blueprint(blueprint1); + verify_blueprint(&blueprint1); // Create an allocator. let mut allocator = DnsSubnetAllocator::new( diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/mod.rs b/nexus/reconfigurator/planning/src/blueprint_builder/mod.rs index b84bef6426..725835f4ae 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/mod.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/mod.rs @@ -11,3 +11,4 @@ mod internal_dns; mod zones; pub use builder::*; +pub use clickhouse::{ClickhouseAllocator, ClickhouseZonesThatShouldBeRunning}; diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs b/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs index 593c57c331..25db378ee7 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/zones.rs @@ -241,11 +241,8 @@ mod tests { use omicron_uuid_kinds::ZpoolUuid; use crate::{ - blueprint_builder::{ - test::{verify_blueprint, DEFAULT_N_SLEDS}, - BlueprintBuilder, Ensure, - }, - example::ExampleSystem, + blueprint_builder::{test::verify_blueprint, BlueprintBuilder, Ensure}, + example::ExampleSystemBuilder, }; use super::*; @@ -255,9 +252,8 @@ mod tests { fn test_builder_zones() { static TEST_NAME: &str = "blueprint_test_builder_zones"; let logctx = test_setup_log(TEST_NAME); - let mut example = - ExampleSystem::new(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); - let blueprint_initial = example.blueprint; + let (mut example, blueprint_initial) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).build(); // Add a completely bare sled to the input. let (new_sled_id, input2) = { diff --git a/nexus/reconfigurator/planning/src/example.rs b/nexus/reconfigurator/planning/src/example.rs index 3104e64f04..f02864d23a 100644 --- a/nexus/reconfigurator/planning/src/example.rs +++ b/nexus/reconfigurator/planning/src/example.rs @@ -4,23 +4,29 @@ //! Example blueprints +use std::net::IpAddr; +use std::net::Ipv4Addr; + use crate::blueprint_builder::BlueprintBuilder; use crate::system::SledBuilder; use crate::system::SystemDescription; use nexus_types::deployment::Blueprint; use nexus_types::deployment::BlueprintZoneFilter; +use nexus_types::deployment::OmicronZoneNic; use nexus_types::deployment::PlanningInput; use nexus_types::deployment::SledFilter; use nexus_types::inventory::Collection; +use omicron_common::policy::CRUCIBLE_PANTRY_REDUNDANCY; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; +use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::SledKind; +use omicron_uuid_kinds::VnicUuid; use typed_rng::TypedUuidRng; pub struct ExampleSystem { pub system: SystemDescription, pub input: PlanningInput, pub collection: Collection, - pub blueprint: Blueprint, // If we add more types of RNGs than just sleds here, we'll need to // expand this to be similar to BlueprintBuilderRng where a root RNG // creates sub-RNGs. @@ -31,20 +37,211 @@ pub struct ExampleSystem { pub(crate) sled_rng: TypedUuidRng, } -impl ExampleSystem { - pub fn new( - log: &slog::Logger, - test_name: &str, - nsleds: usize, - ) -> ExampleSystem { +/// Returns a collection, planning input, and blueprint describing a pretty +/// simple system. +/// +/// The test name is used as the RNG seed. +pub fn example( + log: &slog::Logger, + test_name: &str, +) -> (Collection, PlanningInput, Blueprint) { + let (example, blueprint) = + ExampleSystemBuilder::new(log, test_name).build(); + (example.collection, example.input, blueprint) +} + +/// A builder for the example system. +#[derive(Debug, Clone)] +pub struct ExampleSystemBuilder { + log: slog::Logger, + test_name: String, + // TODO: Store a Policy struct instead of these fields: + // https://github.com/oxidecomputer/omicron/issues/6803 + nsleds: usize, + ndisks_per_sled: u8, + // None means nsleds + nexus_count: Option, + internal_dns_count: ZoneCount, + external_dns_count: ZoneCount, + crucible_pantry_count: ZoneCount, + create_zones: bool, + create_disks_in_blueprint: bool, +} + +impl ExampleSystemBuilder { + /// The default number of sleds in the example system. + pub const DEFAULT_N_SLEDS: usize = 3; + + /// The default number of external DNS instances in the example system. + /// + /// The default value is picked for backwards compatibility -- we may wish + /// to revisit it in the future. + pub const DEFAULT_EXTERNAL_DNS_COUNT: usize = 0; + + pub fn new(log: &slog::Logger, test_name: &str) -> Self { + Self { + log: log.new(slog::o!("component" => "ExampleSystem", "test_name" => test_name.to_string())), + test_name: test_name.to_string(), + nsleds: Self::DEFAULT_N_SLEDS, + ndisks_per_sled: SledBuilder::DEFAULT_NPOOLS, + nexus_count: None, + internal_dns_count: ZoneCount(INTERNAL_DNS_REDUNDANCY), + external_dns_count: ZoneCount(Self::DEFAULT_EXTERNAL_DNS_COUNT), + crucible_pantry_count: ZoneCount(CRUCIBLE_PANTRY_REDUNDANCY), + create_zones: true, + create_disks_in_blueprint: true, + } + } + + /// Set the number of sleds in the example system. + /// + /// Currently, this value can be anywhere between 0 and 5. (More can be + /// added in the future if necessary.) + pub fn nsleds(mut self, nsleds: usize) -> Self { + self.nsleds = nsleds; + self + } + + /// Set the number of disks per sled in the example system. + /// + /// The default value is [`SledBuilder::DEFAULT_NPOOLS`]. A value of 0 is + /// permitted. + /// + /// If [`Self::create_zones`] is set to `false`, this is ignored. + pub fn ndisks_per_sled(mut self, ndisks_per_sled: u8) -> Self { + self.ndisks_per_sled = ndisks_per_sled; + self + } + + /// Set the number of Nexus instances in the example system. + /// + /// The default value is the same as the number of sleds (i.e. one Nexus + /// instance per sled). A value of 0 is permitted. + /// + /// If [`Self::create_zones`] is set to `false`, this is ignored. + pub fn nexus_count(mut self, nexus_count: usize) -> Self { + self.nexus_count = Some(ZoneCount(nexus_count)); + self + } + + /// Set the number of internal DNS instances in the example system. + /// + /// The default value is [`INTERNAL_DNS_REDUNDANCY`]. A value anywhere + /// between 0 and [`INTERNAL_DNS_REDUNDANCY`], inclusive, is permitted. + /// + /// If [`Self::create_zones`] is set to `false`, this is ignored. + pub fn internal_dns_count( + mut self, + internal_dns_count: usize, + ) -> anyhow::Result { + if internal_dns_count > INTERNAL_DNS_REDUNDANCY { + anyhow::bail!( + "internal_dns_count {} is greater than INTERNAL_DNS_REDUNDANCY {}", + internal_dns_count, + INTERNAL_DNS_REDUNDANCY, + ); + } + self.internal_dns_count = ZoneCount(internal_dns_count); + Ok(self) + } + + /// Set the number of external DNS instances in the example system. + /// + /// The default value is [`Self::DEFAULT_EXTERNAL_DNS_COUNT`]. A value + /// anywhere between 0 and 30, inclusive, is permitted. (The limit of 30 is + /// primarily to simplify the implementation.) + /// + /// Each DNS server is assigned an address in the 10.x.x.x range. + pub fn external_dns_count( + mut self, + external_dns_count: usize, + ) -> anyhow::Result { + if external_dns_count > 30 { + anyhow::bail!( + "external_dns_count {} is greater than 30", + external_dns_count, + ); + } + self.external_dns_count = ZoneCount(external_dns_count); + Ok(self) + } + + /// Set the number of Crucible pantry instances in the example system. + /// + /// If [`Self::create_zones`] is set to `false`, this is ignored. + pub fn crucible_pantry_count( + mut self, + crucible_pantry_count: usize, + ) -> Self { + self.crucible_pantry_count = ZoneCount(crucible_pantry_count); + self + } + + /// Create zones in the example system. + /// + /// The default is `true`. + pub fn create_zones(mut self, create_zones: bool) -> Self { + self.create_zones = create_zones; + self + } + + /// Create disks in the blueprint. + /// + /// The default is `true`. + /// + /// If [`Self::ndisks_per_sled`] is set to 0, then this is implied: if no + /// disks are created, then the blueprint won't have any disks. + pub fn create_disks_in_blueprint(mut self, create: bool) -> Self { + self.create_disks_in_blueprint = create; + self + } + + fn get_nexus_zones(&self) -> ZoneCount { + self.nexus_count.unwrap_or(ZoneCount(self.nsleds)) + } + + /// Create a new example system with the given modifications. + /// + /// Return the system, and the initial blueprint that matches it. + pub fn build(&self) -> (ExampleSystem, Blueprint) { + let nexus_count = self.get_nexus_zones(); + + slog::debug!( + &self.log, + "Creating example system"; + "nsleds" => self.nsleds, + "ndisks_per_sled" => self.ndisks_per_sled, + "nexus_count" => nexus_count.0, + "internal_dns_count" => self.internal_dns_count.0, + "external_dns_count" => self.external_dns_count.0, + "crucible_pantry_count" => self.crucible_pantry_count.0, + "create_zones" => self.create_zones, + "create_disks_in_blueprint" => self.create_disks_in_blueprint, + ); + let mut system = SystemDescription::new(); - let mut sled_rng = TypedUuidRng::from_seed(test_name, "ExampleSystem"); - let sled_ids: Vec<_> = (0..nsleds).map(|_| sled_rng.next()).collect(); + // Update the system's target counts with the counts. (Note that + // there's no external DNS count.) + system + .target_nexus_zone_count(nexus_count.0) + .target_internal_dns_zone_count(self.internal_dns_count.0) + .target_crucible_pantry_zone_count(self.crucible_pantry_count.0); + let mut sled_rng = + TypedUuidRng::from_seed(&self.test_name, "ExampleSystem"); + let sled_ids: Vec<_> = + (0..self.nsleds).map(|_| sled_rng.next()).collect(); + for sled_id in &sled_ids { - let _ = system.sled(SledBuilder::new().id(*sled_id)).unwrap(); + let _ = system + .sled( + SledBuilder::new() + .id(*sled_id) + .npools(self.ndisks_per_sled), + ) + .unwrap(); } - let input_builder = system + let mut input_builder = system .to_planning_input_builder() .expect("failed to make planning input builder"); let base_input = input_builder.clone().build(); @@ -53,7 +250,7 @@ impl ExampleSystem { let initial_blueprint = BlueprintBuilder::build_empty_with_sleds_seeded( base_input.all_sled_ids(SledFilter::Commissioned), "test suite", - (test_name, "ExampleSystem initial"), + (&self.test_name, "ExampleSystem initial"), ); // Start with an empty collection @@ -64,48 +261,115 @@ impl ExampleSystem { // Now make a blueprint and collection with some zones on each sled. let mut builder = BlueprintBuilder::new_based_on( - log, + &self.log, &initial_blueprint, &base_input, &collection, "test suite", ) .unwrap(); - builder.set_rng_seed((test_name, "ExampleSystem make_zones")); + builder.set_rng_seed((&self.test_name, "ExampleSystem make_zones")); + + // Add as many external IPs as is necessary for external DNS zones. We + // pick addresses in the TEST-NET-2 (RFC 5737) range. + for i in 0..self.external_dns_count.0 { + builder + .add_external_dns_ip(IpAddr::V4(Ipv4Addr::new( + 198, + 51, + 100, + (i + 1) + .try_into() + .expect("external_dns_count is always <= 30"), + ))) + .expect( + "this shouldn't error because provided external IPs \ + are all unique", + ); + } + for (i, (sled_id, sled_resources)) in base_input.all_sled_resources(SledFilter::Commissioned).enumerate() { - let _ = builder.sled_ensure_zone_ntp(sled_id).unwrap(); - let _ = builder - .sled_ensure_zone_multiple_nexus_with_config( - sled_id, - 1, - false, - vec![], - ) - .unwrap(); - if i < INTERNAL_DNS_REDUNDANCY { + if self.create_zones { + let _ = builder.sled_ensure_zone_ntp(sled_id).unwrap(); let _ = builder - .sled_ensure_zone_multiple_internal_dns(sled_id, 1) + .sled_ensure_zone_multiple_nexus_with_config( + sled_id, + nexus_count.on(i, self.nsleds), + false, + vec![], + ) + .unwrap(); + if i == 0 { + let _ = builder + .sled_ensure_zone_multiple_clickhouse(sled_id, 1); + } + let _ = builder + .sled_ensure_zone_multiple_internal_dns( + sled_id, + self.internal_dns_count.on(i, self.nsleds), + ) .unwrap(); - } - let _ = builder.sled_ensure_disks(sled_id, sled_resources).unwrap(); - for pool_name in sled_resources.zpools.keys() { let _ = builder - .sled_ensure_zone_crucible(sled_id, *pool_name) + .sled_ensure_zone_multiple_external_dns( + sled_id, + self.external_dns_count.on(i, self.nsleds), + ) .unwrap(); + let _ = builder + .sled_ensure_zone_multiple_crucible_pantry( + sled_id, + self.crucible_pantry_count.on(i, self.nsleds), + ) + .unwrap(); + } + if self.create_disks_in_blueprint { + let _ = + builder.sled_ensure_disks(sled_id, sled_resources).unwrap(); + } + if self.create_zones { + for pool_name in sled_resources.zpools.keys() { + let _ = builder + .sled_ensure_zone_crucible(sled_id, *pool_name) + .unwrap(); + } } } let blueprint = builder.build(); - let mut builder = - system.to_collection_builder().expect("failed to build collection"); - builder.set_rng_seed((test_name, "ExampleSystem collection")); + for sled_id in blueprint.sleds() { + let Some(zones) = blueprint.blueprint_zones.get(&sled_id) else { + continue; + }; + for zone in zones.zones.iter() { + let service_id = zone.id; + if let Some((external_ip, nic)) = + zone.zone_type.external_networking() + { + input_builder + .add_omicron_zone_external_ip(service_id, external_ip) + .expect("failed to add Omicron zone external IP"); + input_builder + .add_omicron_zone_nic( + service_id, + OmicronZoneNic { + // TODO-cleanup use `TypedUuid` everywhere + id: VnicUuid::from_untyped_uuid(nic.id), + mac: nic.mac, + ip: nic.ip, + slot: nic.slot, + primary: nic.primary, + }, + ) + .expect("failed to add Omicron zone NIC"); + } + } + } for (sled_id, zones) in &blueprint.blueprint_zones { - builder - .found_sled_omicron_zones( - "fake sled agent", + system + .sled_set_omicron_zones( *sled_id, zones.to_omicron_zones_config( BlueprintZoneFilter::ShouldBeRunning, @@ -114,29 +378,202 @@ impl ExampleSystem { .unwrap(); } - ExampleSystem { + let mut builder = + system.to_collection_builder().expect("failed to build collection"); + builder.set_rng_seed((&self.test_name, "ExampleSystem collection")); + + // The blueprint evolves separately from the system -- so it's returned + // as a separate value. + let example = ExampleSystem { system, input: input_builder.build(), collection: builder.build(), - blueprint, sled_rng, - } + }; + (example, blueprint) } } -/// Returns a collection, planning input, and blueprint describing a pretty -/// simple system. -/// -/// The test name is used as the RNG seed. -/// -/// `n_sleds` is the number of sleds supported. Currently, this value can -/// be anywhere between 0 and 5. (More can be added in the future if -/// necessary.) -pub fn example( - log: &slog::Logger, - test_name: &str, - nsleds: usize, -) -> (Collection, PlanningInput, Blueprint) { - let example = ExampleSystem::new(log, test_name, nsleds); - (example.collection, example.input, example.blueprint) +// A little wrapper to try and avoid having an `on` function which takes 3 +// usize parameters. +#[derive(Clone, Copy, Debug)] +struct ZoneCount(usize); + +impl ZoneCount { + fn on(self, sled_id: usize, total_sleds: usize) -> usize { + // Spread instances out as evenly as possible. If there are 5 sleds and 3 + // instances, we want to spread them out as 2, 2, 1. + let div = self.0 / total_sleds; + let rem = self.0 % total_sleds; + div + if sled_id < rem { 1 } else { 0 } + } +} + +#[cfg(test)] +mod tests { + use chrono::{NaiveDateTime, TimeZone, Utc}; + use nexus_sled_agent_shared::inventory::{OmicronZoneConfig, ZoneKind}; + use nexus_types::deployment::BlueprintZoneConfig; + use omicron_test_utils::dev::test_setup_log; + + use super::*; + + #[test] + fn instances_on_examples() { + assert_eq!(ZoneCount(3).on(0, 5), 1); + assert_eq!(ZoneCount(3).on(1, 5), 1); + assert_eq!(ZoneCount(3).on(2, 5), 1); + assert_eq!(ZoneCount(3).on(3, 5), 0); + assert_eq!(ZoneCount(3).on(4, 5), 0); + + assert_eq!(ZoneCount(5).on(0, 5), 1); + assert_eq!(ZoneCount(5).on(1, 5), 1); + assert_eq!(ZoneCount(5).on(2, 5), 1); + assert_eq!(ZoneCount(5).on(3, 5), 1); + assert_eq!(ZoneCount(5).on(4, 5), 1); + + assert_eq!(ZoneCount(7).on(0, 5), 2); + assert_eq!(ZoneCount(7).on(1, 5), 2); + assert_eq!(ZoneCount(7).on(2, 5), 1); + assert_eq!(ZoneCount(6).on(3, 5), 1); + assert_eq!(ZoneCount(6).on(4, 5), 1); + } + + #[test] + fn builder_zone_counts() { + static TEST_NAME: &str = "example_builder_zone_counts"; + let logctx = test_setup_log(TEST_NAME); + + let (example, mut blueprint) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nsleds(5) + .nexus_count(6) + .crucible_pantry_count(5) + .internal_dns_count(2) + .unwrap() + .external_dns_count(10) + .unwrap() + .build(); + + // Define a time_created for consistent output across runs. + blueprint.time_created = + Utc.from_utc_datetime(&NaiveDateTime::UNIX_EPOCH); + + expectorate::assert_contents( + "tests/output/example_builder_zone_counts_blueprint.txt", + &blueprint.display().to_string(), + ); + + // Check that the system's target counts are set correctly. + assert_eq!(example.system.get_target_nexus_zone_count(), 6); + assert_eq!(example.system.get_target_internal_dns_zone_count(), 2); + assert_eq!(example.system.get_target_crucible_pantry_zone_count(), 5); + + // Check that the right number of zones are present in both the + // blueprint and in the collection. + let nexus_zones = blueprint_zones_of_kind(&blueprint, ZoneKind::Nexus); + assert_eq!( + nexus_zones.len(), + 6, + "expected 6 Nexus zones in blueprint, got {}: {:#?}", + nexus_zones.len(), + nexus_zones, + ); + let nexus_zones = + collection_zones_of_kind(&example.collection, ZoneKind::Nexus); + assert_eq!( + nexus_zones.len(), + 6, + "expected 6 Nexus zones in collection, got {}: {:#?}", + nexus_zones.len(), + nexus_zones, + ); + + let internal_dns_zones = + blueprint_zones_of_kind(&blueprint, ZoneKind::InternalDns); + assert_eq!( + internal_dns_zones.len(), + 2, + "expected 2 internal DNS zones in blueprint, got {}: {:#?}", + internal_dns_zones.len(), + internal_dns_zones, + ); + let internal_dns_zones = collection_zones_of_kind( + &example.collection, + ZoneKind::InternalDns, + ); + assert_eq!( + internal_dns_zones.len(), + 2, + "expected 2 internal DNS zones in collection, got {}: {:#?}", + internal_dns_zones.len(), + internal_dns_zones, + ); + + let external_dns_zones = + blueprint_zones_of_kind(&blueprint, ZoneKind::ExternalDns); + assert_eq!( + external_dns_zones.len(), + 10, + "expected 10 external DNS zones in blueprint, got {}: {:#?}", + external_dns_zones.len(), + external_dns_zones, + ); + let external_dns_zones = collection_zones_of_kind( + &example.collection, + ZoneKind::ExternalDns, + ); + assert_eq!( + external_dns_zones.len(), + 10, + "expected 10 external DNS zones in collection, got {}: {:#?}", + external_dns_zones.len(), + external_dns_zones, + ); + + let crucible_pantry_zones = + blueprint_zones_of_kind(&blueprint, ZoneKind::CruciblePantry); + assert_eq!( + crucible_pantry_zones.len(), + 5, + "expected 5 Crucible pantry zones in blueprint, got {}: {:#?}", + crucible_pantry_zones.len(), + crucible_pantry_zones, + ); + let crucible_pantry_zones = collection_zones_of_kind( + &example.collection, + ZoneKind::CruciblePantry, + ); + assert_eq!( + crucible_pantry_zones.len(), + 5, + "expected 5 Crucible pantry zones in collection, got {}: {:#?}", + crucible_pantry_zones.len(), + crucible_pantry_zones, + ); + + logctx.cleanup_successful(); + } + + fn blueprint_zones_of_kind( + blueprint: &Blueprint, + kind: ZoneKind, + ) -> Vec<&BlueprintZoneConfig> { + blueprint + .all_omicron_zones(BlueprintZoneFilter::All) + .filter_map(|(_, zone)| { + (zone.zone_type.kind() == kind).then_some(zone) + }) + .collect() + } + + fn collection_zones_of_kind( + collection: &Collection, + kind: ZoneKind, + ) -> Vec<&OmicronZoneConfig> { + collection + .all_omicron_zones() + .filter(|zone| zone.zone_type.kind() == kind) + .collect() + } } diff --git a/nexus/reconfigurator/planning/src/planner.rs b/nexus/reconfigurator/planning/src/planner.rs index f02d8ec3de..145be867c6 100644 --- a/nexus/reconfigurator/planning/src/planner.rs +++ b/nexus/reconfigurator/planning/src/planner.rs @@ -290,10 +290,14 @@ impl<'a> Planner<'a> { // problem here. let has_ntp_inventory = self .inventory - .omicron_zones + .sled_agents .get(&sled_id) - .map(|sled_zones| { - sled_zones.zones.zones.iter().any(|z| z.zone_type.is_ntp()) + .map(|sled_agent| { + sled_agent + .omicron_zones + .zones + .iter() + .any(|z| z.zone_type.is_ntp()) }) .unwrap_or(false); if !has_ntp_inventory { @@ -353,10 +357,15 @@ impl<'a> Planner<'a> { for zone_kind in [ DiscretionaryOmicronZone::BoundaryNtp, + DiscretionaryOmicronZone::Clickhouse, + DiscretionaryOmicronZone::ClickhouseKeeper, + DiscretionaryOmicronZone::ClickhouseServer, DiscretionaryOmicronZone::CockroachDb, + DiscretionaryOmicronZone::CruciblePantry, DiscretionaryOmicronZone::InternalDns, DiscretionaryOmicronZone::ExternalDns, DiscretionaryOmicronZone::Nexus, + DiscretionaryOmicronZone::Oximeter, ] { let num_zones_to_add = self.num_additional_zones_needed(zone_kind); if num_zones_to_add == 0 { @@ -431,9 +440,21 @@ impl<'a> Planner<'a> { DiscretionaryOmicronZone::BoundaryNtp => { self.input.target_boundary_ntp_zone_count() } + DiscretionaryOmicronZone::Clickhouse => { + self.input.target_clickhouse_zone_count() + } + DiscretionaryOmicronZone::ClickhouseKeeper => { + self.input.target_clickhouse_keeper_zone_count() + } + DiscretionaryOmicronZone::ClickhouseServer => { + self.input.target_clickhouse_server_zone_count() + } DiscretionaryOmicronZone::CockroachDb => { self.input.target_cockroachdb_zone_count() } + DiscretionaryOmicronZone::CruciblePantry => { + self.input.target_crucible_pantry_zone_count() + } DiscretionaryOmicronZone::InternalDns => { self.input.target_internal_dns_zone_count() } @@ -445,6 +466,9 @@ impl<'a> Planner<'a> { DiscretionaryOmicronZone::Nexus => { self.input.target_nexus_zone_count() } + DiscretionaryOmicronZone::Oximeter => { + self.input.target_oximeter_zone_count() + } }; // TODO-correctness What should we do if we have _too many_ @@ -518,12 +542,36 @@ impl<'a> Planner<'a> { DiscretionaryOmicronZone::BoundaryNtp => self .blueprint .sled_promote_internal_ntp_to_boundary_ntp(sled_id)?, + DiscretionaryOmicronZone::Clickhouse => { + self.blueprint.sled_ensure_zone_multiple_clickhouse( + sled_id, + new_total_zone_count, + )? + } + DiscretionaryOmicronZone::ClickhouseKeeper => { + self.blueprint.sled_ensure_zone_multiple_clickhouse_keeper( + sled_id, + new_total_zone_count, + )? + } + DiscretionaryOmicronZone::ClickhouseServer => { + self.blueprint.sled_ensure_zone_multiple_clickhouse_server( + sled_id, + new_total_zone_count, + )? + } DiscretionaryOmicronZone::CockroachDb => { self.blueprint.sled_ensure_zone_multiple_cockroachdb( sled_id, new_total_zone_count, )? } + DiscretionaryOmicronZone::CruciblePantry => { + self.blueprint.sled_ensure_zone_multiple_crucible_pantry( + sled_id, + new_total_zone_count, + )? + } DiscretionaryOmicronZone::InternalDns => { self.blueprint.sled_ensure_zone_multiple_internal_dns( sled_id, @@ -542,6 +590,12 @@ impl<'a> Planner<'a> { new_total_zone_count, )? } + DiscretionaryOmicronZone::Oximeter => { + self.blueprint.sled_ensure_zone_multiple_oximeter( + sled_id, + new_total_zone_count, + )? + } }; match result { EnsureMultiple::Changed { added, removed } => { @@ -695,6 +749,7 @@ fn sled_needs_all_zones_expunged( pub(crate) fn zone_needs_expungement( sled_details: &SledDetails, zone_config: &BlueprintZoneConfig, + clickhouse_cluster_enabled: bool, ) -> Option { // Should we expunge the zone because the sled is gone? if let Some(reason) = @@ -719,6 +774,16 @@ pub(crate) fn zone_needs_expungement( } }; + // Should we expunge the zone because clickhouse clusters are no longer + // enabled via policy? + if !clickhouse_cluster_enabled { + if zone_config.zone_type.is_clickhouse_keeper() + || zone_config.zone_type.is_clickhouse_server() + { + return Some(ZoneExpungeReason::ClickhouseClusterDisabled); + } + } + None } @@ -731,6 +796,7 @@ pub(crate) enum ZoneExpungeReason { DiskExpunged, SledDecommissioned, SledExpunged, + ClickhouseClusterDisabled, } #[cfg(test)] @@ -738,43 +804,42 @@ mod test { use super::Planner; use crate::blueprint_builder::test::assert_planning_makes_no_changes; use crate::blueprint_builder::test::verify_blueprint; - use crate::blueprint_builder::test::DEFAULT_N_SLEDS; use crate::blueprint_builder::BlueprintBuilder; use crate::blueprint_builder::EnsureMultiple; use crate::example::example; - use crate::example::ExampleSystem; + use crate::example::ExampleSystemBuilder; use crate::system::SledBuilder; use chrono::NaiveDateTime; use chrono::TimeZone; use chrono::Utc; + use clickhouse_admin_types::ClickhouseKeeperClusterMembership; + use clickhouse_admin_types::KeeperId; use expectorate::assert_contents; - use nexus_inventory::now_db_precision; use nexus_sled_agent_shared::inventory::ZoneKind; use nexus_types::deployment::blueprint_zone_type; use nexus_types::deployment::BlueprintDiff; use nexus_types::deployment::BlueprintZoneDisposition; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::BlueprintZoneType; + use nexus_types::deployment::ClickhousePolicy; use nexus_types::deployment::CockroachDbClusterVersion; use nexus_types::deployment::CockroachDbPreserveDowngrade; use nexus_types::deployment::CockroachDbSettings; - use nexus_types::deployment::OmicronZoneNetworkResources; use nexus_types::deployment::SledDisk; use nexus_types::external_api::views::PhysicalDiskPolicy; use nexus_types::external_api::views::PhysicalDiskState; use nexus_types::external_api::views::SledPolicy; use nexus_types::external_api::views::SledProvisionPolicy; use nexus_types::external_api::views::SledState; - use nexus_types::inventory::OmicronZonesFound; use omicron_common::api::external::Generation; use omicron_common::disk::DiskIdentity; + use omicron_common::policy::CRUCIBLE_PANTRY_REDUNDANCY; use omicron_test_utils::dev::test_setup_log; - use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; + use std::collections::BTreeSet; use std::collections::HashMap; - use std::mem; use std::net::IpAddr; use typed_rng::TypedUuidRng; @@ -785,10 +850,9 @@ mod test { let logctx = test_setup_log(TEST_NAME); // Use our example system. - let mut example = - ExampleSystem::new(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); - let blueprint1 = &example.blueprint; - verify_blueprint(blueprint1); + let (mut example, blueprint1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).build(); + verify_blueprint(&blueprint1); println!("{}", blueprint1.display()); @@ -797,7 +861,7 @@ mod test { // fix. let blueprint2 = Planner::new_based_on( logctx.log.clone(), - blueprint1, + &blueprint1, &example.input, "no-op?", &example.collection, @@ -807,7 +871,7 @@ mod test { .plan() .expect("failed to plan"); - let diff = blueprint2.diff_since_blueprint(blueprint1); + let diff = blueprint2.diff_since_blueprint(&blueprint1); println!("1 -> 2 (expected no changes):\n{}", diff.display()); assert_eq!(diff.sleds_added.len(), 0); assert_eq!(diff.sleds_removed.len(), 0); @@ -884,25 +948,24 @@ mod test { verify_blueprint(&blueprint4); // Now update the inventory to have the requested NTP zone. - let mut collection = example.collection.clone(); - assert!(collection - .omicron_zones - .insert( + // + // TODO: mutating example.system doesn't automatically update + // example.collection -- this should be addressed via API improvements. + example + .system + .sled_set_omicron_zones( new_sled_id, - OmicronZonesFound { - time_collected: now_db_precision(), - source: String::from("test suite"), - sled_id: new_sled_id, - zones: blueprint4 - .blueprint_zones - .get(&new_sled_id) - .expect("blueprint should contain zones for new sled") - .to_omicron_zones_config( - BlueprintZoneFilter::ShouldBeRunning - ) - } + blueprint4 + .blueprint_zones + .get(&new_sled_id) + .expect("blueprint should contain zones for new sled") + .to_omicron_zones_config( + BlueprintZoneFilter::ShouldBeRunning, + ), ) - .is_none()); + .unwrap(); + let collection = + example.system.to_collection_builder().unwrap().build(); // Check that the next step is to add Crucible zones let blueprint5 = Planner::new_based_on( @@ -964,63 +1027,16 @@ mod test { static TEST_NAME: &str = "planner_add_multiple_nexus_to_one_sled"; let logctx = test_setup_log(TEST_NAME); - // Use our example system as a starting point, but strip it down to just - // one sled. - let (sled_id, blueprint1, collection, input) = { - let (mut collection, input, mut blueprint) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); - - // Pick one sled ID to keep and remove the rest. - let mut builder = input.into_builder(); - let keep_sled_id = - builder.sleds().keys().next().copied().expect("no sleds"); - builder.sleds_mut().retain(|&k, _v| keep_sled_id == k); - collection.sled_agents.retain(|&k, _v| keep_sled_id == k); - collection.omicron_zones.retain(|&k, _v| keep_sled_id == k); - - assert_eq!(collection.sled_agents.len(), 1); - assert_eq!(collection.omicron_zones.len(), 1); - blueprint.blueprint_zones.retain(|k, _v| keep_sled_id == *k); - blueprint.blueprint_disks.retain(|k, _v| keep_sled_id == *k); - - // Also remove all the networking resources for the zones we just - // stripped out; i.e., only keep those for `keep_sled_id`. - let mut new_network_resources = OmicronZoneNetworkResources::new(); - let old_network_resources = builder.network_resources_mut(); - for old_ip in old_network_resources.omicron_zone_external_ips() { - if blueprint.all_omicron_zones(BlueprintZoneFilter::All).any( - |(_, zone)| { - zone.zone_type - .external_networking() - .map(|(ip, _nic)| ip.id() == old_ip.ip.id()) - .unwrap_or(false) - }, - ) { - new_network_resources - .add_external_ip(old_ip.zone_id, old_ip.ip) - .expect("copied IP to new input"); - } - } - for old_nic in old_network_resources.omicron_zone_nics() { - if blueprint.all_omicron_zones(BlueprintZoneFilter::All).any( - |(_, zone)| { - zone.zone_type - .external_networking() - .map(|(_ip, nic)| { - nic.id == old_nic.nic.id.into_untyped_uuid() - }) - .unwrap_or(false) - }, - ) { - new_network_resources - .add_nic(old_nic.zone_id, old_nic.nic) - .expect("copied NIC to new input"); - } - } - mem::swap(old_network_resources, &mut &mut new_network_resources); - - (keep_sled_id, blueprint, collection, builder.build()) - }; + // Use our example system with one sled and one Nexus instance as a + // starting point. + let (example, blueprint1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME) + .nsleds(1) + .nexus_count(1) + .build(); + let sled_id = *example.collection.sled_agents.keys().next().unwrap(); + let input = example.input; + let collection = example.collection; // This blueprint should only have 1 Nexus instance on the one sled we // kept. @@ -1100,8 +1116,7 @@ mod test { let logctx = test_setup_log(TEST_NAME); // Use our example system as a starting point. - let (collection, input, blueprint1) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + let (collection, input, blueprint1) = example(&logctx.log, TEST_NAME); // This blueprint should only have 3 Nexus zones: one on each sled. assert_eq!(blueprint1.blueprint_zones.len(), 3); @@ -1183,7 +1198,7 @@ mod test { // Use our example system as a starting point. let (collection, input, mut blueprint1) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + example(&logctx.log, TEST_NAME); // This blueprint should have exactly 3 internal DNS zones: one on each // sled. @@ -1297,8 +1312,7 @@ mod test { let logctx = test_setup_log(TEST_NAME); // Use our example system as a starting point. - let (collection, input, blueprint1) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + let (collection, input, blueprint1) = example(&logctx.log, TEST_NAME); // Expunge the first sled we see, which will result in a Nexus external // IP no longer being associated with a running zone, and a new Nexus @@ -1397,8 +1411,7 @@ mod test { let logctx = test_setup_log(TEST_NAME); // Use our example system as a starting point. - let (collection, input, blueprint1) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + let (collection, input, blueprint1) = example(&logctx.log, TEST_NAME); // We should not be able to add any external DNS zones yet, // because we haven't give it any addresses (which currently @@ -1439,7 +1452,7 @@ mod test { .expect("can't parse external DNS IP address") }); for addr in external_dns_ips { - blueprint_builder.add_external_dns_ip(addr); + blueprint_builder.add_external_dns_ip(addr).unwrap(); } // Now we can add external DNS zones. We'll add two to the first @@ -1572,8 +1585,10 @@ mod test { let logctx = test_setup_log(TEST_NAME); // Create an example system with a single sled - let (collection, input, blueprint1) = - example(&logctx.log, TEST_NAME, 1); + let (example, blueprint1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).nsleds(1).build(); + let collection = example.collection; + let input = example.input; let mut builder = input.into_builder(); @@ -1664,8 +1679,10 @@ mod test { let logctx = test_setup_log(TEST_NAME); // Create an example system with a single sled - let (collection, input, blueprint1) = - example(&logctx.log, TEST_NAME, 1); + let (example, blueprint1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).nsleds(1).build(); + let collection = example.collection; + let input = example.input; let mut builder = input.into_builder(); @@ -1762,8 +1779,10 @@ mod test { let logctx = test_setup_log(TEST_NAME); // Create an example system with a single sled - let (collection, input, blueprint1) = - example(&logctx.log, TEST_NAME, 1); + let (example, blueprint1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).nsleds(1).build(); + let collection = example.collection; + let input = example.input; let mut builder = input.into_builder(); @@ -1891,8 +1910,10 @@ mod test { // and decommissioned sleds. (When we add more kinds of // non-provisionable states in the future, we'll have to add more // sleds.) - let (collection, input, mut blueprint1) = - example(&logctx.log, TEST_NAME, 5); + let (example, mut blueprint1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).nsleds(5).build(); + let collection = example.collection; + let input = example.input; // This blueprint should only have 5 Nexus zones: one on each sled. assert_eq!(blueprint1.blueprint_zones.len(), 5); @@ -1955,8 +1976,9 @@ mod test { // * each of those 2 sleds should get exactly 3 new Nexuses builder.policy_mut().target_nexus_zone_count = 9; - // Disable addition of internal DNS zones. + // Disable addition of zone types we're not checking for below. builder.policy_mut().target_internal_dns_zone_count = 0; + builder.policy_mut().target_crucible_pantry_zone_count = 0; let input = builder.build(); let mut blueprint2 = Planner::new_based_on( @@ -2172,8 +2194,7 @@ mod test { let logctx = test_setup_log(TEST_NAME); // Use our example system as a starting point. - let (collection, input, blueprint1) = - example(&logctx.log, TEST_NAME, DEFAULT_N_SLEDS); + let (collection, input, blueprint1) = example(&logctx.log, TEST_NAME); // Expunge one of the sleds. let mut builder = input.into_builder(); @@ -2255,7 +2276,10 @@ mod test { assert_eq!(diff.sleds_added.len(), 0); assert_eq!(diff.sleds_removed.len(), 0); assert_eq!(diff.sleds_modified.len(), 0); - assert_eq!(diff.sleds_unchanged.len(), DEFAULT_N_SLEDS); + assert_eq!( + diff.sleds_unchanged.len(), + ExampleSystemBuilder::DEFAULT_N_SLEDS + ); // Test a no-op planning iteration. assert_planning_makes_no_changes( @@ -2298,7 +2322,10 @@ mod test { assert_eq!(diff.sleds_added.len(), 0); assert_eq!(diff.sleds_removed.len(), 0); assert_eq!(diff.sleds_modified.len(), 0); - assert_eq!(diff.sleds_unchanged.len(), DEFAULT_N_SLEDS); + assert_eq!( + diff.sleds_unchanged.len(), + ExampleSystemBuilder::DEFAULT_N_SLEDS + ); // Test a no-op planning iteration. assert_planning_makes_no_changes( @@ -2316,7 +2343,10 @@ mod test { static TEST_NAME: &str = "planner_ensure_preserve_downgrade_option"; let logctx = test_setup_log(TEST_NAME); - let (collection, input, bp1) = example(&logctx.log, TEST_NAME, 0); + let (example, bp1) = + ExampleSystemBuilder::new(&logctx.log, TEST_NAME).nsleds(0).build(); + let collection = example.collection; + let input = example.input; let mut builder = input.into_builder(); assert!(bp1.cockroachdb_fingerprint.is_empty()); assert_eq!( @@ -2432,4 +2462,765 @@ mod test { logctx.cleanup_successful(); } + + #[test] + fn test_crucible_pantry() { + static TEST_NAME: &str = "test_crucible_pantry"; + let logctx = test_setup_log(TEST_NAME); + + // Use our example system as a starting point. + let (collection, input, blueprint1) = example(&logctx.log, TEST_NAME); + + // We should start with CRUCIBLE_PANTRY_REDUNDANCY pantries spread out + // to at most 1 per sled. Find one of the sleds running one. + let pantry_sleds = blueprint1 + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .filter_map(|(sled_id, zone)| { + zone.zone_type.is_crucible_pantry().then_some(sled_id) + }) + .collect::>(); + assert_eq!( + pantry_sleds.len(), + CRUCIBLE_PANTRY_REDUNDANCY, + "expected {CRUCIBLE_PANTRY_REDUNDANCY} pantries, but found {}", + pantry_sleds.len(), + ); + + // Expunge one of the pantry-hosting sleds and re-plan. The planner + // should immediately replace the zone with one on another + // (non-expunged) sled. + let expunged_sled_id = pantry_sleds[0]; + + let mut input_builder = input.into_builder(); + input_builder + .sleds_mut() + .get_mut(&expunged_sled_id) + .expect("can't find sled") + .policy = SledPolicy::Expunged; + let input = input_builder.build(); + let blueprint2 = Planner::new_based_on( + logctx.log.clone(), + &blueprint1, + &input, + "test_blueprint2", + &collection, + ) + .expect("failed to create planner") + .with_rng_seed((TEST_NAME, "bp2")) + .plan() + .expect("failed to re-plan"); + + let diff = blueprint2.diff_since_blueprint(&blueprint1); + println!("1 -> 2 (expunged sled):\n{}", diff.display()); + assert_eq!( + blueprint2 + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .filter(|(sled_id, zone)| *sled_id != expunged_sled_id + && zone.zone_type.is_crucible_pantry()) + .count(), + CRUCIBLE_PANTRY_REDUNDANCY, + "can't find replacement pantry zone" + ); + + // Test a no-op planning iteration. + assert_planning_makes_no_changes( + &logctx.log, + &blueprint2, + &input, + TEST_NAME, + ); + + logctx.cleanup_successful(); + } + + /// Check that the planner can replace a single-node ClickHouse zone. + /// This is completely distinct from (and much simpler than) the replicated + /// (multi-node) case. + #[test] + fn test_single_node_clickhouse() { + static TEST_NAME: &str = "test_single_node_clickhouse"; + let logctx = test_setup_log(TEST_NAME); + + // Use our example system as a starting point. + let (collection, input, blueprint1) = example(&logctx.log, TEST_NAME); + + // We should start with one ClickHouse zone. Find out which sled it's on. + let clickhouse_sleds = blueprint1 + .all_omicron_zones(BlueprintZoneFilter::All) + .filter_map(|(sled, zone)| { + zone.zone_type.is_clickhouse().then(|| Some(sled)) + }) + .collect::>(); + assert_eq!( + clickhouse_sleds.len(), + 1, + "can't find ClickHouse zone in initial blueprint" + ); + let clickhouse_sled = clickhouse_sleds[0].expect("missing sled id"); + + // Expunge the sled hosting ClickHouse and re-plan. The planner should + // immediately replace the zone with one on another (non-expunged) sled. + let mut input_builder = input.into_builder(); + input_builder + .sleds_mut() + .get_mut(&clickhouse_sled) + .expect("can't find sled") + .policy = SledPolicy::Expunged; + let input = input_builder.build(); + let blueprint2 = Planner::new_based_on( + logctx.log.clone(), + &blueprint1, + &input, + "test_blueprint2", + &collection, + ) + .expect("failed to create planner") + .with_rng_seed((TEST_NAME, "bp2")) + .plan() + .expect("failed to re-plan"); + + let diff = blueprint2.diff_since_blueprint(&blueprint1); + println!("1 -> 2 (expunged sled):\n{}", diff.display()); + assert_eq!( + blueprint2 + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .filter(|(sled, zone)| *sled != clickhouse_sled + && zone.zone_type.is_clickhouse()) + .count(), + 1, + "can't find replacement ClickHouse zone" + ); + + // Test a no-op planning iteration. + assert_planning_makes_no_changes( + &logctx.log, + &blueprint2, + &input, + TEST_NAME, + ); + + logctx.cleanup_successful(); + } + + /// Deploy all keeper nodes server nodes at once for a new cluster. + /// Then add keeper nodes 1 at a time. + #[test] + fn test_plan_deploy_all_clickhouse_cluster_nodes() { + static TEST_NAME: &str = "planner_deploy_all_keeper_nodes"; + let logctx = test_setup_log(TEST_NAME); + let log = logctx.log.clone(); + + // Use our example system. + let (mut collection, input, blueprint1) = example(&log, TEST_NAME); + verify_blueprint(&blueprint1); + + // We shouldn't have a clickhouse cluster config, as we don't have a + // clickhouse policy set yet + assert!(blueprint1.clickhouse_cluster_config.is_none()); + let target_keepers = 3; + let target_servers = 2; + + // Enable clickhouse clusters via policy + let mut input_builder = input.into_builder(); + input_builder.policy_mut().clickhouse_policy = Some(ClickhousePolicy { + deploy_with_standalone: true, + target_servers, + target_keepers, + }); + + // Creating a new blueprint should deploy all the new clickhouse zones + let input = input_builder.build(); + let blueprint2 = Planner::new_based_on( + log.clone(), + &blueprint1, + &input, + "test_blueprint2", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp2")) + .plan() + .expect("plan"); + + // We should see zones for 3 clickhouse keepers, and 2 servers created + let active_zones: Vec<_> = blueprint2 + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .map(|(_, z)| z.clone()) + .collect(); + + let keeper_zone_ids: BTreeSet<_> = active_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_keeper()) + .map(|z| z.id) + .collect(); + let server_zone_ids: BTreeSet<_> = active_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_server()) + .map(|z| z.id) + .collect(); + + assert_eq!(keeper_zone_ids.len(), target_keepers); + assert_eq!(server_zone_ids.len(), target_servers); + + // We should be attempting to allocate all servers and keepers since + // this the initial configuration + { + let clickhouse_cluster_config = + blueprint2.clickhouse_cluster_config.as_ref().unwrap(); + assert_eq!(clickhouse_cluster_config.generation, 2.into()); + assert_eq!( + clickhouse_cluster_config.max_used_keeper_id, + (target_keepers as u64).into() + ); + assert_eq!( + clickhouse_cluster_config.max_used_server_id, + (target_servers as u64).into() + ); + assert_eq!(clickhouse_cluster_config.keepers.len(), target_keepers); + assert_eq!(clickhouse_cluster_config.servers.len(), target_servers); + + // Ensure that the added keepers are in server zones + for zone_id in clickhouse_cluster_config.keepers.keys() { + assert!(keeper_zone_ids.contains(zone_id)); + } + + // Ensure that the added servers are in server zones + for zone_id in clickhouse_cluster_config.servers.keys() { + assert!(server_zone_ids.contains(zone_id)); + } + } + + // Planning again without changing inventory should result in the same + // state + let blueprint3 = Planner::new_based_on( + log.clone(), + &blueprint2, + &input, + "test_blueprint3", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp3")) + .plan() + .expect("plan"); + + assert_eq!( + blueprint2.clickhouse_cluster_config, + blueprint3.clickhouse_cluster_config + ); + + // Updating the inventory to reflect the keepers + // should result in the same state, except for the + // `highest_seen_keeper_leader_committed_log_index` + let (_, keeper_id) = blueprint3 + .clickhouse_cluster_config + .as_ref() + .unwrap() + .keepers + .first_key_value() + .unwrap(); + let membership = ClickhouseKeeperClusterMembership { + queried_keeper: *keeper_id, + leader_committed_log_index: 1, + raft_config: blueprint3 + .clickhouse_cluster_config + .as_ref() + .unwrap() + .keepers + .values() + .cloned() + .collect(), + }; + collection.clickhouse_keeper_cluster_membership.insert(membership); + + let blueprint4 = Planner::new_based_on( + log.clone(), + &blueprint3, + &input, + "test_blueprint4", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp4")) + .plan() + .expect("plan"); + + let bp3_config = blueprint3.clickhouse_cluster_config.as_ref().unwrap(); + let bp4_config = blueprint4.clickhouse_cluster_config.as_ref().unwrap(); + assert_eq!(bp4_config.generation, bp3_config.generation); + assert_eq!( + bp4_config.max_used_keeper_id, + bp3_config.max_used_keeper_id + ); + assert_eq!( + bp4_config.max_used_server_id, + bp3_config.max_used_server_id + ); + assert_eq!(bp4_config.keepers, bp3_config.keepers); + assert_eq!(bp4_config.servers, bp3_config.servers); + assert_eq!( + bp4_config.highest_seen_keeper_leader_committed_log_index, + 1 + ); + + // Let's bump the clickhouse target to 5 via policy so that we can add + // more nodes one at a time. Initial configuration deploys all nodes, + // but reconfigurations may only add or remove one node at a time. + // Enable clickhouse clusters via policy + let target_keepers = 5; + let mut input_builder = input.into_builder(); + input_builder.policy_mut().clickhouse_policy = Some(ClickhousePolicy { + deploy_with_standalone: true, + target_servers, + target_keepers, + }); + let input = input_builder.build(); + let blueprint5 = Planner::new_based_on( + log.clone(), + &blueprint4, + &input, + "test_blueprint5", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp5")) + .plan() + .expect("plan"); + + let active_zones: Vec<_> = blueprint5 + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .map(|(_, z)| z.clone()) + .collect(); + + let new_keeper_zone_ids: BTreeSet<_> = active_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_keeper()) + .map(|z| z.id) + .collect(); + + // We should have allocated 2 new keeper zones + assert_eq!(new_keeper_zone_ids.len(), target_keepers); + assert!(keeper_zone_ids.is_subset(&new_keeper_zone_ids)); + + // We should be trying to provision one new keeper for a keeper zone + let bp4_config = blueprint4.clickhouse_cluster_config.as_ref().unwrap(); + let bp5_config = blueprint5.clickhouse_cluster_config.as_ref().unwrap(); + assert_eq!(bp5_config.generation, bp4_config.generation.next()); + assert_eq!( + bp5_config.max_used_keeper_id, + bp4_config.max_used_keeper_id + 1.into() + ); + assert_eq!( + bp5_config.keepers.len(), + bp5_config.max_used_keeper_id.0 as usize + ); + + // Planning again without updating inventory results in the same `ClickhouseClusterConfig` + let blueprint6 = Planner::new_based_on( + log.clone(), + &blueprint5, + &input, + "test_blueprint6", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp6")) + .plan() + .expect("plan"); + + let bp6_config = blueprint6.clickhouse_cluster_config.as_ref().unwrap(); + assert_eq!(bp5_config, bp6_config); + + // Updating the inventory to include the 4th node should add another + // keeper node + let membership = ClickhouseKeeperClusterMembership { + queried_keeper: *keeper_id, + leader_committed_log_index: 2, + raft_config: blueprint6 + .clickhouse_cluster_config + .as_ref() + .unwrap() + .keepers + .values() + .cloned() + .collect(), + }; + collection.clickhouse_keeper_cluster_membership.insert(membership); + + let blueprint7 = Planner::new_based_on( + log.clone(), + &blueprint6, + &input, + "test_blueprint7", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp7")) + .plan() + .expect("plan"); + + let bp7_config = blueprint7.clickhouse_cluster_config.as_ref().unwrap(); + assert_eq!(bp7_config.generation, bp6_config.generation.next()); + assert_eq!( + bp7_config.max_used_keeper_id, + bp6_config.max_used_keeper_id + 1.into() + ); + assert_eq!( + bp7_config.keepers.len(), + bp7_config.max_used_keeper_id.0 as usize + ); + assert_eq!(bp7_config.keepers.len(), target_keepers); + assert_eq!( + bp7_config.highest_seen_keeper_leader_committed_log_index, + 2 + ); + + // Updating the inventory to reflect the newest keeper node should not + // increase the cluster size since we have reached the target. + let membership = ClickhouseKeeperClusterMembership { + queried_keeper: *keeper_id, + leader_committed_log_index: 3, + raft_config: blueprint7 + .clickhouse_cluster_config + .as_ref() + .unwrap() + .keepers + .values() + .cloned() + .collect(), + }; + collection.clickhouse_keeper_cluster_membership.insert(membership); + let blueprint8 = Planner::new_based_on( + log.clone(), + &blueprint7, + &input, + "test_blueprint8", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp8")) + .plan() + .expect("plan"); + + let bp8_config = blueprint8.clickhouse_cluster_config.as_ref().unwrap(); + assert_eq!(bp8_config.generation, bp7_config.generation); + assert_eq!( + bp8_config.max_used_keeper_id, + bp7_config.max_used_keeper_id + ); + assert_eq!(bp8_config.keepers, bp7_config.keepers); + assert_eq!(bp7_config.keepers.len(), target_keepers); + assert_eq!( + bp8_config.highest_seen_keeper_leader_committed_log_index, + 3 + ); + + logctx.cleanup_successful(); + } + + // Start with an existing clickhouse cluster and expunge a keeper. This + // models what will happen after an RSS deployment with clickhouse policy + // enabled or an existing system already running a clickhouse cluster. + #[test] + fn test_expunge_clickhouse_clusters() { + static TEST_NAME: &str = "planner_expunge_clickhouse_clusters"; + let logctx = test_setup_log(TEST_NAME); + let log = logctx.log.clone(); + + // Use our example system. + let (mut collection, input, blueprint1) = example(&log, TEST_NAME); + + let target_keepers = 3; + let target_servers = 2; + + // Enable clickhouse clusters via policy + let mut input_builder = input.into_builder(); + input_builder.policy_mut().clickhouse_policy = Some(ClickhousePolicy { + deploy_with_standalone: true, + target_servers, + target_keepers, + }); + let input = input_builder.build(); + + // Create a new blueprint to deploy all our clickhouse zones + let mut blueprint2 = Planner::new_based_on( + log.clone(), + &blueprint1, + &input, + "test_blueprint2", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp2")) + .plan() + .expect("plan"); + + // We should see zones for 3 clickhouse keepers, and 2 servers created + let active_zones: Vec<_> = blueprint2 + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .map(|(_, z)| z.clone()) + .collect(); + + let keeper_zone_ids: BTreeSet<_> = active_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_keeper()) + .map(|z| z.id) + .collect(); + let server_zone_ids: BTreeSet<_> = active_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_server()) + .map(|z| z.id) + .collect(); + + assert_eq!(keeper_zone_ids.len(), target_keepers); + assert_eq!(server_zone_ids.len(), target_servers); + + // Directly manipulate the blueprint and inventory so that the + // clickhouse clusters are stable + let config = blueprint2.clickhouse_cluster_config.as_mut().unwrap(); + config.max_used_keeper_id = (target_keepers as u64).into(); + config.keepers = keeper_zone_ids + .iter() + .enumerate() + .map(|(i, zone_id)| (*zone_id, KeeperId(i as u64))) + .collect(); + config.highest_seen_keeper_leader_committed_log_index = 1; + + let raft_config: BTreeSet<_> = + config.keepers.values().cloned().collect(); + + collection.clickhouse_keeper_cluster_membership = config + .keepers + .values() + .map(|keeper_id| ClickhouseKeeperClusterMembership { + queried_keeper: *keeper_id, + leader_committed_log_index: 1, + raft_config: raft_config.clone(), + }) + .collect(); + + // Let's run the planner. The blueprint shouldn't change with regards to + // clickhouse as our inventory reflects our desired state. + let blueprint3 = Planner::new_based_on( + log.clone(), + &blueprint2, + &input, + "test_blueprint3", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp3")) + .plan() + .expect("plan"); + + assert_eq!( + blueprint2.clickhouse_cluster_config, + blueprint3.clickhouse_cluster_config + ); + + // Find the sled containing one of the keeper zones and expunge it + let (sled_id, bp_zone_config) = blueprint3 + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .find(|(_, z)| z.zone_type.is_clickhouse_keeper()) + .unwrap(); + + // What's the keeper id for this expunged zone? + let expunged_keeper_id = blueprint3 + .clickhouse_cluster_config + .as_ref() + .unwrap() + .keepers + .get(&bp_zone_config.id) + .unwrap(); + + // Expunge a keeper zone + let mut builder = input.into_builder(); + builder.sleds_mut().get_mut(&sled_id).unwrap().policy = + SledPolicy::Expunged; + let input = builder.build(); + + let blueprint4 = Planner::new_based_on( + log.clone(), + &blueprint3, + &input, + "test_blueprint4", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp4")) + .plan() + .expect("plan"); + + // The planner should expunge a zone based on the sled being expunged. Since this + // is a clickhouse keeper zone, the clickhouse keeper configuration should change + // to reflect this. + let old_config = blueprint3.clickhouse_cluster_config.as_ref().unwrap(); + let config = blueprint4.clickhouse_cluster_config.as_ref().unwrap(); + assert_eq!(config.generation, old_config.generation.next()); + assert!(!config.keepers.contains_key(&bp_zone_config.id)); + // We've only removed one keeper from our desired state + assert_eq!(config.keepers.len() + 1, old_config.keepers.len()); + // We haven't allocated any new keepers + assert_eq!(config.max_used_keeper_id, old_config.max_used_keeper_id); + + // Planning again will not change the keeper state because we haven't updated the inventory + let blueprint5 = Planner::new_based_on( + log.clone(), + &blueprint4, + &input, + "test_blueprint5", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp5")) + .plan() + .expect("plan"); + + assert_eq!( + blueprint4.clickhouse_cluster_config, + blueprint5.clickhouse_cluster_config + ); + + // Updating the inventory to reflect the removed keeper results in a new one being added + + // Remove the keeper for the expunged zone + collection + .clickhouse_keeper_cluster_membership + .retain(|m| m.queried_keeper != *expunged_keeper_id); + + // Update the inventory on at least one of the remaining nodes. + let mut existing = collection + .clickhouse_keeper_cluster_membership + .pop_first() + .unwrap(); + existing.leader_committed_log_index = 3; + existing.raft_config = config.keepers.values().cloned().collect(); + collection.clickhouse_keeper_cluster_membership.insert(existing); + + let blueprint6 = Planner::new_based_on( + log.clone(), + &blueprint5, + &input, + "test_blueprint6", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp6")) + .plan() + .expect("plan"); + + let old_config = blueprint5.clickhouse_cluster_config.as_ref().unwrap(); + let config = blueprint6.clickhouse_cluster_config.as_ref().unwrap(); + + // Our generation has changed to reflect the added keeper + assert_eq!(config.generation, old_config.generation.next()); + assert!(!config.keepers.contains_key(&bp_zone_config.id)); + // We've only added one keeper from our desired state + // This brings us back up to our target count + assert_eq!(config.keepers.len(), old_config.keepers.len() + 1); + assert_eq!(config.keepers.len(), target_keepers); + // We've allocated one new keeper + assert_eq!( + config.max_used_keeper_id, + old_config.max_used_keeper_id + 1.into() + ); + + logctx.cleanup_successful(); + } + + #[test] + fn test_expunge_all_clickhouse_cluster_zones_after_policy_is_disabled() { + static TEST_NAME: &str = "planner_expunge_all_clickhouse_cluster_zones_after_policy_is_disabled"; + let logctx = test_setup_log(TEST_NAME); + let log = logctx.log.clone(); + + // Use our example system. + let (collection, input, blueprint1) = example(&log, TEST_NAME); + + let target_keepers = 3; + let target_servers = 2; + + // Enable clickhouse clusters via policy + let mut input_builder = input.into_builder(); + input_builder.policy_mut().clickhouse_policy = Some(ClickhousePolicy { + deploy_with_standalone: true, + target_servers, + target_keepers, + }); + let input = input_builder.build(); + + // Create a new blueprint to deploy all our clickhouse zones + let blueprint2 = Planner::new_based_on( + log.clone(), + &blueprint1, + &input, + "test_blueprint2", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp2")) + .plan() + .expect("plan"); + + // We should see zones for 3 clickhouse keepers, and 2 servers created + let active_zones: Vec<_> = blueprint2 + .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning) + .map(|(_, z)| z.clone()) + .collect(); + + let keeper_zone_ids: BTreeSet<_> = active_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_keeper()) + .map(|z| z.id) + .collect(); + let server_zone_ids: BTreeSet<_> = active_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_server()) + .map(|z| z.id) + .collect(); + + assert_eq!(keeper_zone_ids.len(), target_keepers); + assert_eq!(server_zone_ids.len(), target_servers); + + // Disable clickhouse clusters via policy + let mut input_builder = input.into_builder(); + input_builder.policy_mut().clickhouse_policy = None; + let input = input_builder.build(); + + // Create a new blueprint with the disabled policy + let blueprint3 = Planner::new_based_on( + log.clone(), + &blueprint2, + &input, + "test_blueprint3", + &collection, + ) + .expect("created planner") + .with_rng_seed((TEST_NAME, "bp3")) + .plan() + .expect("plan"); + + // All our clickhouse keeper and server zones that we created when we + // enabled our clickhouse policy should be expunged when we disable it. + let expunged_zones: Vec<_> = blueprint3 + .all_omicron_zones(BlueprintZoneFilter::Expunged) + .map(|(_, z)| z.clone()) + .collect(); + + let expunged_keeper_zone_ids: BTreeSet<_> = expunged_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_keeper()) + .map(|z| z.id) + .collect(); + let expunged_server_zone_ids: BTreeSet<_> = expunged_zones + .iter() + .filter(|z| z.zone_type.is_clickhouse_server()) + .map(|z| z.id) + .collect(); + + assert_eq!(keeper_zone_ids, expunged_keeper_zone_ids); + assert_eq!(server_zone_ids, expunged_server_zone_ids); + + logctx.cleanup_successful(); + } } diff --git a/nexus/reconfigurator/planning/src/planner/omicron_zone_placement.rs b/nexus/reconfigurator/planning/src/planner/omicron_zone_placement.rs index 0af0321768..20b908867d 100644 --- a/nexus/reconfigurator/planning/src/planner/omicron_zone_placement.rs +++ b/nexus/reconfigurator/planning/src/planner/omicron_zone_placement.rs @@ -15,11 +15,15 @@ use std::mem; #[cfg_attr(test, derive(test_strategy::Arbitrary))] pub(crate) enum DiscretionaryOmicronZone { BoundaryNtp, + Clickhouse, + ClickhouseKeeper, + ClickhouseServer, CockroachDb, + CruciblePantry, InternalDns, ExternalDns, Nexus, - // TODO expand this enum as we start to place more services + Oximeter, } impl DiscretionaryOmicronZone { @@ -28,19 +32,22 @@ impl DiscretionaryOmicronZone { ) -> Option { match zone_type { BlueprintZoneType::BoundaryNtp(_) => Some(Self::BoundaryNtp), + BlueprintZoneType::Clickhouse(_) => Some(Self::Clickhouse), + BlueprintZoneType::ClickhouseKeeper(_) => { + Some(Self::ClickhouseKeeper) + } + BlueprintZoneType::ClickhouseServer(_) => { + Some(Self::ClickhouseServer) + } BlueprintZoneType::CockroachDb(_) => Some(Self::CockroachDb), + BlueprintZoneType::CruciblePantry(_) => Some(Self::CruciblePantry), BlueprintZoneType::InternalDns(_) => Some(Self::InternalDns), BlueprintZoneType::ExternalDns(_) => Some(Self::ExternalDns), BlueprintZoneType::Nexus(_) => Some(Self::Nexus), - // Zones that we should place but don't yet. - BlueprintZoneType::Clickhouse(_) - | BlueprintZoneType::ClickhouseKeeper(_) - | BlueprintZoneType::ClickhouseServer(_) - | BlueprintZoneType::CruciblePantry(_) - | BlueprintZoneType::Oximeter(_) => None, + BlueprintZoneType::Oximeter(_) => Some(Self::Oximeter), // Zones that get special handling for placement (all sleds get // them, although internal NTP has some interactions with boundary - // NTP that we don't yet handle, so this may change). + // NTP that are handled separately). BlueprintZoneType::Crucible(_) | BlueprintZoneType::InternalNtp(_) => None, } @@ -51,10 +58,19 @@ impl From for ZoneKind { fn from(zone: DiscretionaryOmicronZone) -> Self { match zone { DiscretionaryOmicronZone::BoundaryNtp => Self::BoundaryNtp, + DiscretionaryOmicronZone::Clickhouse => Self::Clickhouse, + DiscretionaryOmicronZone::ClickhouseKeeper => { + Self::ClickhouseKeeper + } + DiscretionaryOmicronZone::ClickhouseServer => { + Self::ClickhouseServer + } DiscretionaryOmicronZone::CockroachDb => Self::CockroachDb, + DiscretionaryOmicronZone::CruciblePantry => Self::CruciblePantry, DiscretionaryOmicronZone::InternalDns => Self::InternalDns, DiscretionaryOmicronZone::ExternalDns => Self::ExternalDns, DiscretionaryOmicronZone::Nexus => Self::Nexus, + DiscretionaryOmicronZone::Oximeter => Self::Oximeter, } } } diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index ccb4d4ab27..4a8c2e8831 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -13,7 +13,9 @@ use nexus_inventory::CollectionBuilder; use nexus_sled_agent_shared::inventory::Baseboard; use nexus_sled_agent_shared::inventory::Inventory; use nexus_sled_agent_shared::inventory::InventoryDisk; +use nexus_sled_agent_shared::inventory::OmicronZonesConfig; use nexus_sled_agent_shared::inventory::SledRole; +use nexus_types::deployment::ClickhousePolicy; use nexus_types::deployment::CockroachDbClusterVersion; use nexus_types::deployment::CockroachDbSettings; use nexus_types::deployment::PlanningInputBuilder; @@ -41,7 +43,6 @@ use omicron_common::disk::DiskIdentity; use omicron_common::disk::DiskVariant; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_common::policy::NEXUS_REDUNDANCY; -use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use std::collections::BTreeMap; @@ -82,11 +83,14 @@ pub struct SystemDescription { target_boundary_ntp_zone_count: usize, target_nexus_zone_count: usize, target_internal_dns_zone_count: usize, + target_oximeter_zone_count: usize, target_cockroachdb_zone_count: usize, target_cockroachdb_cluster_version: CockroachDbClusterVersion, + target_crucible_pantry_zone_count: usize, service_ip_pool_ranges: Vec, internal_dns_version: Generation, external_dns_version: Generation, + clickhouse_policy: Option, } impl SystemDescription { @@ -134,11 +138,13 @@ impl SystemDescription { let target_internal_dns_zone_count = INTERNAL_DNS_REDUNDANCY; // TODO-cleanup These are wrong, but we don't currently set up any - // boundary NTP or CRDB nodes in our fake system, so this prevents - // downstream test issues with the planner thinking our system is out of - // date from the gate. + // of these zones in our fake system, so this prevents downstream test + // issues with the planner thinking our system is out of date from the + // gate. let target_boundary_ntp_zone_count = 0; let target_cockroachdb_zone_count = 0; + let target_oximeter_zone_count = 0; + let target_crucible_pantry_zone_count = 0; let target_cockroachdb_cluster_version = CockroachDbClusterVersion::POLICY; @@ -159,11 +165,14 @@ impl SystemDescription { target_boundary_ntp_zone_count, target_nexus_zone_count, target_internal_dns_zone_count, + target_oximeter_zone_count, target_cockroachdb_zone_count, target_cockroachdb_cluster_version, + target_crucible_pantry_zone_count, service_ip_pool_ranges, internal_dns_version: Generation::new(), external_dns_version: Generation::new(), + clickhouse_policy: None, } } @@ -201,6 +210,34 @@ impl SystemDescription { self } + pub fn get_target_nexus_zone_count(&self) -> usize { + self.target_nexus_zone_count + } + + pub fn target_crucible_pantry_zone_count( + &mut self, + count: usize, + ) -> &mut Self { + self.target_crucible_pantry_zone_count = count; + self + } + + pub fn get_target_crucible_pantry_zone_count(&self) -> usize { + self.target_crucible_pantry_zone_count + } + + pub fn target_internal_dns_zone_count( + &mut self, + count: usize, + ) -> &mut Self { + self.target_internal_dns_zone_count = count; + self + } + + pub fn get_target_internal_dns_zone_count(&self) -> usize { + self.target_internal_dns_zone_count + } + pub fn service_ip_pool_ranges( &mut self, ranges: Vec, @@ -209,6 +246,12 @@ impl SystemDescription { self } + /// Set the clickhouse policy + pub fn clickhouse_policy(&mut self, policy: ClickhousePolicy) -> &mut Self { + self.clickhouse_policy = Some(policy); + self + } + /// Add a sled to the system, as described by a SledBuilder pub fn sled(&mut self, sled: SledBuilder) -> anyhow::Result<&mut Self> { let sled_id = sled.id.unwrap_or_else(SledUuid::new_v4); @@ -252,6 +295,7 @@ impl SystemDescription { sled.unique, sled.hardware, hardware_slot, + sled.omicron_zones, sled.npools, ); self.sleds.insert(sled_id, sled); @@ -292,6 +336,34 @@ impl SystemDescription { Ok(self) } + /// Return true if the system has any sleds in it. + pub fn has_sleds(&self) -> bool { + !self.sleds.is_empty() + } + + /// Set Omicron zones for a sled. + /// + /// The zones will be reported in the collection generated by + /// [`Self::to_collection_builder`]. + /// + /// Returns an error if the sled is not found. + /// + /// # Notes + /// + /// It is okay to call `sled_set_omicron_zones` in ways that wouldn't + /// happen in production, such as to test illegal states. + pub fn sled_set_omicron_zones( + &mut self, + sled_id: SledUuid, + omicron_zones: OmicronZonesConfig, + ) -> anyhow::Result<&mut Self> { + let sled = self.sleds.get_mut(&sled_id).with_context(|| { + format!("attempted to access sled {} not found in system", sled_id) + })?; + sled.inventory_sled_agent.omicron_zones = omicron_zones; + Ok(self) + } + pub fn to_collection_builder(&self) -> anyhow::Result { let collector_label = self .collector @@ -335,10 +407,13 @@ impl SystemDescription { target_boundary_ntp_zone_count: self.target_boundary_ntp_zone_count, target_nexus_zone_count: self.target_nexus_zone_count, target_internal_dns_zone_count: self.target_internal_dns_zone_count, + target_oximeter_zone_count: self.target_oximeter_zone_count, target_cockroachdb_zone_count: self.target_cockroachdb_zone_count, target_cockroachdb_cluster_version: self .target_cockroachdb_cluster_version, - clickhouse_policy: None, + target_crucible_pantry_zone_count: self + .target_crucible_pantry_zone_count, + clickhouse_policy: self.clickhouse_policy.clone(), }; let mut builder = PlanningInputBuilder::new( policy, @@ -375,10 +450,16 @@ pub struct SledBuilder { hardware: SledHardware, hardware_slot: Option, sled_role: SledRole, + omicron_zones: OmicronZonesConfig, npools: u8, } impl SledBuilder { + /// The default number of U.2 (external) pools for a sled. + /// + /// The default is `10` based on the typical value for a Gimlet. + pub const DEFAULT_NPOOLS: u8 = 10; + /// Begin describing a sled to be added to a `SystemDescription` pub fn new() -> Self { SledBuilder { @@ -387,7 +468,12 @@ impl SledBuilder { hardware: SledHardware::Gimlet, hardware_slot: None, sled_role: SledRole::Gimlet, - npools: 10, + omicron_zones: OmicronZonesConfig { + // The initial generation is the one with no zones. + generation: OmicronZonesConfig::INITIAL_GENERATION, + zones: Vec::new(), + }, + npools: Self::DEFAULT_NPOOLS, } } @@ -413,7 +499,7 @@ impl SledBuilder { /// Set the number of U.2 (external) pools this sled should have /// - /// Default is currently `10` based on the typical value for a Gimlet + /// The default is [`Self::DEFAULT_NPOOLS`]. pub fn npools(mut self, npools: u8) -> Self { self.npools = npools; self @@ -467,6 +553,7 @@ struct Sled { impl Sled { /// Create a `Sled` using faked-up information based on a `SledBuilder` + #[allow(clippy::too_many_arguments)] fn new_simulated( sled_id: SledUuid, sled_subnet: Ipv6Subnet, @@ -474,6 +561,7 @@ impl Sled { unique: Option, hardware: SledHardware, hardware_slot: u16, + omicron_zones: OmicronZonesConfig, nzpools: u8, ) -> Sled { use typed_rng::TypedUuidRng; @@ -485,6 +573,10 @@ impl Sled { "SystemSimultatedSled", (sled_id, "ZpoolUuid"), ); + let mut physical_disk_rng = TypedUuidRng::from_seed( + "SystemSimulatedSled", + (sled_id, "PhysicalDiskUuid"), + ); let zpools: BTreeMap<_, _> = (0..nzpools) .map(|_| { let zpool = ZpoolUuid::from(zpool_rng.next()); @@ -494,7 +586,7 @@ impl Sled { serial: format!("serial-{zpool}"), model: String::from("fake-model"), }, - disk_id: PhysicalDiskUuid::new_v4(), + disk_id: physical_disk_rng.next(), policy: PhysicalDiskPolicy::InService, state: PhysicalDiskState::Active, }; @@ -556,6 +648,7 @@ impl Sled { sled_id, usable_hardware_threads: 10, usable_physical_ram: ByteCount::from(1024 * 1024), + omicron_zones, // Populate disks, appearing like a real device. disks: zpools .values() @@ -711,6 +804,7 @@ impl Sled { sled_id, usable_hardware_threads: inv_sled_agent.usable_hardware_threads, usable_physical_ram: inv_sled_agent.usable_physical_ram, + omicron_zones: inv_sled_agent.omicron_zones.clone(), disks: vec![], zpools: vec![], datasets: vec![], diff --git a/nexus/reconfigurator/planning/tests/output/blueprint_builder_initial_diff.txt b/nexus/reconfigurator/planning/tests/output/blueprint_builder_initial_diff.txt index 6eb24e9aa0..1c792a15c2 100644 --- a/nexus/reconfigurator/planning/tests/output/blueprint_builder_initial_diff.txt +++ b/nexus/reconfigurator/planning/tests/output/blueprint_builder_initial_diff.txt @@ -21,22 +21,24 @@ to: blueprint e4aeb3b3-272f-4967-be34-2d34daa46aa1 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 38b047ea-e3de-4859-b8e0-70cac5871446 in service fd00:1122:3344:103::2c - crucible 4644ea0c-0ec3-41be-a356-660308e1c3fc in service fd00:1122:3344:103::2b - crucible 55f4d117-0b9d-4256-a2c0-f46d3ed5fff9 in service fd00:1122:3344:103::24 - crucible 5c6a4628-8831-483b-995f-79b9126c4d04 in service fd00:1122:3344:103::27 - crucible 6a01210c-45ed-41a5-9230-8e05ecf5dd8f in service fd00:1122:3344:103::28 - crucible 7004cab9-dfc0-43ba-92d3-58d4ced66025 in service fd00:1122:3344:103::23 - crucible 79552859-fbd3-43bb-a9d3-6baba25558f8 in service fd00:1122:3344:103::25 - crucible 90696819-9b53-485a-9c65-ca63602e843e in service fd00:1122:3344:103::26 - crucible c99525b3-3680-4df6-9214-2ee3e1020e8b in service fd00:1122:3344:103::29 - crucible f42959d3-9eef-4e3b-b404-6177ce3ec7a1 in service fd00:1122:3344:103::2a - internal_dns 44afce85-3377-4b20-a398-517c1579df4d in service fd00:1122:3344:1::1 - internal_ntp c81c9d4a-36d7-4796-9151-f564d3735152 in service fd00:1122:3344:103::21 - nexus b2573120-9c91-4ed7-8b4f-a7bfe8dbc807 in service fd00:1122:3344:103::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse 44afce85-3377-4b20-a398-517c1579df4d in service fd00:1122:3344:103::23 + crucible 38b047ea-e3de-4859-b8e0-70cac5871446 in service fd00:1122:3344:103::2c + crucible 4644ea0c-0ec3-41be-a356-660308e1c3fc in service fd00:1122:3344:103::2b + crucible 5c6a4628-8831-483b-995f-79b9126c4d04 in service fd00:1122:3344:103::27 + crucible 6a01210c-45ed-41a5-9230-8e05ecf5dd8f in service fd00:1122:3344:103::28 + crucible 79552859-fbd3-43bb-a9d3-6baba25558f8 in service fd00:1122:3344:103::25 + crucible 90696819-9b53-485a-9c65-ca63602e843e in service fd00:1122:3344:103::26 + crucible a9a6a974-8953-4783-b815-da46884f2c02 in service fd00:1122:3344:103::2e + crucible c99525b3-3680-4df6-9214-2ee3e1020e8b in service fd00:1122:3344:103::29 + crucible f42959d3-9eef-4e3b-b404-6177ce3ec7a1 in service fd00:1122:3344:103::2a + crucible fb36b9dc-273a-4bc3-aaa9-19ee4d0ef552 in service fd00:1122:3344:103::2d + crucible_pantry 55f4d117-0b9d-4256-a2c0-f46d3ed5fff9 in service fd00:1122:3344:103::24 + internal_dns 7004cab9-dfc0-43ba-92d3-58d4ced66025 in service fd00:1122:3344:1::1 + internal_ntp c81c9d4a-36d7-4796-9151-f564d3735152 in service fd00:1122:3344:103::21 + nexus b2573120-9c91-4ed7-8b4f-a7bfe8dbc807 in service fd00:1122:3344:103::22 sled 84ac367e-9b03-4e9d-a846-df1a08deee6c (active): @@ -58,22 +60,23 @@ to: blueprint e4aeb3b3-272f-4967-be34-2d34daa46aa1 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 0faa9350-2c02-47c7-a0a6-9f4afd69152c in service fd00:1122:3344:101::2a - crucible 29278a22-1ba1-4117-bfdb-39fcb9ae7fd1 in service fd00:1122:3344:101::2c - crucible 5b44003e-1a3d-4152-b606-872c72efce0e in service fd00:1122:3344:101::23 - crucible 943fea7a-9458-4935-9dc7-01ee5cfe5a02 in service fd00:1122:3344:101::27 - crucible a5a0b7a9-37c9-4dbd-8393-ec7748ada3b0 in service fd00:1122:3344:101::29 - crucible aa25add8-60b0-4ace-ac60-15adcdd32d50 in service fd00:1122:3344:101::28 - crucible aac3ab51-9e2b-4605-9bf6-e3eb3681c2b5 in service fd00:1122:3344:101::2b - crucible b6f2dd1e-7f98-4a68-9df2-b33c69d1f7ea in service fd00:1122:3344:101::25 - crucible dc22d470-dc46-436b-9750-25c8d7d369e2 in service fd00:1122:3344:101::24 - crucible f7e434f9-6d4a-476b-a9e2-48d6ee28a08e in service fd00:1122:3344:101::26 - internal_dns 95c3b6d1-2592-4252-b5c1-5d0faf3ce9c9 in service fd00:1122:3344:2::1 - internal_ntp fb36b9dc-273a-4bc3-aaa9-19ee4d0ef552 in service fd00:1122:3344:101::21 - nexus a9a6a974-8953-4783-b815-da46884f2c02 in service fd00:1122:3344:101::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0faa9350-2c02-47c7-a0a6-9f4afd69152c in service fd00:1122:3344:101::28 + crucible 29278a22-1ba1-4117-bfdb-39fcb9ae7fd1 in service fd00:1122:3344:101::2a + crucible 4330134c-41b9-4097-aa0b-3eaefa06d473 in service fd00:1122:3344:101::2c + crucible 65d03287-e43f-45f4-902e-0a5e4638f31a in service fd00:1122:3344:101::2d + crucible 943fea7a-9458-4935-9dc7-01ee5cfe5a02 in service fd00:1122:3344:101::25 + crucible 9b722fea-a186-4bc3-bc37-ce7f6de6a796 in service fd00:1122:3344:101::2b + crucible a5a0b7a9-37c9-4dbd-8393-ec7748ada3b0 in service fd00:1122:3344:101::27 + crucible aa25add8-60b0-4ace-ac60-15adcdd32d50 in service fd00:1122:3344:101::26 + crucible aac3ab51-9e2b-4605-9bf6-e3eb3681c2b5 in service fd00:1122:3344:101::29 + crucible f7e434f9-6d4a-476b-a9e2-48d6ee28a08e in service fd00:1122:3344:101::24 + crucible_pantry b6f2dd1e-7f98-4a68-9df2-b33c69d1f7ea in service fd00:1122:3344:101::23 + internal_dns dc22d470-dc46-436b-9750-25c8d7d369e2 in service fd00:1122:3344:2::1 + internal_ntp 95c3b6d1-2592-4252-b5c1-5d0faf3ce9c9 in service fd00:1122:3344:101::21 + nexus 5b44003e-1a3d-4152-b606-872c72efce0e in service fd00:1122:3344:101::22 sled be7f4375-2a6b-457f-b1a4-3074a715e5fe (active): @@ -95,22 +98,23 @@ to: blueprint e4aeb3b3-272f-4967-be34-2d34daa46aa1 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 248db330-56e6-4c7e-b5ff-9cd6cbcb210a in service fd00:1122:3344:102::29 - crucible 353b0aff-4c71-4fae-a6bd-adcb1d2a1a1d in service fd00:1122:3344:102::26 - crucible 6a5901b1-f9d7-425c-8ecb-a786c900f217 in service fd00:1122:3344:102::24 - crucible b3583b5f-4a62-4471-9be7-41e61578de4c in service fd00:1122:3344:102::27 - crucible b97bdef5-ed14-4e11-9d3b-3379c18ea694 in service fd00:1122:3344:102::2c - crucible bac92034-b9e6-4e8b-9ffb-dbba9caec88d in service fd00:1122:3344:102::25 - crucible c240ec8c-cec5-4117-944d-faeb5672d568 in service fd00:1122:3344:102::2b - crucible cf766535-9b6f-4263-a83a-86f45f7b005b in service fd00:1122:3344:102::2a - crucible d9653001-f671-4905-a410-6a7abc358318 in service fd00:1122:3344:102::28 - crucible edaca77e-5806-446a-b00c-125962cd551d in service fd00:1122:3344:102::23 - internal_dns 65d03287-e43f-45f4-902e-0a5e4638f31a in service fd00:1122:3344:3::1 - internal_ntp 9b722fea-a186-4bc3-bc37-ce7f6de6a796 in service fd00:1122:3344:102::21 - nexus 4330134c-41b9-4097-aa0b-3eaefa06d473 in service fd00:1122:3344:102::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 248db330-56e6-4c7e-b5ff-9cd6cbcb210a in service fd00:1122:3344:102::26 + crucible 3bff7b8a-1737-4f13-ba1c-713785f00c69 in service fd00:1122:3344:102::2c + crucible 75b0a160-7923-4f87-b7f3-f2d40b340e27 in service fd00:1122:3344:102::2b + crucible b3583b5f-4a62-4471-9be7-41e61578de4c in service fd00:1122:3344:102::24 + crucible b7bf29a5-ef5f-4942-a3be-e943f7e6be80 in service fd00:1122:3344:102::2a + crucible b97bdef5-ed14-4e11-9d3b-3379c18ea694 in service fd00:1122:3344:102::29 + crucible c240ec8c-cec5-4117-944d-faeb5672d568 in service fd00:1122:3344:102::28 + crucible cf766535-9b6f-4263-a83a-86f45f7b005b in service fd00:1122:3344:102::27 + crucible d9653001-f671-4905-a410-6a7abc358318 in service fd00:1122:3344:102::25 + crucible d9f181c5-bda0-409f-ae72-a46a906ca931 in service fd00:1122:3344:102::2d + crucible_pantry 353b0aff-4c71-4fae-a6bd-adcb1d2a1a1d in service fd00:1122:3344:102::23 + internal_dns bac92034-b9e6-4e8b-9ffb-dbba9caec88d in service fd00:1122:3344:3::1 + internal_ntp edaca77e-5806-446a-b00c-125962cd551d in service fd00:1122:3344:102::21 + nexus 6a5901b1-f9d7-425c-8ecb-a786c900f217 in service fd00:1122:3344:102::22 COCKROACHDB SETTINGS: diff --git a/nexus/reconfigurator/planning/tests/output/example_builder_zone_counts_blueprint.txt b/nexus/reconfigurator/planning/tests/output/example_builder_zone_counts_blueprint.txt new file mode 100644 index 0000000000..92327f0adf --- /dev/null +++ b/nexus/reconfigurator/planning/tests/output/example_builder_zone_counts_blueprint.txt @@ -0,0 +1,217 @@ +blueprint 4a0b8410-b14f-41e7-85e7-3c0fe7050ccc +parent: e35b2fdd-354d-48d9-acb5-703b2c269a54 + + sled: 0dbf1e39-e265-4071-a8df-6d1225b46694 (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-2b9c7004-fa39-4cf0-ae41-c299d3191f26 + fake-vendor fake-model serial-3ad934d9-90ee-4805-881c-20108827773f + fake-vendor fake-model serial-69db5dd6-795e-4e04-bfb3-f51962c49853 + fake-vendor fake-model serial-8557a3fb-cc12-497f-86d2-9f1a463b3685 + fake-vendor fake-model serial-9bd3cc34-4891-4c28-a4de-c4fcf01b6215 + fake-vendor fake-model serial-9dafffa2-31b7-43c0-b673-0c946be799f0 + fake-vendor fake-model serial-9e626a52-b5f1-4776-9cb8-271848b9c651 + fake-vendor fake-model serial-a645a1ac-4c49-4c7e-ba53-3dc60b737f06 + fake-vendor fake-model serial-b5ae209c-9226-44a0-8a6b-03b44f93d456 + fake-vendor fake-model serial-cd783b74-e400-41e0-9bb7-1d1d2f8958ce + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse fadaa1c6-81a2-46ed-a10f-c42c29b85de9 in service fd00:1122:3344:105::24 + crucible 4f673614-7a9d-4860-859c-9627de33e03b in service fd00:1122:3344:105::2b + crucible 6422c22d-00bf-4826-9aeb-cf4e4dcae1c7 in service fd00:1122:3344:105::2d + crucible 76edbb5e-7038-4160-833b-131abc2b2a12 in service fd00:1122:3344:105::31 + crucible 85631c30-4b81-4a22-a0e5-e110034cc980 in service fd00:1122:3344:105::29 + crucible 9334f86c-6555-499a-9ad7-5acc987e2b87 in service fd00:1122:3344:105::2f + crucible a0624221-da2b-4f45-862e-effec567dcd1 in service fd00:1122:3344:105::2a + crucible c33f069b-e412-4309-a62b-e1cbef3ec234 in service fd00:1122:3344:105::30 + crucible c863bdbb-f270-47e1-bf7b-2220cf129266 in service fd00:1122:3344:105::28 + crucible d42c15ed-d303-43d4-b9e9-c5772505c2d7 in service fd00:1122:3344:105::2c + crucible d660e9f0-2f8f-4540-bd59-a32845d002ce in service fd00:1122:3344:105::2e + crucible_pantry a9980763-1f8c-452d-a542-551d1001131e in service fd00:1122:3344:105::27 + external_dns 5002e44e-eef1-4f3c-81fc-78b62134400e in service fd00:1122:3344:105::26 + external_dns 82db7bcc-9e4c-418f-a46a-eb8ff561de7b in service fd00:1122:3344:105::25 + internal_dns faac06c4-9aea-44a3-a94f-3da1adddb7b9 in service fd00:1122:3344:1::1 + internal_ntp a3a3fccb-1127-4e61-9bdf-13eb58365a8a in service fd00:1122:3344:105::21 + nexus 1a741601-83b2-40b7-b59b-1a9b2c853411 in service fd00:1122:3344:105::23 + nexus a7743bca-5056-479f-9151-6f2288c0ae28 in service fd00:1122:3344:105::22 + + + + sled: 15cf73a6-445b-4d36-9232-5ed364019bc6 (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-3cba16f1-1a3d-44e5-ba1d-e68cd2188615 + fake-vendor fake-model serial-3fc3ec87-1c39-4df8-99bf-30ca97ec5fac + fake-vendor fake-model serial-4c18c1af-dc1c-4de5-92a1-1b2923ea6a87 + fake-vendor fake-model serial-6f3a85db-de97-40d9-bf66-e6643ac1c114 + fake-vendor fake-model serial-96f39cf4-b2ac-413d-8f94-ba66b127cddd + fake-vendor fake-model serial-99c392f3-77b8-4f60-9efa-4efae0c92721 + fake-vendor fake-model serial-bc3195df-61ca-4111-863b-08b5cc243eab + fake-vendor fake-model serial-c787b52c-2cb8-4da2-a17a-128feb5eea0c + fake-vendor fake-model serial-d59d419e-c4d3-41ef-ab6d-0df3620dc84b + fake-vendor fake-model serial-dfaae221-11a9-4db0-b861-41fe5648f185 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0dba037d-0b14-4e10-b430-c271f87d8e9a in service fd00:1122:3344:104::27 + crucible 10f24d62-dec3-4f6f-9f42-722763e1fcbd in service fd00:1122:3344:104::2b + crucible 313bb913-2e8b-4c27-89e6-79c92db8c03c in service fd00:1122:3344:104::26 + crucible 3c900bee-7455-4a41-8175-fe952484753c in service fd00:1122:3344:104::2f + crucible 68ff33cc-852b-4f2c-bc5c-9f7bdc32110e in service fd00:1122:3344:104::2e + crucible 69b59e77-9da3-49d3-bf83-a464570aea97 in service fd00:1122:3344:104::28 + crucible 7943dee3-d4ad-4ffb-93c7-2c1079e708a1 in service fd00:1122:3344:104::2c + crucible b2f90d4d-34ff-4aae-8cfd-4773a30c372f in service fd00:1122:3344:104::29 + crucible caf0bd1b-1da6-46b0-bb64-d6b780ad7a4c in service fd00:1122:3344:104::2d + crucible f3c76e5a-779a-4c3c-87ce-dad309f612c4 in service fd00:1122:3344:104::2a + crucible_pantry 362b181b-452e-4f54-a42f-04bef00fa272 in service fd00:1122:3344:104::25 + external_dns 60a0051f-d530-49e6-ab11-e2098856afba in service fd00:1122:3344:104::23 + external_dns 8343556a-7827-4fdd-90df-a4b603b177cc in service fd00:1122:3344:104::24 + internal_dns 1c0614f1-b653-43c2-b278-5372036a87c8 in service fd00:1122:3344:2::1 + internal_ntp 35a629e9-5140-48bf-975d-ce66d9c3be59 in service fd00:1122:3344:104::21 + nexus 58d88005-e182-4463-96ed-9220e59facb7 in service fd00:1122:3344:104::22 + + + + sled: 50e6c1c0-43b2-4abc-9041-41165597f639 (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-00fb9aa9-0bbf-49ab-a712-6e8feaf719e2 + fake-vendor fake-model serial-37f466d7-510b-40e4-b9a4-c5c092a6a5f6 + fake-vendor fake-model serial-734b7b3a-86af-48a7-bd00-8d79fa2690c3 + fake-vendor fake-model serial-747d2504-36a4-4acc-ad73-22291b5bbedb + fake-vendor fake-model serial-7dd422ab-4839-4a7a-8109-ba1941357c70 + fake-vendor fake-model serial-8b020037-bc77-48b2-9280-a622f571908b + fake-vendor fake-model serial-96753d7f-de6b-4ce6-a9dc-004f6a0ba0cf + fake-vendor fake-model serial-a0bd8e79-1113-4c40-8705-ed00e66f0c35 + fake-vendor fake-model serial-b30e150e-c83e-4c1e-b3bf-91a330d42135 + fake-vendor fake-model serial-ff911b9b-57a8-4318-a253-e2363b70083d + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0d5f20e4-fdca-4611-ac07-053bc28c5088 in service fd00:1122:3344:102::2c + crucible 234514a8-7a70-4dc5-905b-a29d1cfdee99 in service fd00:1122:3344:102::29 + crucible 29400193-6c20-4c41-8c9a-5259803c7bfd in service fd00:1122:3344:102::28 + crucible 7cbc95ce-ab55-4b70-851d-33a0164ca362 in service fd00:1122:3344:102::2e + crucible 7e6fa426-5200-47b4-b791-393dd17b09d9 in service fd00:1122:3344:102::2f + crucible a16b5904-5ba8-42db-940c-8b852b052995 in service fd00:1122:3344:102::2d + crucible a1a32afe-10db-4dbf-ac2d-6745f6f55417 in service fd00:1122:3344:102::2b + crucible bec70cc9-82f9-42d3-935b-4f141c3c3503 in service fd00:1122:3344:102::2a + crucible de2023c8-6976-40fa-80d7-a8b0ea9546a1 in service fd00:1122:3344:102::27 + crucible ed528efc-4d32-43cc-a0a2-b412693a7eca in service fd00:1122:3344:102::26 + crucible_pantry a3333bac-4989-4d9a-8257-8c7a0614cef0 in service fd00:1122:3344:102::25 + external_dns 31fdc4e5-25f2-42c5-8c2c-7f8bf3a0b89e in service fd00:1122:3344:102::23 + external_dns e4aeecc2-f9ae-4b4f-963c-9c017e9df189 in service fd00:1122:3344:102::24 + internal_ntp cb4a2547-7798-48d2-839d-29f7d33d1486 in service fd00:1122:3344:102::21 + nexus 5895eb4f-7132-4530-8fc1-f9b719981856 in service fd00:1122:3344:102::22 + + + + sled: 969ff976-df34-402c-a362-53db03a6b97f (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-07444848-952b-4333-aa72-401c7bf5d724 + fake-vendor fake-model serial-242f8f98-fdc2-4ea9-ab69-e57b993df0df + fake-vendor fake-model serial-26cc7ce1-dc59-4398-8083-a4e1db957a46 + fake-vendor fake-model serial-3b757772-8c62-4543-a276-7c0051280687 + fake-vendor fake-model serial-981430ec-a43e-4418-bd2c-28db344c8b06 + fake-vendor fake-model serial-9dbfe441-887c-45d0-a3ed-7d8e1a63327f + fake-vendor fake-model serial-b37f5663-bedb-42a3-9b1a-5e417ee6c3d2 + fake-vendor fake-model serial-b48a178d-f7fd-4b50-811d-f7d195752710 + fake-vendor fake-model serial-be1784b0-017a-436f-8a6a-1884cddc5fa1 + fake-vendor fake-model serial-f9415bcf-5757-442a-a400-5a9ccfb5d80a + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0a6c78bb-f778-4b6d-9d20-370ae7c89135 in service fd00:1122:3344:103::29 + crucible 19faa2c3-b077-470a-a424-6821dd49bd11 in service fd00:1122:3344:103::2f + crucible 236e5a84-cebe-40b8-8832-56a8e06b08b0 in service fd00:1122:3344:103::2c + crucible 590907c2-e61e-4c2f-8a36-e705a24600c8 in service fd00:1122:3344:103::27 + crucible 77511292-f3aa-4a07-970a-1159e2c38ec9 in service fd00:1122:3344:103::2b + crucible c3b6651b-6a62-4292-972c-481390f4a044 in service fd00:1122:3344:103::26 + crucible dd9050e5-77ae-4dd0-98aa-d34fc2be78d4 in service fd00:1122:3344:103::2d + crucible e8ac4104-9789-4ba1-90c5-aded124cc079 in service fd00:1122:3344:103::2e + crucible ea723ef1-b23e-433d-bef5-90142476225d in service fd00:1122:3344:103::28 + crucible fbc67c83-773a-48ff-857c-61ddbf1ebf52 in service fd00:1122:3344:103::2a + crucible_pantry 9ff080b2-7bf9-4fe3-8d5f-367adb3525c8 in service fd00:1122:3344:103::25 + external_dns 983ed36f-baed-4dd1-bec7-4780f070eeee in service fd00:1122:3344:103::24 + external_dns e8bb3c35-b1ee-4171-b8ff-924e6a560fbf in service fd00:1122:3344:103::23 + internal_ntp e8d99319-3796-4605-bd96-e17011d77016 in service fd00:1122:3344:103::21 + nexus efa54182-a3e6-4600-9e88-044a6a8ae350 in service fd00:1122:3344:103::22 + + + + sled: ec5c0b37-b651-4c45-ac1c-24541ef9c44b (active) + + physical disks at generation 1: + ---------------------------------------------------------------------- + vendor model serial + ---------------------------------------------------------------------- + fake-vendor fake-model serial-148436fe-d3e9-4371-8d2e-ec950cc8a84c + fake-vendor fake-model serial-7dd076f1-9d62-49a8-bc0c-5ff5d045c917 + fake-vendor fake-model serial-a50f4bb9-d19a-4be8-ad49-b9a552a21062 + fake-vendor fake-model serial-b4ee33bb-03f1-4085-9830-9da92002a969 + fake-vendor fake-model serial-b50bec8b-a8d3-4ba6-ba3d-12c2a0da911c + fake-vendor fake-model serial-b64f79f6-188f-4e98-9eac-d8111673a130 + fake-vendor fake-model serial-c144a26e-f859-42a0-adca-00d9091d98e4 + fake-vendor fake-model serial-c45c08e4-aade-4333-9dad-935ccf4e8352 + fake-vendor fake-model serial-df62d5da-7da0-468b-b328-0fefbf57568b + fake-vendor fake-model serial-e6433ded-7c90-46a9-8bda-648bcc9fbf07 + + + omicron zones at generation 2: + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 00e6f92a-997a-4c1f-8c84-be547eb012d3 in service fd00:1122:3344:101::2a + crucible 0e12b408-dc46-4827-82e5-4589cb895b58 in service fd00:1122:3344:101::2d + crucible 0ed1d110-0727-4a4c-9ef1-f74fa885dce2 in service fd00:1122:3344:101::2e + crucible 7a520d9b-5f7d-4e5c-b1bc-7e65a35984b6 in service fd00:1122:3344:101::2f + crucible be3b1ad8-3de2-4ad7-89b8-d279558121ae in service fd00:1122:3344:101::28 + crucible c6192cad-a9c5-4b4d-bc8f-ce0fb066a0ed in service fd00:1122:3344:101::27 + crucible d3445179-d068-4406-9aae-255bd0008280 in service fd00:1122:3344:101::2b + crucible e8dea18b-1697-4349-ae54-47f95cb3d907 in service fd00:1122:3344:101::26 + crucible f1f1ee53-a974-45f8-9173-211ed366ab7d in service fd00:1122:3344:101::2c + crucible f4053fb9-ce8c-4dcf-8dd3-bf6d305c29fc in service fd00:1122:3344:101::29 + crucible_pantry 25fb011b-ea9d-462c-a89e-5f7782858a4f in service fd00:1122:3344:101::25 + external_dns 1b122e0d-8b22-464f-b1af-589246d636dc in service fd00:1122:3344:101::24 + external_dns 808e6761-6ddc-42f0-a82f-fd2049c751dc in service fd00:1122:3344:101::23 + internal_ntp a1e77a19-9302-416a-afe9-cfdded5054d5 in service fd00:1122:3344:101::21 + nexus d39f526b-9799-42a6-a610-f0f5605dac44 in service fd00:1122:3344:101::22 + + + COCKROACHDB SETTINGS: + state fingerprint::::::::::::::::: (none) + cluster.preserve_downgrade_option: (do not modify) + + METADATA: + created by::::::::::: test suite + created at::::::::::: 1970-01-01T00:00:00.000Z + comment:::::::::::::: (none) + internal DNS version: 1 + external DNS version: 1 + diff --git a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt index 78bcc2720c..afb18592fc 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_2_3.txt @@ -22,22 +22,24 @@ to: blueprint 4171ad05-89dd-474b-846b-b007e4346366 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service fd00:1122:3344:103::2c - crucible 322ee9f1-8903-4542-a0a8-a54cefabdeca in service fd00:1122:3344:103::23 - crucible 4ab1650f-32c5-447f-939d-64b8103a7645 in service fd00:1122:3344:103::29 - crucible 64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service fd00:1122:3344:103::26 - crucible 6e811d86-8aa7-4660-935b-84b4b7721b10 in service fd00:1122:3344:103::2a - crucible 747d2426-68bf-4c22-8806-41d290b5d5f5 in service fd00:1122:3344:103::24 - crucible 7fbd2c38-5dc3-48c4-b061-558a2041d70f in service fd00:1122:3344:103::2b - crucible 8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service fd00:1122:3344:103::28 - crucible b14d5478-1a0e-4b90-b526-36b06339dfc4 in service fd00:1122:3344:103::27 - crucible be97b92b-38d6-422a-8c76-d37060f75bd2 in service fd00:1122:3344:103::25 - internal_dns b40f7c7b-526c-46c8-ae33-67280c280eb7 in service fd00:1122:3344:1::1 - internal_ntp 267ed614-92af-4b9d-bdba-c2881c2e43a2 in service fd00:1122:3344:103::21 - nexus cc816cfe-3869-4dde-b596-397d41198628 in service fd00:1122:3344:103::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse b40f7c7b-526c-46c8-ae33-67280c280eb7 in service fd00:1122:3344:103::23 + crucible 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service fd00:1122:3344:103::2c + crucible 3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service fd00:1122:3344:103::2e + crucible 4ab1650f-32c5-447f-939d-64b8103a7645 in service fd00:1122:3344:103::29 + crucible 64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service fd00:1122:3344:103::26 + crucible 6e811d86-8aa7-4660-935b-84b4b7721b10 in service fd00:1122:3344:103::2a + crucible 7fbd2c38-5dc3-48c4-b061-558a2041d70f in service fd00:1122:3344:103::2b + crucible 8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service fd00:1122:3344:103::28 + crucible b14d5478-1a0e-4b90-b526-36b06339dfc4 in service fd00:1122:3344:103::27 + crucible be97b92b-38d6-422a-8c76-d37060f75bd2 in service fd00:1122:3344:103::25 + crucible c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service fd00:1122:3344:103::2d + crucible_pantry 747d2426-68bf-4c22-8806-41d290b5d5f5 in service fd00:1122:3344:103::24 + internal_dns 322ee9f1-8903-4542-a0a8-a54cefabdeca in service fd00:1122:3344:1::1 + internal_ntp 267ed614-92af-4b9d-bdba-c2881c2e43a2 in service fd00:1122:3344:103::21 + nexus cc816cfe-3869-4dde-b596-397d41198628 in service fd00:1122:3344:103::22 sled 43677374-8d2f-4deb-8a41-eeea506db8e0 (active): @@ -59,22 +61,23 @@ to: blueprint 4171ad05-89dd-474b-846b-b007e4346366 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 02acbe6a-1c88-47e3-94c3-94084cbde098 in service fd00:1122:3344:101::25 - crucible 07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service fd00:1122:3344:101::24 - crucible 2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service fd00:1122:3344:101::27 - crucible 47199d48-534c-4267-a654-d2d90e64b498 in service fd00:1122:3344:101::2b - crucible 587be699-a320-4c79-b320-128d9ecddc0b in service fd00:1122:3344:101::29 - crucible 6fa06115-4959-4913-8e7b-dd70d7651f07 in service fd00:1122:3344:101::2a - crucible 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service fd00:1122:3344:101::2c - crucible 8f3a1cc5-9195-4a30-ad02-b804278fe639 in service fd00:1122:3344:101::26 - crucible a1696cd4-588c-484a-b95b-66e824c0ce05 in service fd00:1122:3344:101::23 - crucible a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service fd00:1122:3344:101::28 - internal_dns 10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service fd00:1122:3344:2::1 - internal_ntp c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service fd00:1122:3344:101::21 - nexus 3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service fd00:1122:3344:101::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service fd00:1122:3344:101::25 + crucible 47199d48-534c-4267-a654-d2d90e64b498 in service fd00:1122:3344:101::29 + crucible 587be699-a320-4c79-b320-128d9ecddc0b in service fd00:1122:3344:101::27 + crucible 6fa06115-4959-4913-8e7b-dd70d7651f07 in service fd00:1122:3344:101::28 + crucible 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service fd00:1122:3344:101::2a + crucible 8f3a1cc5-9195-4a30-ad02-b804278fe639 in service fd00:1122:3344:101::24 + crucible a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service fd00:1122:3344:101::26 + crucible af322036-371f-437c-8c08-7f40f3f1403b in service fd00:1122:3344:101::2b + crucible d637264f-6f40-44c2-8b7e-a179430210d2 in service fd00:1122:3344:101::2d + crucible edabedf3-839c-488d-ad6f-508ffa864674 in service fd00:1122:3344:101::2c + crucible_pantry 02acbe6a-1c88-47e3-94c3-94084cbde098 in service fd00:1122:3344:101::23 + internal_dns 07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service fd00:1122:3344:2::1 + internal_ntp 10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service fd00:1122:3344:101::21 + nexus a1696cd4-588c-484a-b95b-66e824c0ce05 in service fd00:1122:3344:101::22 sled 590e3034-d946-4166-b0e5-2d0034197a07 (active): @@ -96,22 +99,23 @@ to: blueprint 4171ad05-89dd-474b-846b-b007e4346366 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 0565e7e4-f13a-4123-8928-d715f83e36aa in service fd00:1122:3344:102::2b - crucible 18f8fe40-646e-4962-b17a-20e201f3a6e5 in service fd00:1122:3344:102::27 - crucible 1cc3f503-2001-4d85-80e5-c7c40d2e3b10 in service fd00:1122:3344:102::2a - crucible 56d5d7cf-db2c-40a3-a775-003241ad4820 in service fd00:1122:3344:102::26 - crucible 62058f4c-c747-4e21-a8dc-2fd4a160c98c in service fd00:1122:3344:102::2c - crucible 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service fd00:1122:3344:102::28 - crucible 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service fd00:1122:3344:102::23 - crucible 93f2f40c-5616-4d8d-8519-ec6debdcede0 in service fd00:1122:3344:102::29 - crucible ab7ba6df-d401-40bd-940e-faf57c57aa2a in service fd00:1122:3344:102::25 - crucible dce226c9-7373-4bfa-8a94-79dc472857a6 in service fd00:1122:3344:102::24 - internal_dns d637264f-6f40-44c2-8b7e-a179430210d2 in service fd00:1122:3344:3::1 - internal_ntp af322036-371f-437c-8c08-7f40f3f1403b in service fd00:1122:3344:102::21 - nexus edabedf3-839c-488d-ad6f-508ffa864674 in service fd00:1122:3344:102::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0565e7e4-f13a-4123-8928-d715f83e36aa in service fd00:1122:3344:102::28 + crucible 062ce37d-7448-4d44-b1f4-4937cd2eb174 in service fd00:1122:3344:102::2a + crucible 1211a68e-69a1-4ef4-b790-45b0279f9159 in service fd00:1122:3344:102::2d + crucible 18f8fe40-646e-4962-b17a-20e201f3a6e5 in service fd00:1122:3344:102::24 + crucible 1cc3f503-2001-4d85-80e5-c7c40d2e3b10 in service fd00:1122:3344:102::27 + crucible 62058f4c-c747-4e21-a8dc-2fd4a160c98c in service fd00:1122:3344:102::29 + crucible 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service fd00:1122:3344:102::25 + crucible 78d6ab36-e8c8-4ff8-9f89-75c7fe2d32e6 in service fd00:1122:3344:102::2b + crucible 93f2f40c-5616-4d8d-8519-ec6debdcede0 in service fd00:1122:3344:102::26 + crucible 9f824c30-6360-46b9-87c4-cd60586476fe in service fd00:1122:3344:102::2c + crucible_pantry 56d5d7cf-db2c-40a3-a775-003241ad4820 in service fd00:1122:3344:102::23 + internal_dns ab7ba6df-d401-40bd-940e-faf57c57aa2a in service fd00:1122:3344:3::1 + internal_ntp 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service fd00:1122:3344:102::21 + nexus dce226c9-7373-4bfa-8a94-79dc472857a6 in service fd00:1122:3344:102::22 ADDED SLEDS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt index ebddb48f42..9a5ed1c37a 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_basic_add_sled_3_5.txt @@ -22,22 +22,24 @@ to: blueprint f432fcd5-1284-4058-8b4a-9286a3de6163 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service fd00:1122:3344:103::2c - crucible 322ee9f1-8903-4542-a0a8-a54cefabdeca in service fd00:1122:3344:103::23 - crucible 4ab1650f-32c5-447f-939d-64b8103a7645 in service fd00:1122:3344:103::29 - crucible 64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service fd00:1122:3344:103::26 - crucible 6e811d86-8aa7-4660-935b-84b4b7721b10 in service fd00:1122:3344:103::2a - crucible 747d2426-68bf-4c22-8806-41d290b5d5f5 in service fd00:1122:3344:103::24 - crucible 7fbd2c38-5dc3-48c4-b061-558a2041d70f in service fd00:1122:3344:103::2b - crucible 8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service fd00:1122:3344:103::28 - crucible b14d5478-1a0e-4b90-b526-36b06339dfc4 in service fd00:1122:3344:103::27 - crucible be97b92b-38d6-422a-8c76-d37060f75bd2 in service fd00:1122:3344:103::25 - internal_dns b40f7c7b-526c-46c8-ae33-67280c280eb7 in service fd00:1122:3344:1::1 - internal_ntp 267ed614-92af-4b9d-bdba-c2881c2e43a2 in service fd00:1122:3344:103::21 - nexus cc816cfe-3869-4dde-b596-397d41198628 in service fd00:1122:3344:103::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse b40f7c7b-526c-46c8-ae33-67280c280eb7 in service fd00:1122:3344:103::23 + crucible 08c7f8aa-1ea9-469b-8cac-2fdbfc11ebcb in service fd00:1122:3344:103::2c + crucible 3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service fd00:1122:3344:103::2e + crucible 4ab1650f-32c5-447f-939d-64b8103a7645 in service fd00:1122:3344:103::29 + crucible 64aa65f8-1ccb-4cd6-9953-027aebdac8ff in service fd00:1122:3344:103::26 + crucible 6e811d86-8aa7-4660-935b-84b4b7721b10 in service fd00:1122:3344:103::2a + crucible 7fbd2c38-5dc3-48c4-b061-558a2041d70f in service fd00:1122:3344:103::2b + crucible 8e9e923e-62b1-4cbc-9f59-d6397e338b6b in service fd00:1122:3344:103::28 + crucible b14d5478-1a0e-4b90-b526-36b06339dfc4 in service fd00:1122:3344:103::27 + crucible be97b92b-38d6-422a-8c76-d37060f75bd2 in service fd00:1122:3344:103::25 + crucible c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service fd00:1122:3344:103::2d + crucible_pantry 747d2426-68bf-4c22-8806-41d290b5d5f5 in service fd00:1122:3344:103::24 + internal_dns 322ee9f1-8903-4542-a0a8-a54cefabdeca in service fd00:1122:3344:1::1 + internal_ntp 267ed614-92af-4b9d-bdba-c2881c2e43a2 in service fd00:1122:3344:103::21 + nexus cc816cfe-3869-4dde-b596-397d41198628 in service fd00:1122:3344:103::22 sled 43677374-8d2f-4deb-8a41-eeea506db8e0 (active): @@ -59,22 +61,23 @@ to: blueprint f432fcd5-1284-4058-8b4a-9286a3de6163 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 02acbe6a-1c88-47e3-94c3-94084cbde098 in service fd00:1122:3344:101::25 - crucible 07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service fd00:1122:3344:101::24 - crucible 2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service fd00:1122:3344:101::27 - crucible 47199d48-534c-4267-a654-d2d90e64b498 in service fd00:1122:3344:101::2b - crucible 587be699-a320-4c79-b320-128d9ecddc0b in service fd00:1122:3344:101::29 - crucible 6fa06115-4959-4913-8e7b-dd70d7651f07 in service fd00:1122:3344:101::2a - crucible 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service fd00:1122:3344:101::2c - crucible 8f3a1cc5-9195-4a30-ad02-b804278fe639 in service fd00:1122:3344:101::26 - crucible a1696cd4-588c-484a-b95b-66e824c0ce05 in service fd00:1122:3344:101::23 - crucible a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service fd00:1122:3344:101::28 - internal_dns 10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service fd00:1122:3344:2::1 - internal_ntp c66ab6d5-ff7a-46d1-9fd0-70cefa352d25 in service fd00:1122:3344:101::21 - nexus 3eda924f-22a9-4f3e-9a1b-91d1c47601ab in service fd00:1122:3344:101::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 2a455c35-eb3c-4c73-ab6c-d0a706e25316 in service fd00:1122:3344:101::25 + crucible 47199d48-534c-4267-a654-d2d90e64b498 in service fd00:1122:3344:101::29 + crucible 587be699-a320-4c79-b320-128d9ecddc0b in service fd00:1122:3344:101::27 + crucible 6fa06115-4959-4913-8e7b-dd70d7651f07 in service fd00:1122:3344:101::28 + crucible 704e1fed-f8d6-4cfa-a470-bad27fdc06d1 in service fd00:1122:3344:101::2a + crucible 8f3a1cc5-9195-4a30-ad02-b804278fe639 in service fd00:1122:3344:101::24 + crucible a2079cbc-a69e-41a1-b1e0-fbcb972d03f6 in service fd00:1122:3344:101::26 + crucible af322036-371f-437c-8c08-7f40f3f1403b in service fd00:1122:3344:101::2b + crucible d637264f-6f40-44c2-8b7e-a179430210d2 in service fd00:1122:3344:101::2d + crucible edabedf3-839c-488d-ad6f-508ffa864674 in service fd00:1122:3344:101::2c + crucible_pantry 02acbe6a-1c88-47e3-94c3-94084cbde098 in service fd00:1122:3344:101::23 + internal_dns 07c3c805-8888-4fe5-9543-3d2479dbe6f3 in service fd00:1122:3344:2::1 + internal_ntp 10d98a73-ec88-4aff-a7e8-7db6a87880e6 in service fd00:1122:3344:101::21 + nexus a1696cd4-588c-484a-b95b-66e824c0ce05 in service fd00:1122:3344:101::22 sled 590e3034-d946-4166-b0e5-2d0034197a07 (active): @@ -96,22 +99,23 @@ to: blueprint f432fcd5-1284-4058-8b4a-9286a3de6163 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 0565e7e4-f13a-4123-8928-d715f83e36aa in service fd00:1122:3344:102::2b - crucible 18f8fe40-646e-4962-b17a-20e201f3a6e5 in service fd00:1122:3344:102::27 - crucible 1cc3f503-2001-4d85-80e5-c7c40d2e3b10 in service fd00:1122:3344:102::2a - crucible 56d5d7cf-db2c-40a3-a775-003241ad4820 in service fd00:1122:3344:102::26 - crucible 62058f4c-c747-4e21-a8dc-2fd4a160c98c in service fd00:1122:3344:102::2c - crucible 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service fd00:1122:3344:102::28 - crucible 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service fd00:1122:3344:102::23 - crucible 93f2f40c-5616-4d8d-8519-ec6debdcede0 in service fd00:1122:3344:102::29 - crucible ab7ba6df-d401-40bd-940e-faf57c57aa2a in service fd00:1122:3344:102::25 - crucible dce226c9-7373-4bfa-8a94-79dc472857a6 in service fd00:1122:3344:102::24 - internal_dns d637264f-6f40-44c2-8b7e-a179430210d2 in service fd00:1122:3344:3::1 - internal_ntp af322036-371f-437c-8c08-7f40f3f1403b in service fd00:1122:3344:102::21 - nexus edabedf3-839c-488d-ad6f-508ffa864674 in service fd00:1122:3344:102::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0565e7e4-f13a-4123-8928-d715f83e36aa in service fd00:1122:3344:102::28 + crucible 062ce37d-7448-4d44-b1f4-4937cd2eb174 in service fd00:1122:3344:102::2a + crucible 1211a68e-69a1-4ef4-b790-45b0279f9159 in service fd00:1122:3344:102::2d + crucible 18f8fe40-646e-4962-b17a-20e201f3a6e5 in service fd00:1122:3344:102::24 + crucible 1cc3f503-2001-4d85-80e5-c7c40d2e3b10 in service fd00:1122:3344:102::27 + crucible 62058f4c-c747-4e21-a8dc-2fd4a160c98c in service fd00:1122:3344:102::29 + crucible 6af7f4d6-33b6-4eb3-a146-d8e9e4ae9d66 in service fd00:1122:3344:102::25 + crucible 78d6ab36-e8c8-4ff8-9f89-75c7fe2d32e6 in service fd00:1122:3344:102::2b + crucible 93f2f40c-5616-4d8d-8519-ec6debdcede0 in service fd00:1122:3344:102::26 + crucible 9f824c30-6360-46b9-87c4-cd60586476fe in service fd00:1122:3344:102::2c + crucible_pantry 56d5d7cf-db2c-40a3-a775-003241ad4820 in service fd00:1122:3344:102::23 + internal_dns ab7ba6df-d401-40bd-940e-faf57c57aa2a in service fd00:1122:3344:3::1 + internal_ntp 7a9f60d3-2b66-4547-9b63-7d4f7a8b6382 in service fd00:1122:3344:102::21 + nexus dce226c9-7373-4bfa-8a94-79dc472857a6 in service fd00:1122:3344:102::22 MODIFIED SLEDS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_1_2.txt b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_1_2.txt index ca7f27f19e..ee0dc4bbe2 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_1_2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_1_2.txt @@ -22,35 +22,39 @@ to: blueprint 1ac2d88f-27dd-4506-8585-6b2be832528e omicron zones generation 2 -> 3: - ------------------------------------------------------------------------------------------- - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------- -* crucible 1e1ed0cc-1adc-410f-943a-d1a3107de619 - in service fd00:1122:3344:103::26 - └─ + expunged -* crucible 2307bbed-02ba-493b-89e3-46585c74c8fc - in service fd00:1122:3344:103::27 - └─ + expunged -* crucible 603e629d-2599-400e-b879-4134d4cc426e - in service fd00:1122:3344:103::2b - └─ + expunged -* crucible 9179d6dc-387d-424e-8d62-ed59b2c728f6 - in service fd00:1122:3344:103::29 - └─ + expunged -* crucible ad76d200-5675-444b-b19c-684689ff421f - in service fd00:1122:3344:103::2c - └─ + expunged -* crucible c28d7b4b-a259-45ad-945d-f19ca3c6964c - in service fd00:1122:3344:103::28 - └─ + expunged -* crucible e29998e7-9ed2-46b6-bb70-4118159fe07f - in service fd00:1122:3344:103::25 - └─ + expunged -* crucible f06e91a1-0c17-4cca-adbc-1c9b67bdb11d - in service fd00:1122:3344:103::2a - └─ + expunged -* crucible f11f5c60-1ac7-4630-9a3a-a9bc85c75203 - in service fd00:1122:3344:103::24 - └─ + expunged -* crucible f231e4eb-3fc9-4964-9d71-2c41644852d9 - in service fd00:1122:3344:103::23 - └─ + expunged -* internal_dns 4e36b7ef-5684-4304-b7c3-3c31aaf83d4f - in service fd00:1122:3344:1::1 - └─ + expunged -* internal_ntp c62b87b6-b98d-4d22-ba4f-cee4499e2ba8 - in service fd00:1122:3344:103::21 - └─ + expunged -* nexus 6a70a233-1900-43c0-9c00-aa9d1f7adfbc - in service fd00:1122:3344:103::22 - └─ + expunged + ---------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + ---------------------------------------------------------------------------------------------- +* clickhouse 4e36b7ef-5684-4304-b7c3-3c31aaf83d4f - in service fd00:1122:3344:103::23 + └─ + expunged +* crucible 1e1ed0cc-1adc-410f-943a-d1a3107de619 - in service fd00:1122:3344:103::26 + └─ + expunged +* crucible 2307bbed-02ba-493b-89e3-46585c74c8fc - in service fd00:1122:3344:103::27 + └─ + expunged +* crucible 2e65b765-5c41-4519-bf4e-e2a68569afc1 - in service fd00:1122:3344:103::2e + └─ + expunged +* crucible 603e629d-2599-400e-b879-4134d4cc426e - in service fd00:1122:3344:103::2b + └─ + expunged +* crucible 9179d6dc-387d-424e-8d62-ed59b2c728f6 - in service fd00:1122:3344:103::29 + └─ + expunged +* crucible ad76d200-5675-444b-b19c-684689ff421f - in service fd00:1122:3344:103::2c + └─ + expunged +* crucible c28d7b4b-a259-45ad-945d-f19ca3c6964c - in service fd00:1122:3344:103::28 + └─ + expunged +* crucible e29998e7-9ed2-46b6-bb70-4118159fe07f - in service fd00:1122:3344:103::25 + └─ + expunged +* crucible e9bf2525-5fa0-4c1b-b52d-481225083845 - in service fd00:1122:3344:103::2d + └─ + expunged +* crucible f06e91a1-0c17-4cca-adbc-1c9b67bdb11d - in service fd00:1122:3344:103::2a + └─ + expunged +* crucible_pantry f11f5c60-1ac7-4630-9a3a-a9bc85c75203 - in service fd00:1122:3344:103::24 + └─ + expunged +* internal_dns f231e4eb-3fc9-4964-9d71-2c41644852d9 - in service fd00:1122:3344:1::1 + └─ + expunged +* internal_ntp c62b87b6-b98d-4d22-ba4f-cee4499e2ba8 - in service fd00:1122:3344:103::21 + └─ + expunged +* nexus 6a70a233-1900-43c0-9c00-aa9d1f7adfbc - in service fd00:1122:3344:103::22 + └─ + expunged sled d67ce8f0-a691-4010-b414-420d82e80527 (active): @@ -72,23 +76,25 @@ to: blueprint 1ac2d88f-27dd-4506-8585-6b2be832528e omicron zones generation 2 -> 3: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 15dbaa30-1539-49d6-970d-ba5962960f33 in service fd00:1122:3344:101::25 - crucible 3d4143df-e212-4774-9258-7d9b421fac2e in service fd00:1122:3344:101::23 - crucible 5d9d8fa7-8379-470b-90ba-fe84a3c45512 in service fd00:1122:3344:101::28 - crucible 70232a6d-6c9d-4fa6-a34d-9c73d940db33 in service fd00:1122:3344:101::26 - crucible 8567a616-a709-4c8c-a323-4474675dad5c in service fd00:1122:3344:101::2a - crucible 8b0b8623-930a-41af-9f9b-ca28b1b11139 in service fd00:1122:3344:101::27 - crucible 99c6401d-9796-4ae1-bf0c-9a097cf21c33 in service fd00:1122:3344:101::2c - crucible cf87d2a3-d323-44a3-a87e-adc4ef6c75f4 in service fd00:1122:3344:101::29 - crucible eac6c0a0-baa5-4490-9cee-65198b7fbd9c in service fd00:1122:3344:101::24 - crucible f68846ad-4619-4747-8293-a2b4aeeafc5b in service fd00:1122:3344:101::2b - internal_dns 1ec4cc7b-2f00-4d13-8176-3b9815533ae9 in service fd00:1122:3344:2::1 - internal_ntp e9bf2525-5fa0-4c1b-b52d-481225083845 in service fd00:1122:3344:101::21 - nexus 2e65b765-5c41-4519-bf4e-e2a68569afc1 in service fd00:1122:3344:101::22 -+ nexus ff9ce09c-afbf-425b-bbfa-3d8fb254f98e in service fd00:1122:3344:101::2d + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 4f8ce495-21dd-48a1-859c-80d34ce394ed in service fd00:1122:3344:101::2b + crucible 5d9d8fa7-8379-470b-90ba-fe84a3c45512 in service fd00:1122:3344:101::26 + crucible 70232a6d-6c9d-4fa6-a34d-9c73d940db33 in service fd00:1122:3344:101::24 + crucible 8567a616-a709-4c8c-a323-4474675dad5c in service fd00:1122:3344:101::28 + crucible 8b0b8623-930a-41af-9f9b-ca28b1b11139 in service fd00:1122:3344:101::25 + crucible 99c6401d-9796-4ae1-bf0c-9a097cf21c33 in service fd00:1122:3344:101::2a + crucible a1ae92ac-e1f1-4654-ab54-5b75ba7c44d6 in service fd00:1122:3344:101::2c + crucible a308d3e1-118c-440a-947a-8b6ab7d833ab in service fd00:1122:3344:101::2d + crucible cf87d2a3-d323-44a3-a87e-adc4ef6c75f4 in service fd00:1122:3344:101::27 + crucible f68846ad-4619-4747-8293-a2b4aeeafc5b in service fd00:1122:3344:101::29 + crucible_pantry 15dbaa30-1539-49d6-970d-ba5962960f33 in service fd00:1122:3344:101::23 + internal_dns eac6c0a0-baa5-4490-9cee-65198b7fbd9c in service fd00:1122:3344:2::1 + internal_ntp 1ec4cc7b-2f00-4d13-8176-3b9815533ae9 in service fd00:1122:3344:101::21 + nexus 3d4143df-e212-4774-9258-7d9b421fac2e in service fd00:1122:3344:101::22 ++ crucible_pantry ff9ce09c-afbf-425b-bbfa-3d8fb254f98e in service fd00:1122:3344:101::2e ++ nexus 845869e9-ecb2-4ec3-b6b8-2a836e459243 in service fd00:1122:3344:101::2f sled fefcf4cf-f7e7-46b3-b629-058526ce440e (active): @@ -110,23 +116,25 @@ to: blueprint 1ac2d88f-27dd-4506-8585-6b2be832528e omicron zones generation 2 -> 3: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 0e2b035e-1de1-48af-8ac0-5316418e3de1 in service fd00:1122:3344:102::27 - crucible 2bf9ee97-90e1-48a7-bb06-a35cec63b7fe in service fd00:1122:3344:102::2b - crucible 5c78756d-6182-4c27-a507-3419e8dbe76b in service fd00:1122:3344:102::25 - crucible b7402110-d88f-4ca4-8391-4a2fda6ad271 in service fd00:1122:3344:102::26 - crucible b7ae596e-0c85-40b2-bb47-df9f76db3cca in service fd00:1122:3344:102::28 - crucible c552280f-ba02-4f8d-9049-bd269e6b7845 in service fd00:1122:3344:102::23 - crucible cf13b878-47f1-4ba0-b8c2-9f3e15f2ee87 in service fd00:1122:3344:102::29 - crucible e3bfcb1e-3708-45e7-a45a-2a2cab7ad829 in service fd00:1122:3344:102::2c - crucible e6d0df1f-9f98-4c5a-9540-8444d1185c7d in service fd00:1122:3344:102::24 - crucible eb034526-1767-4cc4-8225-ec962265710b in service fd00:1122:3344:102::2a - internal_dns a308d3e1-118c-440a-947a-8b6ab7d833ab in service fd00:1122:3344:3::1 - internal_ntp 4f8ce495-21dd-48a1-859c-80d34ce394ed in service fd00:1122:3344:102::21 - nexus a1ae92ac-e1f1-4654-ab54-5b75ba7c44d6 in service fd00:1122:3344:102::22 -+ internal_dns c8851a11-a4f7-4b21-9281-6182fd15dc8d in service fd00:1122:3344:1::1 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 0e2b035e-1de1-48af-8ac0-5316418e3de1 in service fd00:1122:3344:102::24 + crucible 15f29557-d4da-45ef-b435-a0a1cd586e0c in service fd00:1122:3344:102::2a + crucible 2bf9ee97-90e1-48a7-bb06-a35cec63b7fe in service fd00:1122:3344:102::28 + crucible 5cf79919-b28e-4064-b6f8-8906c471b5ce in service fd00:1122:3344:102::2d + crucible 751bc6fe-22ad-4ce1-bc51-cf31fdf02bfa in service fd00:1122:3344:102::2b + crucible b7ae596e-0c85-40b2-bb47-df9f76db3cca in service fd00:1122:3344:102::25 + crucible cf13b878-47f1-4ba0-b8c2-9f3e15f2ee87 in service fd00:1122:3344:102::26 + crucible e3bfcb1e-3708-45e7-a45a-2a2cab7ad829 in service fd00:1122:3344:102::29 + crucible e5121f83-faf2-4928-b5a8-94a1da99e8eb in service fd00:1122:3344:102::2c + crucible eb034526-1767-4cc4-8225-ec962265710b in service fd00:1122:3344:102::27 + crucible_pantry b7402110-d88f-4ca4-8391-4a2fda6ad271 in service fd00:1122:3344:102::23 + internal_dns 5c78756d-6182-4c27-a507-3419e8dbe76b in service fd00:1122:3344:3::1 + internal_ntp c552280f-ba02-4f8d-9049-bd269e6b7845 in service fd00:1122:3344:102::21 + nexus e6d0df1f-9f98-4c5a-9540-8444d1185c7d in service fd00:1122:3344:102::22 ++ clickhouse c8851a11-a4f7-4b21-9281-6182fd15dc8d in service fd00:1122:3344:102::2e ++ internal_dns e639b672-27c4-4ecb-82c1-d672eb1ccf4e in service fd00:1122:3344:1::1 COCKROACHDB SETTINGS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt index b9ff4e8140..b0b82a31f8 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_decommissions_sleds_bp2.txt @@ -20,23 +20,25 @@ parent: 516e80a3-b362-4fac-bd3c-4559717120dd omicron zones at generation 3: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 15dbaa30-1539-49d6-970d-ba5962960f33 in service fd00:1122:3344:101::25 - crucible 3d4143df-e212-4774-9258-7d9b421fac2e in service fd00:1122:3344:101::23 - crucible 5d9d8fa7-8379-470b-90ba-fe84a3c45512 in service fd00:1122:3344:101::28 - crucible 70232a6d-6c9d-4fa6-a34d-9c73d940db33 in service fd00:1122:3344:101::26 - crucible 8567a616-a709-4c8c-a323-4474675dad5c in service fd00:1122:3344:101::2a - crucible 8b0b8623-930a-41af-9f9b-ca28b1b11139 in service fd00:1122:3344:101::27 - crucible 99c6401d-9796-4ae1-bf0c-9a097cf21c33 in service fd00:1122:3344:101::2c - crucible cf87d2a3-d323-44a3-a87e-adc4ef6c75f4 in service fd00:1122:3344:101::29 - crucible eac6c0a0-baa5-4490-9cee-65198b7fbd9c in service fd00:1122:3344:101::24 - crucible f68846ad-4619-4747-8293-a2b4aeeafc5b in service fd00:1122:3344:101::2b - internal_dns 1ec4cc7b-2f00-4d13-8176-3b9815533ae9 in service fd00:1122:3344:2::1 - internal_ntp e9bf2525-5fa0-4c1b-b52d-481225083845 in service fd00:1122:3344:101::21 - nexus 2e65b765-5c41-4519-bf4e-e2a68569afc1 in service fd00:1122:3344:101::22 - nexus ff9ce09c-afbf-425b-bbfa-3d8fb254f98e in service fd00:1122:3344:101::2d + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 4f8ce495-21dd-48a1-859c-80d34ce394ed in service fd00:1122:3344:101::2b + crucible 5d9d8fa7-8379-470b-90ba-fe84a3c45512 in service fd00:1122:3344:101::26 + crucible 70232a6d-6c9d-4fa6-a34d-9c73d940db33 in service fd00:1122:3344:101::24 + crucible 8567a616-a709-4c8c-a323-4474675dad5c in service fd00:1122:3344:101::28 + crucible 8b0b8623-930a-41af-9f9b-ca28b1b11139 in service fd00:1122:3344:101::25 + crucible 99c6401d-9796-4ae1-bf0c-9a097cf21c33 in service fd00:1122:3344:101::2a + crucible a1ae92ac-e1f1-4654-ab54-5b75ba7c44d6 in service fd00:1122:3344:101::2c + crucible a308d3e1-118c-440a-947a-8b6ab7d833ab in service fd00:1122:3344:101::2d + crucible cf87d2a3-d323-44a3-a87e-adc4ef6c75f4 in service fd00:1122:3344:101::27 + crucible f68846ad-4619-4747-8293-a2b4aeeafc5b in service fd00:1122:3344:101::29 + crucible_pantry 15dbaa30-1539-49d6-970d-ba5962960f33 in service fd00:1122:3344:101::23 + crucible_pantry ff9ce09c-afbf-425b-bbfa-3d8fb254f98e in service fd00:1122:3344:101::2e + internal_dns eac6c0a0-baa5-4490-9cee-65198b7fbd9c in service fd00:1122:3344:2::1 + internal_ntp 1ec4cc7b-2f00-4d13-8176-3b9815533ae9 in service fd00:1122:3344:101::21 + nexus 3d4143df-e212-4774-9258-7d9b421fac2e in service fd00:1122:3344:101::22 + nexus 845869e9-ecb2-4ec3-b6b8-2a836e459243 in service fd00:1122:3344:101::2f @@ -59,45 +61,49 @@ parent: 516e80a3-b362-4fac-bd3c-4559717120dd omicron zones at generation 3: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 0e2b035e-1de1-48af-8ac0-5316418e3de1 in service fd00:1122:3344:102::27 - crucible 2bf9ee97-90e1-48a7-bb06-a35cec63b7fe in service fd00:1122:3344:102::2b - crucible 5c78756d-6182-4c27-a507-3419e8dbe76b in service fd00:1122:3344:102::25 - crucible b7402110-d88f-4ca4-8391-4a2fda6ad271 in service fd00:1122:3344:102::26 - crucible b7ae596e-0c85-40b2-bb47-df9f76db3cca in service fd00:1122:3344:102::28 - crucible c552280f-ba02-4f8d-9049-bd269e6b7845 in service fd00:1122:3344:102::23 - crucible cf13b878-47f1-4ba0-b8c2-9f3e15f2ee87 in service fd00:1122:3344:102::29 - crucible e3bfcb1e-3708-45e7-a45a-2a2cab7ad829 in service fd00:1122:3344:102::2c - crucible e6d0df1f-9f98-4c5a-9540-8444d1185c7d in service fd00:1122:3344:102::24 - crucible eb034526-1767-4cc4-8225-ec962265710b in service fd00:1122:3344:102::2a - internal_dns a308d3e1-118c-440a-947a-8b6ab7d833ab in service fd00:1122:3344:3::1 - internal_dns c8851a11-a4f7-4b21-9281-6182fd15dc8d in service fd00:1122:3344:1::1 - internal_ntp 4f8ce495-21dd-48a1-859c-80d34ce394ed in service fd00:1122:3344:102::21 - nexus a1ae92ac-e1f1-4654-ab54-5b75ba7c44d6 in service fd00:1122:3344:102::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse c8851a11-a4f7-4b21-9281-6182fd15dc8d in service fd00:1122:3344:102::2e + crucible 0e2b035e-1de1-48af-8ac0-5316418e3de1 in service fd00:1122:3344:102::24 + crucible 15f29557-d4da-45ef-b435-a0a1cd586e0c in service fd00:1122:3344:102::2a + crucible 2bf9ee97-90e1-48a7-bb06-a35cec63b7fe in service fd00:1122:3344:102::28 + crucible 5cf79919-b28e-4064-b6f8-8906c471b5ce in service fd00:1122:3344:102::2d + crucible 751bc6fe-22ad-4ce1-bc51-cf31fdf02bfa in service fd00:1122:3344:102::2b + crucible b7ae596e-0c85-40b2-bb47-df9f76db3cca in service fd00:1122:3344:102::25 + crucible cf13b878-47f1-4ba0-b8c2-9f3e15f2ee87 in service fd00:1122:3344:102::26 + crucible e3bfcb1e-3708-45e7-a45a-2a2cab7ad829 in service fd00:1122:3344:102::29 + crucible e5121f83-faf2-4928-b5a8-94a1da99e8eb in service fd00:1122:3344:102::2c + crucible eb034526-1767-4cc4-8225-ec962265710b in service fd00:1122:3344:102::27 + crucible_pantry b7402110-d88f-4ca4-8391-4a2fda6ad271 in service fd00:1122:3344:102::23 + internal_dns 5c78756d-6182-4c27-a507-3419e8dbe76b in service fd00:1122:3344:3::1 + internal_dns e639b672-27c4-4ecb-82c1-d672eb1ccf4e in service fd00:1122:3344:1::1 + internal_ntp c552280f-ba02-4f8d-9049-bd269e6b7845 in service fd00:1122:3344:102::21 + nexus e6d0df1f-9f98-4c5a-9540-8444d1185c7d in service fd00:1122:3344:102::22 !a1b477db-b629-48eb-911d-1ccdafca75b9 WARNING: Zones exist without physical disks! omicron zones at generation 3: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 1e1ed0cc-1adc-410f-943a-d1a3107de619 expunged fd00:1122:3344:103::26 - crucible 2307bbed-02ba-493b-89e3-46585c74c8fc expunged fd00:1122:3344:103::27 - crucible 603e629d-2599-400e-b879-4134d4cc426e expunged fd00:1122:3344:103::2b - crucible 9179d6dc-387d-424e-8d62-ed59b2c728f6 expunged fd00:1122:3344:103::29 - crucible ad76d200-5675-444b-b19c-684689ff421f expunged fd00:1122:3344:103::2c - crucible c28d7b4b-a259-45ad-945d-f19ca3c6964c expunged fd00:1122:3344:103::28 - crucible e29998e7-9ed2-46b6-bb70-4118159fe07f expunged fd00:1122:3344:103::25 - crucible f06e91a1-0c17-4cca-adbc-1c9b67bdb11d expunged fd00:1122:3344:103::2a - crucible f11f5c60-1ac7-4630-9a3a-a9bc85c75203 expunged fd00:1122:3344:103::24 - crucible f231e4eb-3fc9-4964-9d71-2c41644852d9 expunged fd00:1122:3344:103::23 - internal_dns 4e36b7ef-5684-4304-b7c3-3c31aaf83d4f expunged fd00:1122:3344:1::1 - internal_ntp c62b87b6-b98d-4d22-ba4f-cee4499e2ba8 expunged fd00:1122:3344:103::21 - nexus 6a70a233-1900-43c0-9c00-aa9d1f7adfbc expunged fd00:1122:3344:103::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse 4e36b7ef-5684-4304-b7c3-3c31aaf83d4f expunged fd00:1122:3344:103::23 + crucible 1e1ed0cc-1adc-410f-943a-d1a3107de619 expunged fd00:1122:3344:103::26 + crucible 2307bbed-02ba-493b-89e3-46585c74c8fc expunged fd00:1122:3344:103::27 + crucible 2e65b765-5c41-4519-bf4e-e2a68569afc1 expunged fd00:1122:3344:103::2e + crucible 603e629d-2599-400e-b879-4134d4cc426e expunged fd00:1122:3344:103::2b + crucible 9179d6dc-387d-424e-8d62-ed59b2c728f6 expunged fd00:1122:3344:103::29 + crucible ad76d200-5675-444b-b19c-684689ff421f expunged fd00:1122:3344:103::2c + crucible c28d7b4b-a259-45ad-945d-f19ca3c6964c expunged fd00:1122:3344:103::28 + crucible e29998e7-9ed2-46b6-bb70-4118159fe07f expunged fd00:1122:3344:103::25 + crucible e9bf2525-5fa0-4c1b-b52d-481225083845 expunged fd00:1122:3344:103::2d + crucible f06e91a1-0c17-4cca-adbc-1c9b67bdb11d expunged fd00:1122:3344:103::2a + crucible_pantry f11f5c60-1ac7-4630-9a3a-a9bc85c75203 expunged fd00:1122:3344:103::24 + internal_dns f231e4eb-3fc9-4964-9d71-2c41644852d9 expunged fd00:1122:3344:1::1 + internal_ntp c62b87b6-b98d-4d22-ba4f-cee4499e2ba8 expunged fd00:1122:3344:103::21 + nexus 6a70a233-1900-43c0-9c00-aa9d1f7adfbc expunged fd00:1122:3344:103::22 @@ -108,7 +114,7 @@ WARNING: Zones exist without physical disks! METADATA: created by::::::::::: test_blueprint2 created at::::::::::: 1970-01-01T00:00:00.000Z - comment:::::::::::::: sled a1b477db-b629-48eb-911d-1ccdafca75b9: expunged 13 zones because: sled policy is expunged + comment:::::::::::::: sled a1b477db-b629-48eb-911d-1ccdafca75b9: expunged 15 zones because: sled policy is expunged internal DNS version: 1 external DNS version: 1 diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt index 18199e157e..72832a0b9c 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_1_2.txt @@ -22,22 +22,24 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be in service fd00:1122:3344:105::25 - crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b - crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c - crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 - crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 - crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 - crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::23 - crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 - crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a - crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 - internal_dns 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:1::1 - internal_ntp 7f4e9f9f-08f8-4d14-885d-e977c05525ad in service fd00:1122:3344:105::21 - nexus 6dff7633-66bb-4924-a6ff-2c896e66964b in service fd00:1122:3344:105::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:105::23 + crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be in service fd00:1122:3344:105::25 + crucible 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb in service fd00:1122:3344:105::2d + crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b + crucible 67622d61-2df4-414d-aa0e-d1277265f405 in service fd00:1122:3344:105::2e + crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c + crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 + crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 + crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 + crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a + crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 + crucible_pantry 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 + internal_dns c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:1::1 + internal_ntp 7f4e9f9f-08f8-4d14-885d-e977c05525ad in service fd00:1122:3344:105::21 + nexus 6dff7633-66bb-4924-a6ff-2c896e66964b in service fd00:1122:3344:105::22 MODIFIED SLEDS: @@ -61,35 +63,37 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 omicron zones generation 2 -> 3: - ------------------------------------------------------------------------------------------- - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------- -* crucible 01d58626-e1b0-480f-96be-ac784863c7dc - in service fd00:1122:3344:103::2c - └─ + expunged -* crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 - in service fd00:1122:3344:103::2a - └─ + expunged -* crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea - in service fd00:1122:3344:103::23 - └─ + expunged -* crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f - in service fd00:1122:3344:103::25 - └─ + expunged -* crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 - in service fd00:1122:3344:103::26 - └─ + expunged -* crucible b91b271d-8d80-4f49-99a0-34006ae86063 - in service fd00:1122:3344:103::28 - └─ + expunged -* crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 - in service fd00:1122:3344:103::24 - └─ + expunged -* crucible e39d7c9e-182b-48af-af87-58079d723583 - in service fd00:1122:3344:103::27 - └─ + expunged -* crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d - in service fd00:1122:3344:103::2b - └─ + expunged -* crucible f69f92a1-5007-4bb0-a85b-604dc217154b - in service fd00:1122:3344:103::29 - └─ + expunged -* internal_dns 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb - in service fd00:1122:3344:2::1 - └─ + expunged -* internal_ntp 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb - in service fd00:1122:3344:103::21 - └─ + expunged -* nexus 67622d61-2df4-414d-aa0e-d1277265f405 - in service fd00:1122:3344:103::22 - └─ + expunged + ---------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + ---------------------------------------------------------------------------------------------- +* crucible 01d58626-e1b0-480f-96be-ac784863c7dc - in service fd00:1122:3344:103::2a + └─ + expunged +* crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 - in service fd00:1122:3344:103::28 + └─ + expunged +* crucible 47a87c6e-ef45-4d52-9a3e-69cdd96737cc - in service fd00:1122:3344:103::2b + └─ + expunged +* crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 - in service fd00:1122:3344:103::24 + └─ + expunged +* crucible 6464d025-4652-4948-919e-740bec5699b1 - in service fd00:1122:3344:103::2c + └─ + expunged +* crucible 878dfddd-3113-4197-a3ea-e0d4dbe9b476 - in service fd00:1122:3344:103::2d + └─ + expunged +* crucible b91b271d-8d80-4f49-99a0-34006ae86063 - in service fd00:1122:3344:103::26 + └─ + expunged +* crucible e39d7c9e-182b-48af-af87-58079d723583 - in service fd00:1122:3344:103::25 + └─ + expunged +* crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d - in service fd00:1122:3344:103::29 + └─ + expunged +* crucible f69f92a1-5007-4bb0-a85b-604dc217154b - in service fd00:1122:3344:103::27 + └─ + expunged +* crucible_pantry 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f - in service fd00:1122:3344:103::23 + └─ + expunged +* internal_dns d6ee1338-3127-43ec-9aaa-b973ccf05496 - in service fd00:1122:3344:2::1 + └─ + expunged +* internal_ntp 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb - in service fd00:1122:3344:103::21 + └─ + expunged +* nexus 0dcfdfc5-481e-4153-b97c-11cf02b648ea - in service fd00:1122:3344:103::22 + └─ + expunged sled 68d24ac5-f341-49ea-a92a-0381b52ab387 (active): @@ -111,22 +115,23 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::29 - crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::2a - crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::26 - crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::28 - crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::23 - crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::24 - crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::2b - crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::25 - crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::27 - crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::2c - internal_dns 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:3::1 - internal_ntp 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::21 - nexus 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 15bb9def-69b8-4d2e-b04f-9fee1143387c expunged fd00:1122:3344:102::2b + crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::26 + crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::27 + crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::25 + crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 expunged fd00:1122:3344:102::2c + crucible b1783e95-9598-451d-b6ba-c50b52b428c3 expunged fd00:1122:3344:102::2a + crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::28 + crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::24 + crucible c6dd531e-2d1d-423b-acc8-358533dab78c expunged fd00:1122:3344:102::2d + crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::29 + crucible_pantry 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::23 + internal_dns b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:3::1 + internal_ntp 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::21 + nexus b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::22 sled 75bc286f-2b4b-482c-9431-59272af529da (active): @@ -151,18 +156,18 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::2b - crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::29 - crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::25 - crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::27 - crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::2c - crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:104::23 - crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::26 - crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::24 - crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::28 - crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::2a - internal_ntp b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::21 - nexus 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::22 + crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:104::2c + crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::27 + crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::25 + crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:104::29 + crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:104::2b + crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::23 + crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::28 + crucible c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:104::2a + crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::24 + crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::26 + internal_ntp 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::21 + nexus a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::22 + nexus 2ec75441-3d7d-4b4b-9614-af03de5a3666 in service fd00:1122:3344:104::2d + nexus 508abd03-cbfe-4654-9a6d-7f15a1ad32e5 in service fd00:1122:3344:104::2e + nexus 59950bc8-1497-44dd-8cbf-b6502ba921b2 in service fd00:1122:3344:104::2f @@ -190,18 +195,18 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:101::24 - crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::26 - crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::23 - crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::2a - crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::27 - crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::29 - crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::2b - crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::25 - crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::28 - crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::2c - internal_ntp 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:101::21 - nexus c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::22 + crucible 414830dc-c8c1-4748-9e9e-bc3a6435a93c in service fd00:1122:3344:101::29 + crucible 66ecd4a6-73a7-4e26-9711-17abdd67a66e in service fd00:1122:3344:101::2c + crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::26 + crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::23 + crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::25 + crucible a73f322a-9463-4d18-8f60-7ddf6f59f231 in service fd00:1122:3344:101::2b + crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::27 + crucible be920398-024a-4655-8c49-69b5ac48dfff in service fd00:1122:3344:101::2a + crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::24 + crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::28 + internal_ntp bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::21 + nexus 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::22 + nexus 3ca5292f-8a59-4475-bb72-0f43714d0fff in service fd00:1122:3344:101::2e + nexus 99f6d544-8599-4e2b-a55a-82d9e0034662 in service fd00:1122:3344:101::2d + nexus c26b3bda-5561-44a1-a69f-22103fe209a1 in service fd00:1122:3344:101::2f diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt index 928730df53..7b2d023d46 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_2_2a.txt @@ -25,21 +25,21 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::2b - crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::29 - crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::25 - crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::27 - crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::2c - crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:104::23 - crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::26 - crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::24 - crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::28 - crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::2a - internal_ntp b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::21 - nexus 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::22 + crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:104::2c + crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::27 + crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::25 + crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:104::29 + crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:104::2b + crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::23 + crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::28 + crucible c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:104::2a + crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::24 + crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::26 + internal_ntp 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::21 nexus 2ec75441-3d7d-4b4b-9614-af03de5a3666 in service fd00:1122:3344:104::2d nexus 508abd03-cbfe-4654-9a6d-7f15a1ad32e5 in service fd00:1122:3344:104::2e nexus 59950bc8-1497-44dd-8cbf-b6502ba921b2 in service fd00:1122:3344:104::2f + nexus a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::22 sled affab35f-600a-4109-8ea0-34a067a4e0bc (active): @@ -64,21 +64,21 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:101::24 - crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::26 - crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::23 - crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::2a - crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::27 - crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::29 - crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::2b - crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::25 - crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::28 - crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::2c - internal_ntp 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:101::21 + crucible 414830dc-c8c1-4748-9e9e-bc3a6435a93c in service fd00:1122:3344:101::29 + crucible 66ecd4a6-73a7-4e26-9711-17abdd67a66e in service fd00:1122:3344:101::2c + crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::26 + crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::23 + crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::25 + crucible a73f322a-9463-4d18-8f60-7ddf6f59f231 in service fd00:1122:3344:101::2b + crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::27 + crucible be920398-024a-4655-8c49-69b5ac48dfff in service fd00:1122:3344:101::2a + crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::24 + crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::28 + internal_ntp bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::21 nexus 3ca5292f-8a59-4475-bb72-0f43714d0fff in service fd00:1122:3344:101::2e + nexus 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::22 nexus 99f6d544-8599-4e2b-a55a-82d9e0034662 in service fd00:1122:3344:101::2d nexus c26b3bda-5561-44a1-a69f-22103fe209a1 in service fd00:1122:3344:101::2f - nexus c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::22 REMOVED SLEDS: @@ -86,22 +86,23 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 sled 68d24ac5-f341-49ea-a92a-0381b52ab387 (was active): omicron zones from generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ -- crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::29 -- crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::2a -- crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::26 -- crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::28 -- crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::23 -- crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::24 -- crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::2b -- crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::25 -- crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::27 -- crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::2c -- internal_dns 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:3::1 -- internal_ntp 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::21 -- nexus 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- +- crucible 15bb9def-69b8-4d2e-b04f-9fee1143387c expunged fd00:1122:3344:102::2b +- crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::26 +- crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::27 +- crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::25 +- crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 expunged fd00:1122:3344:102::2c +- crucible b1783e95-9598-451d-b6ba-c50b52b428c3 expunged fd00:1122:3344:102::2a +- crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::28 +- crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::24 +- crucible c6dd531e-2d1d-423b-acc8-358533dab78c expunged fd00:1122:3344:102::2d +- crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::29 +- crucible_pantry 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::23 +- internal_dns b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:3::1 +- internal_ntp 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::21 +- nexus b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::22 MODIFIED SLEDS: @@ -125,42 +126,45 @@ to: blueprint 9f71f5d3-a272-4382-9154-6ea2e171a6c6 omicron zones at generation 2: - ------------------------------------------------------------------------------------------- - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------- - crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c - crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 - crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 - crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 - crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::23 - crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 - crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a - crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 - internal_dns 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:1::1 -- crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b -* crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be - in service fd00:1122:3344:105::25 - └─ + quiesced + ---------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + ---------------------------------------------------------------------------------------------- + clickhouse 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:105::23 + crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b + crucible 67622d61-2df4-414d-aa0e-d1277265f405 in service fd00:1122:3344:105::2e + crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c + crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 + crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 + crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 + crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a + crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 + crucible_pantry 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 + internal_dns c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:1::1 +- crucible 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb in service fd00:1122:3344:105::2d +* crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be - in service fd00:1122:3344:105::25 + └─ + quiesced sled 48d95fef-bc9f-4f50-9a53-1e075836291d (decommissioned): omicron zones generation 3 -> 4: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ -- crucible 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:103::2c -- crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 expunged fd00:1122:3344:103::2a -- crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea expunged fd00:1122:3344:103::23 -- crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f expunged fd00:1122:3344:103::25 -- crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 expunged fd00:1122:3344:103::26 -- crucible b91b271d-8d80-4f49-99a0-34006ae86063 expunged fd00:1122:3344:103::28 -- crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 expunged fd00:1122:3344:103::24 -- crucible e39d7c9e-182b-48af-af87-58079d723583 expunged fd00:1122:3344:103::27 -- crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:103::2b -- crucible f69f92a1-5007-4bb0-a85b-604dc217154b expunged fd00:1122:3344:103::29 -- internal_dns 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb expunged fd00:1122:3344:2::1 -- internal_ntp 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb expunged fd00:1122:3344:103::21 -- nexus 67622d61-2df4-414d-aa0e-d1277265f405 expunged fd00:1122:3344:103::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- +- crucible 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:103::2a +- crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 expunged fd00:1122:3344:103::28 +- crucible 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:103::2b +- crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 expunged fd00:1122:3344:103::24 +- crucible 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:103::2c +- crucible 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:103::2d +- crucible b91b271d-8d80-4f49-99a0-34006ae86063 expunged fd00:1122:3344:103::26 +- crucible e39d7c9e-182b-48af-af87-58079d723583 expunged fd00:1122:3344:103::25 +- crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:103::29 +- crucible f69f92a1-5007-4bb0-a85b-604dc217154b expunged fd00:1122:3344:103::27 +- crucible_pantry 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f expunged fd00:1122:3344:103::23 +- internal_dns d6ee1338-3127-43ec-9aaa-b973ccf05496 expunged fd00:1122:3344:2::1 +- internal_ntp 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb expunged fd00:1122:3344:103::21 +- nexus 0dcfdfc5-481e-4153-b97c-11cf02b648ea expunged fd00:1122:3344:103::22 ERRORS: diff --git a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt index 1b8d0fe658..04c119ea9f 100644 --- a/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt +++ b/nexus/reconfigurator/planning/tests/output/planner_nonprovisionable_bp2.txt @@ -20,22 +20,24 @@ parent: 4d4e6c38-cd95-4c4e-8f45-6af4d686964b omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be in service fd00:1122:3344:105::25 - crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b - crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c - crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 - crucible 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 - crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 - crucible c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:105::23 - crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 - crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a - crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 - internal_dns 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:1::1 - internal_ntp 7f4e9f9f-08f8-4d14-885d-e977c05525ad in service fd00:1122:3344:105::21 - nexus 6dff7633-66bb-4924-a6ff-2c896e66964b in service fd00:1122:3344:105::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + clickhouse 93b137a1-a1d6-4b5b-b2cb-21a9f11e2883 in service fd00:1122:3344:105::23 + crucible 19fbc4f8-a683-4f22-8f5a-e74782b935be in service fd00:1122:3344:105::25 + crucible 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb in service fd00:1122:3344:105::2d + crucible 4f1ce8a2-d3a5-4a38-be4c-9817de52db37 in service fd00:1122:3344:105::2b + crucible 67622d61-2df4-414d-aa0e-d1277265f405 in service fd00:1122:3344:105::2e + crucible 67d913e0-0005-4599-9b28-0abbf6cc2916 in service fd00:1122:3344:105::2c + crucible 6b53ab2e-d98c-485f-87a3-4d5df595390f in service fd00:1122:3344:105::26 + crucible b0c63f48-01ea-4aae-bb26-fb0dd59d1662 in service fd00:1122:3344:105::27 + crucible d660d7ed-28c0-45ae-9ace-dc3ecf7e8786 in service fd00:1122:3344:105::29 + crucible e98cc0de-abf6-4da4-a20d-d05c7a9bb1d7 in service fd00:1122:3344:105::2a + crucible f55e6aaf-e8fc-4913-9e3c-8cd1bd4bdad3 in service fd00:1122:3344:105::28 + crucible_pantry 9f0abbad-dbd3-4d43-9675-78092217ffd9 in service fd00:1122:3344:105::24 + internal_dns c406da50-34b9-4bb4-a460-8f49875d2a6a in service fd00:1122:3344:1::1 + internal_ntp 7f4e9f9f-08f8-4d14-885d-e977c05525ad in service fd00:1122:3344:105::21 + nexus 6dff7633-66bb-4924-a6ff-2c896e66964b in service fd00:1122:3344:105::22 @@ -61,21 +63,21 @@ parent: 4d4e6c38-cd95-4c4e-8f45-6af4d686964b ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::2b - crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::29 - crucible 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::25 - crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::27 - crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::2c - crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 in service fd00:1122:3344:104::23 - crucible a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::26 - crucible c6dd531e-2d1d-423b-acc8-358533dab78c in service fd00:1122:3344:104::24 - crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::28 - crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::2a - internal_ntp b1783e95-9598-451d-b6ba-c50b52b428c3 in service fd00:1122:3344:104::21 - nexus 15bb9def-69b8-4d2e-b04f-9fee1143387c in service fd00:1122:3344:104::22 + crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:104::2c + crucible 15c103f0-ac63-423b-ba5d-1b5fcd563ba3 in service fd00:1122:3344:104::27 + crucible 23a8fa2b-ef3e-4017-a43f-f7a83953bd7c in service fd00:1122:3344:104::25 + crucible 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:104::29 + crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:104::2b + crucible 85b8c68a-160d-461d-94dd-1baf175fa75c in service fd00:1122:3344:104::23 + crucible 95482c25-1e7f-43e8-adf1-e3548a1b3ae0 in service fd00:1122:3344:104::28 + crucible c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:104::2a + crucible f0ff59e8-4105-4980-a4bb-a1f4c58de1e3 in service fd00:1122:3344:104::24 + crucible f1a7b9a7-fc6a-4b23-b829-045ff33117ff in service fd00:1122:3344:104::26 + internal_ntp 621509d6-3772-4009-aca1-35eefd1098fb in service fd00:1122:3344:104::21 nexus 2ec75441-3d7d-4b4b-9614-af03de5a3666 in service fd00:1122:3344:104::2d nexus 508abd03-cbfe-4654-9a6d-7f15a1ad32e5 in service fd00:1122:3344:104::2e nexus 59950bc8-1497-44dd-8cbf-b6502ba921b2 in service fd00:1122:3344:104::2f + nexus a732c489-d29a-4f75-b900-5966385943af in service fd00:1122:3344:104::22 @@ -101,43 +103,44 @@ parent: 4d4e6c38-cd95-4c4e-8f45-6af4d686964b ------------------------------------------------------------------------------------------ zone type zone id disposition underlay IP ------------------------------------------------------------------------------------------ - crucible 0dfbf374-9ef9-430f-b06d-f271bf7f84c4 in service fd00:1122:3344:101::24 - crucible 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::26 - crucible 72c5a909-077d-4ec1-a9d5-ae64ef9d716e in service fd00:1122:3344:101::23 - crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::2a - crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::27 - crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::29 - crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::2b - crucible bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::25 - crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::28 - crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::2c - internal_ntp 3aa07966-5899-4789-ace5-f8eeb375c6c3 in service fd00:1122:3344:101::21 + crucible 414830dc-c8c1-4748-9e9e-bc3a6435a93c in service fd00:1122:3344:101::29 + crucible 66ecd4a6-73a7-4e26-9711-17abdd67a66e in service fd00:1122:3344:101::2c + crucible 772cbcbd-58be-4158-be85-be744871fa22 in service fd00:1122:3344:101::26 + crucible a1c03689-fc62-4ea5-bb72-4d01f5138614 in service fd00:1122:3344:101::23 + crucible a568e92e-4fbd-4b69-acd8-f16277073031 in service fd00:1122:3344:101::25 + crucible a73f322a-9463-4d18-8f60-7ddf6f59f231 in service fd00:1122:3344:101::2b + crucible be75764a-491b-4aec-992e-1c39e25de975 in service fd00:1122:3344:101::27 + crucible be920398-024a-4655-8c49-69b5ac48dfff in service fd00:1122:3344:101::2a + crucible d47f4996-fac0-4657-bcea-01b1fee6404d in service fd00:1122:3344:101::24 + crucible e001fea0-6594-4ece-97e3-6198c293e931 in service fd00:1122:3344:101::28 + internal_ntp bf79a56a-97af-4cc4-94a5-8b20d64c2cda in service fd00:1122:3344:101::21 nexus 3ca5292f-8a59-4475-bb72-0f43714d0fff in service fd00:1122:3344:101::2e + nexus 4ad0e9da-08f8-4d40-b4d3-d17e711b5bbf in service fd00:1122:3344:101::22 nexus 99f6d544-8599-4e2b-a55a-82d9e0034662 in service fd00:1122:3344:101::2d nexus c26b3bda-5561-44a1-a69f-22103fe209a1 in service fd00:1122:3344:101::2f - nexus c60379ba-4e30-4628-a79a-0ae509aef4c5 in service fd00:1122:3344:101::22 !48d95fef-bc9f-4f50-9a53-1e075836291d WARNING: Zones exist without physical disks! omicron zones at generation 3: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:103::2c - crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 expunged fd00:1122:3344:103::2a - crucible 0dcfdfc5-481e-4153-b97c-11cf02b648ea expunged fd00:1122:3344:103::23 - crucible 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f expunged fd00:1122:3344:103::25 - crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 expunged fd00:1122:3344:103::26 - crucible b91b271d-8d80-4f49-99a0-34006ae86063 expunged fd00:1122:3344:103::28 - crucible d6ee1338-3127-43ec-9aaa-b973ccf05496 expunged fd00:1122:3344:103::24 - crucible e39d7c9e-182b-48af-af87-58079d723583 expunged fd00:1122:3344:103::27 - crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:103::2b - crucible f69f92a1-5007-4bb0-a85b-604dc217154b expunged fd00:1122:3344:103::29 - internal_dns 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb expunged fd00:1122:3344:2::1 - internal_ntp 2aa0ea4f-3561-4989-a98c-9ab7d9a240fb expunged fd00:1122:3344:103::21 - nexus 67622d61-2df4-414d-aa0e-d1277265f405 expunged fd00:1122:3344:103::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 01d58626-e1b0-480f-96be-ac784863c7dc expunged fd00:1122:3344:103::2a + crucible 094f27af-1acb-4d1e-ba97-1fc1377d4bf2 expunged fd00:1122:3344:103::28 + crucible 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:103::2b + crucible 4a9a0a9d-87f0-4f1d-9181-27f6b435e637 expunged fd00:1122:3344:103::24 + crucible 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:103::2c + crucible 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:103::2d + crucible b91b271d-8d80-4f49-99a0-34006ae86063 expunged fd00:1122:3344:103::26 + crucible e39d7c9e-182b-48af-af87-58079d723583 expunged fd00:1122:3344:103::25 + crucible f3f2e4f3-0985-4ef6-8336-ce479382d05d expunged fd00:1122:3344:103::29 + crucible f69f92a1-5007-4bb0-a85b-604dc217154b expunged fd00:1122:3344:103::27 + crucible_pantry 2f5e8010-a94d-43a4-9c5c-3f52832f5f7f expunged fd00:1122:3344:103::23 + internal_dns d6ee1338-3127-43ec-9aaa-b973ccf05496 expunged fd00:1122:3344:2::1 + internal_ntp 56ac1706-9e2a-49ba-bd6f-a99c44cb2ccb expunged fd00:1122:3344:103::21 + nexus 0dcfdfc5-481e-4153-b97c-11cf02b648ea expunged fd00:1122:3344:103::22 @@ -145,22 +148,23 @@ WARNING: Zones exist without physical disks! !68d24ac5-f341-49ea-a92a-0381b52ab387 WARNING: Zones exist without physical disks! omicron zones at generation 2: - ------------------------------------------------------------------------------------------ - zone type zone id disposition underlay IP - ------------------------------------------------------------------------------------------ - crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::29 - crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::2a - crucible 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::26 - crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::28 - crucible 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::23 - crucible b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::24 - crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::2b - crucible b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:102::25 - crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::27 - crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::2c - internal_dns 878dfddd-3113-4197-a3ea-e0d4dbe9b476 expunged fd00:1122:3344:3::1 - internal_ntp 47a87c6e-ef45-4d52-9a3e-69cdd96737cc expunged fd00:1122:3344:102::21 - nexus 6464d025-4652-4948-919e-740bec5699b1 expunged fd00:1122:3344:102::22 + --------------------------------------------------------------------------------------------- + zone type zone id disposition underlay IP + --------------------------------------------------------------------------------------------- + crucible 15bb9def-69b8-4d2e-b04f-9fee1143387c expunged fd00:1122:3344:102::2b + crucible 3b3c14b6-a8e2-4054-a577-8d96cb576230 expunged fd00:1122:3344:102::26 + crucible 57b96d5c-b71e-43e4-8869-7d514003d00d expunged fd00:1122:3344:102::27 + crucible 8d4d2b28-82bb-4e36-80da-1408d8c35d82 expunged fd00:1122:3344:102::25 + crucible 996d7570-b0df-46d5-aaa4-0c97697cf484 expunged fd00:1122:3344:102::2c + crucible b1783e95-9598-451d-b6ba-c50b52b428c3 expunged fd00:1122:3344:102::2a + crucible b4947d31-f70e-4ee0-8817-0ca6cea9b16b expunged fd00:1122:3344:102::28 + crucible c407795c-6c8b-428e-8ab8-b962913c447f expunged fd00:1122:3344:102::24 + crucible c6dd531e-2d1d-423b-acc8-358533dab78c expunged fd00:1122:3344:102::2d + crucible e4b3e159-3dbe-48cb-8497-e3da92a90e5a expunged fd00:1122:3344:102::29 + crucible_pantry 6939ce48-b17c-4616-b176-8a419a7697be expunged fd00:1122:3344:102::23 + internal_dns b6b759d0-f60d-42b7-bbbc-9d61c9e895a9 expunged fd00:1122:3344:3::1 + internal_ntp 9fd52961-426f-4e62-a644-b70871103fca expunged fd00:1122:3344:102::21 + nexus b44cdbc0-0ce0-46eb-8b21-a09e113aa1d0 expunged fd00:1122:3344:102::22 @@ -171,7 +175,7 @@ WARNING: Zones exist without physical disks! METADATA: created by::::::::::: test_blueprint2 created at::::::::::: 1970-01-01T00:00:00.000Z - comment:::::::::::::: sled 48d95fef-bc9f-4f50-9a53-1e075836291d: expunged 13 zones because: sled policy is expunged + comment:::::::::::::: sled 48d95fef-bc9f-4f50-9a53-1e075836291d: expunged 14 zones because: sled policy is expunged internal DNS version: 1 external DNS version: 1 diff --git a/nexus/reconfigurator/preparation/src/lib.rs b/nexus/reconfigurator/preparation/src/lib.rs index ac0f990ac2..9e14289e8a 100644 --- a/nexus/reconfigurator/preparation/src/lib.rs +++ b/nexus/reconfigurator/preparation/src/lib.rs @@ -40,8 +40,10 @@ use omicron_common::api::external::LookupType; use omicron_common::disk::DiskIdentity; use omicron_common::policy::BOUNDARY_NTP_REDUNDANCY; use omicron_common::policy::COCKROACHDB_REDUNDANCY; +use omicron_common::policy::CRUCIBLE_PANTRY_REDUNDANCY; use omicron_common::policy::INTERNAL_DNS_REDUNDANCY; use omicron_common::policy::NEXUS_REDUNDANCY; +use omicron_common::policy::OXIMETER_REDUNDANCY; use omicron_uuid_kinds::GenericUuid; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::PhysicalDiskUuid; @@ -66,8 +68,10 @@ pub struct PlanningInputFromDb<'a> { pub target_boundary_ntp_zone_count: usize, pub target_nexus_zone_count: usize, pub target_internal_dns_zone_count: usize, + pub target_oximeter_zone_count: usize, pub target_cockroachdb_zone_count: usize, pub target_cockroachdb_cluster_version: CockroachDbClusterVersion, + pub target_crucible_pantry_zone_count: usize, pub internal_dns_version: nexus_db_model::Generation, pub external_dns_version: nexus_db_model::Generation, pub cockroachdb_settings: &'a CockroachDbSettings, @@ -141,9 +145,11 @@ impl PlanningInputFromDb<'_> { target_boundary_ntp_zone_count: BOUNDARY_NTP_REDUNDANCY, target_nexus_zone_count: NEXUS_REDUNDANCY, target_internal_dns_zone_count: INTERNAL_DNS_REDUNDANCY, + target_oximeter_zone_count: OXIMETER_REDUNDANCY, target_cockroachdb_zone_count: COCKROACHDB_REDUNDANCY, target_cockroachdb_cluster_version: CockroachDbClusterVersion::POLICY, + target_crucible_pantry_zone_count: CRUCIBLE_PANTRY_REDUNDANCY, external_ip_rows: &external_ip_rows, service_nic_rows: &service_nic_rows, log: &opctx.log, @@ -165,9 +171,12 @@ impl PlanningInputFromDb<'_> { target_boundary_ntp_zone_count: self.target_boundary_ntp_zone_count, target_nexus_zone_count: self.target_nexus_zone_count, target_internal_dns_zone_count: self.target_internal_dns_zone_count, + target_oximeter_zone_count: self.target_oximeter_zone_count, target_cockroachdb_zone_count: self.target_cockroachdb_zone_count, target_cockroachdb_cluster_version: self .target_cockroachdb_cluster_version, + target_crucible_pantry_zone_count: self + .target_crucible_pantry_zone_count, clickhouse_policy: None, }; let mut builder = PlanningInputBuilder::new( diff --git a/nexus/src/app/background/init.rs b/nexus/src/app/background/init.rs index 69221779ee..fdd2fb7c90 100644 --- a/nexus/src/app/background/init.rs +++ b/nexus/src/app/background/init.rs @@ -857,7 +857,7 @@ pub struct BackgroundTasksData { pub nexus_id: OmicronZoneUuid, /// internal DNS DNS resolver, used when tasks need to contact other /// internal services - pub resolver: internal_dns::resolver::Resolver, + pub resolver: internal_dns_resolver::Resolver, /// handle to saga subsystem for starting sagas pub saga_starter: Arc, /// Oximeter producer registry (for metrics) @@ -874,7 +874,7 @@ fn init_dns( opctx: &OpContext, datastore: Arc, dns_group: DnsGroup, - resolver: internal_dns::resolver::Resolver, + resolver: internal_dns_resolver::Resolver, config: &DnsTasksConfig, task_config: &Activator, task_servers: &Activator, @@ -949,12 +949,14 @@ pub mod test { use crate::app::saga::StartSaga; use dropshot::HandlerTaskMode; use futures::FutureExt; + use internal_dns_types::names::ServiceName; use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder; use nexus_db_queries::db::DataStore; use nexus_test_utils_macros::nexus_test; use nexus_types::internal_api::params as nexus_params; + use nexus_types::internal_api::params::DnsRecord; use omicron_common::api::external::Error; use omicron_test_utils::dev::poll; use std::net::SocketAddr; @@ -1057,8 +1059,7 @@ pub mod test { .expect("failed to get initial DNS server config"); assert_eq!(config.generation, 1); - let internal_dns_srv_name = - internal_dns::ServiceName::InternalDns.dns_name(); + let internal_dns_srv_name = ServiceName::InternalDns.dns_name(); let initial_srv_record = { let zone = @@ -1067,7 +1068,7 @@ pub mod test { panic!("zone must have a record for {internal_dns_srv_name}") }; match record.get(0) { - Some(dns_service_client::types::DnsRecord::Srv(srv)) => srv, + Some(DnsRecord::Srv(srv)) => srv, record => panic!( "expected a SRV record for {internal_dns_srv_name}, found \ {record:?}" diff --git a/nexus/src/app/background/tasks/bfd.rs b/nexus/src/app/background/tasks/bfd.rs index 67b15ee3d3..c37d3e5c58 100644 --- a/nexus/src/app/background/tasks/bfd.rs +++ b/nexus/src/app/background/tasks/bfd.rs @@ -12,7 +12,8 @@ use crate::app::{ use crate::app::background::BackgroundTask; use futures::future::BoxFuture; use futures::FutureExt; -use internal_dns::{resolver::Resolver, ServiceName}; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use mg_admin_client::types::{BfdPeerConfig, SessionMode}; use nexus_db_model::{BfdMode, BfdSession}; use nexus_db_queries::{context::OpContext, db::DataStore}; diff --git a/nexus/src/app/background/tasks/blueprint_execution.rs b/nexus/src/app/background/tasks/blueprint_execution.rs index c4fd916d5f..f229632f2f 100644 --- a/nexus/src/app/background/tasks/blueprint_execution.rs +++ b/nexus/src/app/background/tasks/blueprint_execution.rs @@ -7,7 +7,7 @@ use crate::app::background::{Activator, BackgroundTask}; use futures::future::BoxFuture; use futures::FutureExt; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_reconfigurator_execution::RealizeBlueprintOutput; @@ -247,6 +247,7 @@ mod test { internal_dns_version: dns_version, external_dns_version: dns_version, cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: chrono::Utc::now(), creator: "test".to_string(), comment: "test blueprint".to_string(), @@ -281,6 +282,34 @@ mod test { // sleds to CRDB. let mut s1 = httptest::Server::run(); let mut s2 = httptest::Server::run(); + + // The only sled-agent endpoint we care about in this test is `PUT + // /omicron-zones`. Add a closure to avoid repeating it multiple times + // below. We don't do a careful check of the _contents_ of what's being + // sent; for that, see the tests in nexus-reconfigurator-execution. + let match_put_omicron_zones = + || request::method_path("PUT", "/omicron-zones"); + + // Helper for our mock sled-agent http servers to blanket ignore and + // return 200 OK for anything _except_ `PUT /omciron-zones`, which is + // the endpoint we care about in this test. + // + // Other Nexus background tasks created by our test context may notice + // the sled-agent records we're about to insert into CRDB and query them + // (e.g., for inventory, vpc routing info, ...). We don't want those to + // cause spurious test failures, so just tell our sled-agents to accept + // any number of them. + let mock_server_ignore_spurious_http_requests = + |s: &mut httptest::Server| { + s.expect( + Expectation::matching(not(match_put_omicron_zones())) + .times(..) + .respond_with(status_code(200)), + ); + }; + mock_server_ignore_spurious_http_requests(&mut s1); + mock_server_ignore_spurious_http_requests(&mut s2); + let sled_id1 = SledUuid::new_v4(); let sled_id2 = SledUuid::new_v4(); let rack_id = Uuid::new_v4(); @@ -409,33 +438,6 @@ mod test { ) .await; - // The only sled-agent endpoint we care about in this test is `PUT - // /omicron-zones`. Add a closure to avoid repeating it multiple times - // below. We don't do a careful check of the _contents_ of what's being - // sent; for that, see the tests in nexus-reconfigurator-execution. - let match_put_omicron_zones = - || request::method_path("PUT", "/omicron-zones"); - - // Helper for our mock sled-agent http servers to blanket ignore and - // return 200 OK for anything _except_ `PUT /omciron-zones`, which is - // the endpoint we care about in this test. - // - // Other Nexus background tasks created by our test context may notice - // the sled-agent records we're about to insert into CRDB and query them - // (e.g., for inventory, vpc routing info, ...). We don't want those to - // cause spurious test failures, so just tell our sled-agents to accept - // any number of them. - let mock_server_ignore_spurious_http_requests = - |s: &mut httptest::Server| { - s.expect( - Expectation::matching(not(match_put_omicron_zones())) - .times(..) - .respond_with(status_code(200)), - ); - }; - mock_server_ignore_spurious_http_requests(&mut s1); - mock_server_ignore_spurious_http_requests(&mut s2); - // Insert records for the zpools backing the datasets in these zones. for (sled_id, config) in blueprint.1.all_omicron_zones(BlueprintZoneFilter::All) diff --git a/nexus/src/app/background/tasks/blueprint_load.rs b/nexus/src/app/background/tasks/blueprint_load.rs index 70fcf713bc..8b5c02dd80 100644 --- a/nexus/src/app/background/tasks/blueprint_load.rs +++ b/nexus/src/app/background/tasks/blueprint_load.rs @@ -225,6 +225,7 @@ mod test { internal_dns_version: Generation::new(), external_dns_version: Generation::new(), cockroachdb_fingerprint: String::new(), + clickhouse_cluster_config: None, time_created: now_db_precision(), creator: "test".to_string(), comment: "test blueprint".to_string(), diff --git a/nexus/src/app/background/tasks/dns_config.rs b/nexus/src/app/background/tasks/dns_config.rs index 1b0f627870..192724a89c 100644 --- a/nexus/src/app/background/tasks/dns_config.rs +++ b/nexus/src/app/background/tasks/dns_config.rs @@ -5,9 +5,9 @@ //! Background task for keeping track of DNS configuration use crate::app::background::BackgroundTask; -use dns_service_client::types::DnsConfigParams; use futures::future::BoxFuture; use futures::FutureExt; +use internal_dns_types::config::DnsConfigParams; use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; diff --git a/nexus/src/app/background/tasks/dns_propagation.rs b/nexus/src/app/background/tasks/dns_propagation.rs index c680a6f010..9dd698fa37 100644 --- a/nexus/src/app/background/tasks/dns_propagation.rs +++ b/nexus/src/app/background/tasks/dns_propagation.rs @@ -7,11 +7,11 @@ use super::dns_servers::DnsServersList; use crate::app::background::BackgroundTask; use anyhow::Context; -use dns_service_client::types::DnsConfigParams; use futures::future::BoxFuture; use futures::stream; use futures::FutureExt; use futures::StreamExt; +use internal_dns_types::config::DnsConfigParams; use nexus_db_queries::context::OpContext; use serde_json::json; use std::collections::BTreeMap; @@ -180,12 +180,12 @@ mod test { use super::DnsPropagator; use crate::app::background::tasks::dns_servers::DnsServersList; use crate::app::background::BackgroundTask; - use dns_service_client::types::DnsConfigParams; use httptest::matchers::request; use httptest::responders::status_code; use httptest::Expectation; use nexus_db_queries::context::OpContext; use nexus_test_utils_macros::nexus_test; + use nexus_types::internal_api::params::DnsConfigParams; use serde::Deserialize; use serde_json::json; use std::collections::BTreeMap; diff --git a/nexus/src/app/background/tasks/dns_servers.rs b/nexus/src/app/background/tasks/dns_servers.rs index 9d99460917..3b1e32a237 100644 --- a/nexus/src/app/background/tasks/dns_servers.rs +++ b/nexus/src/app/background/tasks/dns_servers.rs @@ -7,8 +7,8 @@ use crate::app::background::BackgroundTask; use futures::future::BoxFuture; use futures::FutureExt; -use internal_dns::names::ServiceName; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use serde::Serialize; diff --git a/nexus/src/app/background/tasks/instance_reincarnation.rs b/nexus/src/app/background/tasks/instance_reincarnation.rs index 25704df6a5..9657980c82 100644 --- a/nexus/src/app/background/tasks/instance_reincarnation.rs +++ b/nexus/src/app/background/tasks/instance_reincarnation.rs @@ -352,9 +352,10 @@ mod test { cptestctx: &ControlPlaneTestContext, opctx: &OpContext, name: &str, - auto_restart: InstanceAutoRestartPolicy, + auto_restart: impl Into>, state: InstanceState, ) -> authz::Instance { + let auto_restart_policy = auto_restart.into(); let instances_url = format!("/v1/instances?project={}", PROJECT_NAME); // Use the first chunk of the UUID as the name, to avoid conflicts. // Start with a lower ascii character to satisfy the name constraints. @@ -385,7 +386,7 @@ mod test { boot_disk: None, ssh_public_keys: None, start: state == InstanceState::Vmm, - auto_restart_policy: Some(auto_restart), + auto_restart_policy, }, ) .await; @@ -406,7 +407,8 @@ mod test { }; eprintln!( - "instance {id}: auto_restart_policy={auto_restart:?}; state={state:?}" + "instance {id}: auto_restart_policy={auto_restart_policy:?}; \ + state={state:?}" ); authz_instance } @@ -582,6 +584,51 @@ mod test { .await; } + #[nexus_test(server = crate::Server)] + async fn test_default_policy_is_reincarnatable( + cptestctx: &ControlPlaneTestContext, + ) { + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + + setup_test_project(&cptestctx, &opctx).await; + + let mut task = InstanceReincarnation::new( + datastore.clone(), + nexus.sagas.clone(), + false, + ); + + // Create an instance in the `Failed` state that's eligible to be + // restarted. + let instance = create_instance( + &cptestctx, + &opctx, + "my-cool-instance", + None, + InstanceState::Failed, + ) + .await; + + // Activate the task again, and check that our instance had an + // instance-start saga started. + let status = assert_activation_ok!(task.activate(&opctx).await); + assert_eq!(status.total_instances_found(), 1); + assert_eq!(status.instances_reincarnated, vec![failed(instance.id())]); + assert_eq!(status.changed_state, Vec::new()); + + test_helpers::instance_wait_for_state( + &cptestctx, + InstanceUuid::from_untyped_uuid(instance.id()), + InstanceState::Vmm, + ) + .await; + } + #[nexus_test(server = crate::Server)] async fn test_only_reincarnates_eligible_instances( cptestctx: &ControlPlaneTestContext, diff --git a/nexus/src/app/background/tasks/inventory_collection.rs b/nexus/src/app/background/tasks/inventory_collection.rs index 1e2d3bda1f..c4271c58d8 100644 --- a/nexus/src/app/background/tasks/inventory_collection.rs +++ b/nexus/src/app/background/tasks/inventory_collection.rs @@ -9,7 +9,7 @@ use anyhow::ensure; use anyhow::Context; use futures::future::BoxFuture; use futures::FutureExt; -use internal_dns::ServiceName; +use internal_dns_types::names::ServiceName; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use nexus_inventory::InventoryError; @@ -23,7 +23,7 @@ use tokio::sync::watch; /// Background task that reads inventory for the rack pub struct InventoryCollector { datastore: Arc, - resolver: internal_dns::resolver::Resolver, + resolver: internal_dns_resolver::Resolver, creator: String, nkeep: u32, disable: bool, @@ -33,7 +33,7 @@ pub struct InventoryCollector { impl InventoryCollector { pub fn new( datastore: Arc, - resolver: internal_dns::resolver::Resolver, + resolver: internal_dns_resolver::Resolver, creator: &str, nkeep: u32, disable: bool, @@ -99,7 +99,7 @@ impl BackgroundTask for InventoryCollector { async fn inventory_activate( opctx: &OpContext, datastore: &DataStore, - resolver: &internal_dns::resolver::Resolver, + resolver: &internal_dns_resolver::Resolver, creator: &str, nkeep: u32, disabled: bool, @@ -128,7 +128,21 @@ async fn inventory_activate( .map(|sockaddr| { let url = format!("http://{}", sockaddr); let log = opctx.log.new(o!("gateway_url" => url.clone())); - Arc::new(gateway_client::Client::new(&url, log)) + gateway_client::Client::new(&url, log) + }) + .collect::>(); + + // Find clickhouse-admin-keeper clients + let keeper_admin_clients = resolver + .lookup_socket_v6(ServiceName::ClickhouseAdminKeeper) + .await + .context("looking up ClickhouseAdminKeeper addresses") + .into_iter() + .map(|sockaddr| { + let url = format!("http://{}", sockaddr); + let log = + opctx.log.new(o!("clickhouse_admin_keeper_url" => url.clone())); + clickhouse_admin_keeper_client::Client::new(&url, log) }) .collect::>(); @@ -138,7 +152,8 @@ async fn inventory_activate( // Run a collection. let inventory = nexus_inventory::Collector::new( creator, - &mgs_clients, + mgs_clients, + keeper_admin_clients, &sled_enum, opctx.log.clone(), ); @@ -221,7 +236,7 @@ mod test { datastore.clone(), ); - let resolver = internal_dns::resolver::Resolver::new_from_addrs( + let resolver = internal_dns_resolver::Resolver::new_from_addrs( cptestctx.logctx.log.clone(), &[cptestctx.internal_dns.dns_server.local_address()], ) diff --git a/nexus/src/app/background/tasks/metrics_producer_gc.rs b/nexus/src/app/background/tasks/metrics_producer_gc.rs index 1df0afb7ed..7e06148041 100644 --- a/nexus/src/app/background/tasks/metrics_producer_gc.rs +++ b/nexus/src/app/background/tasks/metrics_producer_gc.rs @@ -116,10 +116,9 @@ mod tests { use httptest::Expectation; use nexus_db_model::OximeterInfo; use nexus_db_queries::context::OpContext; - use nexus_db_queries::db::model::ProducerEndpoint; use nexus_test_utils_macros::nexus_test; - use nexus_types::identity::Asset; use nexus_types::internal_api::params; + use omicron_common::api::external::DataPageParams; use omicron_common::api::internal::nexus; use omicron_common::api::internal::nexus::ProducerRegistrationResponse; use serde_json::json; @@ -158,17 +157,24 @@ mod tests { datastore.clone(), ); - let mut collector = httptest::Server::run(); - - // Insert an Oximeter collector - let collector_info = OximeterInfo::new(¶ms::OximeterInfo { - collector_id: Uuid::new_v4(), - address: collector.addr(), - }); - datastore - .oximeter_create(&opctx, &collector_info) + // Producer <-> collector assignment is random. We're going to create a + // mock collector below then insert a producer, and we want to guarantee + // the producer is assigned to the mock collector. To do so, we need to + // expunge the "real" collector set up by `nexus_test`. We'll phrase + // this as a loop to match the datastore methods and in case nexus_test + // ever starts multiple collectors. + for oximeter_info in datastore + .oximeter_list(&opctx, &DataPageParams::max_page()) .await - .expect("failed to insert collector"); + .expect("listed oximeters") + { + datastore + .oximeter_expunge(&opctx, oximeter_info.id) + .await + .expect("expunged oximeter"); + } + + let mut collector = httptest::Server::run(); // There are several producers which automatically register themselves // during tests, from Nexus and the simulated sled-agent for example. We @@ -184,18 +190,25 @@ mod tests { .respond_with(status_code(201).body(body)), ); + // Insert an Oximeter collector + let collector_info = OximeterInfo::new(¶ms::OximeterInfo { + collector_id: Uuid::new_v4(), + address: collector.addr(), + }); + datastore + .oximeter_create(&opctx, &collector_info) + .await + .expect("failed to insert collector"); + // Insert a producer. - let producer = ProducerEndpoint::new( - &nexus::ProducerEndpoint { - id: Uuid::new_v4(), - kind: nexus::ProducerKind::Service, - address: "[::1]:0".parse().unwrap(), // unused - interval: Duration::from_secs(0), // unused - }, - collector_info.id, - ); + let producer = nexus::ProducerEndpoint { + id: Uuid::new_v4(), + kind: nexus::ProducerKind::Service, + address: "[::1]:0".parse().unwrap(), // unused + interval: Duration::from_secs(0), // unused + }; datastore - .producer_endpoint_create(&opctx, &producer) + .producer_endpoint_upsert_and_assign(&opctx, &producer) .await .expect("failed to insert producer"); @@ -215,7 +228,7 @@ mod tests { // ago, which should result in it being pruned. set_time_modified( &datastore, - producer.id(), + producer.id, Utc::now() - chrono::TimeDelta::hours(2), ) .await; @@ -224,7 +237,7 @@ mod tests { collector.expect( Expectation::matching(request::method_path( "DELETE", - format!("/producers/{}", producer.id()), + format!("/producers/{}", producer.id), )) .respond_with(status_code(204)), ); @@ -235,7 +248,7 @@ mod tests { assert!(value.contains_key("expiration")); assert_eq!( *value.get("pruned").expect("missing `pruned`"), - json!([producer.id()]) + json!([producer.id]) ); collector.verify_and_clear(); diff --git a/nexus/src/app/background/tasks/nat_cleanup.rs b/nexus/src/app/background/tasks/nat_cleanup.rs index 675f4fc809..f8bfdc0bd9 100644 --- a/nexus/src/app/background/tasks/nat_cleanup.rs +++ b/nexus/src/app/background/tasks/nat_cleanup.rs @@ -13,8 +13,8 @@ use crate::app::background::BackgroundTask; use chrono::{Duration, Utc}; use futures::future::BoxFuture; use futures::FutureExt; -use internal_dns::resolver::Resolver; -use internal_dns::ServiceName; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::DataStore; use serde_json::json; diff --git a/nexus/src/app/background/tasks/region_replacement.rs b/nexus/src/app/background/tasks/region_replacement.rs index b1c717c77b..f86ba8eb8f 100644 --- a/nexus/src/app/background/tasks/region_replacement.rs +++ b/nexus/src/app/background/tasks/region_replacement.rs @@ -170,8 +170,79 @@ impl BackgroundTask for RegionReplacementDetector { }; for request in requests { + // If the replacement request is in the `requested` state and + // the request's volume was soft-deleted or hard-deleted, avoid + // sending the start request and instead transition the request + // to completed + + let volume_deleted = match self + .datastore + .volume_deleted(request.volume_id) + .await + { + Ok(volume_deleted) => volume_deleted, + + Err(e) => { + let s = format!( + "error checking if volume id {} was deleted: {e}", + request.volume_id, + ); + error!(&log, "{s}"); + + status.errors.push(s); + continue; + } + }; + let request_id = request.id; + if volume_deleted { + // Volume was soft or hard deleted, so proceed with clean + // up, which if this is in state Requested there won't be + // any additional associated state, so transition the record + // to Completed. + + info!( + &log, + "request {} volume {} was soft or hard deleted!", + request_id, + request.volume_id, + ); + + let result = self + .datastore + .set_region_replacement_complete_from_requested( + opctx, request, + ) + .await; + + match result { + Ok(()) => { + let s = format!( + "request {} transitioned from requested to \ + complete", + request_id, + ); + + info!(&log, "{s}"); + status.requests_completed_ok.push(s); + } + + Err(e) => { + let s = format!( + "error transitioning {} from requested to \ + complete: {e}", + request_id, + ); + + error!(&log, "{s}"); + status.errors.push(s); + } + } + + continue; + } + let result = self .send_start_request( authn::saga::Serialized::for_opctx(opctx), @@ -213,7 +284,10 @@ mod test { use super::*; use crate::app::background::init::test::NoopStartSaga; use nexus_db_model::RegionReplacement; + use nexus_db_model::Volume; use nexus_test_utils_macros::nexus_test; + use sled_agent_client::types::CrucibleOpts; + use sled_agent_client::types::VolumeConstructionRequest; use uuid::Uuid; type ControlPlaneTestContext = @@ -239,7 +313,8 @@ mod test { assert_eq!(result, json!(RegionReplacementStatus::default())); // Add a region replacement request for a fake region - let request = RegionReplacement::new(Uuid::new_v4(), Uuid::new_v4()); + let volume_id = Uuid::new_v4(); + let request = RegionReplacement::new(Uuid::new_v4(), volume_id); let request_id = request.id; datastore @@ -247,6 +322,40 @@ mod test { .await .unwrap(); + let volume_construction_request = VolumeConstructionRequest::Volume { + id: volume_id, + block_size: 0, + sub_volumes: vec![VolumeConstructionRequest::Region { + block_size: 0, + blocks_per_extent: 0, + extent_count: 0, + gen: 0, + opts: CrucibleOpts { + id: volume_id, + target: vec![ + // if you put something here, you'll need a synthetic + // dataset record + ], + lossy: false, + flush_timeout: None, + key: None, + cert_pem: None, + key_pem: None, + root_cert_pem: None, + control: None, + read_only: false, + }, + }], + read_only_parent: None, + }; + + let volume_data = + serde_json::to_string(&volume_construction_request).unwrap(); + + let volume = Volume::new(volume_id, volume_data); + + datastore.volume_create(volume).await.unwrap(); + // Activate the task - it should pick that up and try to run the region // replacement start saga let result: RegionReplacementStatus = diff --git a/nexus/src/app/background/tasks/sync_service_zone_nat.rs b/nexus/src/app/background/tasks/sync_service_zone_nat.rs index 4fbef3ae2e..972c9702cf 100644 --- a/nexus/src/app/background/tasks/sync_service_zone_nat.rs +++ b/nexus/src/app/background/tasks/sync_service_zone_nat.rs @@ -12,8 +12,8 @@ use crate::app::background::BackgroundTask; use anyhow::Context; use futures::future::BoxFuture; use futures::FutureExt; -use internal_dns::resolver::Resolver; -use internal_dns::ServiceName; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use nexus_db_model::Ipv4NatValues; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::lookup::LookupPath; @@ -107,7 +107,7 @@ impl BackgroundTask for ServiceZoneNatTracker { let mut nexus_count = 0; let mut dns_count = 0; - for (sled_id, zones_found) in collection.omicron_zones { + for (sled_id, sa) in collection.sled_agents { let (_, sled) = match LookupPath::new(opctx, &self.datastore) .sled_id(sled_id.into_untyped_uuid()) .fetch() @@ -128,7 +128,7 @@ impl BackgroundTask for ServiceZoneNatTracker { let sled_address = oxnet::Ipv6Net::host_net(*sled.ip); - let zones_config: OmicronZonesConfig = zones_found.zones; + let zones_config: OmicronZonesConfig = sa.omicron_zones; let zones: Vec = zones_config.zones; for zone in zones { diff --git a/nexus/src/app/background/tasks/sync_switch_configuration.rs b/nexus/src/app/background/tasks/sync_switch_configuration.rs index fae97233c3..49dbfb2e52 100644 --- a/nexus/src/app/background/tasks/sync_switch_configuration.rs +++ b/nexus/src/app/background/tasks/sync_switch_configuration.rs @@ -14,8 +14,8 @@ use crate::app::{ use oxnet::Ipv4Net; use slog::o; -use internal_dns::resolver::Resolver; -use internal_dns::ServiceName; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use ipnetwork::IpNetwork; use nexus_db_model::{ AddressLotBlock, BgpConfig, BootstoreConfig, LoopbackAddress, @@ -71,6 +71,10 @@ const PHY0: &str = "phy0"; // before breaking to check for shutdown conditions. const BGP_SESSION_RESOLUTION: u64 = 100; +// This is the default RIB Priority used for static routes. This mirrors +// the const defined in maghemite in rdb/src/lib.rs. +const DEFAULT_RIB_PRIORITY_STATIC: u8 = 1; + pub struct SwitchPortSettingsManager { datastore: Arc, resolver: Resolver, @@ -978,7 +982,7 @@ impl BackgroundTask for SwitchPortSettingsManager { destination: r.dst.into(), nexthop: r.gw.ip(), vlan_id: r.vid.map(|x| x.0), - local_pref: r.local_pref.map(|x| x.0), + rib_priority: r.rib_priority.map(|x| x.0), }) .collect(), switch: *location, @@ -1497,8 +1501,7 @@ fn build_sled_agent_clients( sled_agent_clients } -type SwitchStaticRoutes = - HashSet<(Ipv4Addr, Prefix4, Option, Option)>; +type SwitchStaticRoutes = HashSet<(Ipv4Addr, Prefix4, Option, Option)>; fn static_routes_to_del( current_static_routes: HashMap, @@ -1514,11 +1517,12 @@ fn static_routes_to_del( // if it's on the switch but not desired (in our db), it should be removed let stale_routes = routes_on_switch .difference(routes_wanted) - .map(|(nexthop, prefix, vlan_id, local_pref)| StaticRoute4 { + .map(|(nexthop, prefix, vlan_id, rib_priority)| StaticRoute4 { nexthop: *nexthop, prefix: *prefix, vlan_id: *vlan_id, - local_pref: *local_pref, + rib_priority: rib_priority + .unwrap_or(DEFAULT_RIB_PRIORITY_STATIC), }) .collect::>(); @@ -1532,11 +1536,12 @@ fn static_routes_to_del( // if no desired routes are present, all routes on this switch should be deleted let stale_routes = routes_on_switch .iter() - .map(|(nexthop, prefix, vlan_id, local_pref)| StaticRoute4 { + .map(|(nexthop, prefix, vlan_id, rib_priority)| StaticRoute4 { nexthop: *nexthop, prefix: *prefix, vlan_id: *vlan_id, - local_pref: *local_pref, + rib_priority: rib_priority + .unwrap_or(DEFAULT_RIB_PRIORITY_STATIC), }) .collect::>(); @@ -1583,11 +1588,12 @@ fn static_routes_to_add( }; let missing_routes = routes_wanted .difference(routes_on_switch) - .map(|(nexthop, prefix, vlan_id, local_pref)| StaticRoute4 { + .map(|(nexthop, prefix, vlan_id, rib_priority)| StaticRoute4 { nexthop: *nexthop, prefix: *prefix, vlan_id: *vlan_id, - local_pref: *local_pref, + rib_priority: rib_priority + .unwrap_or(DEFAULT_RIB_PRIORITY_STATIC), }) .collect::>(); @@ -1640,7 +1646,7 @@ fn static_routes_in_db( nexthop, prefix, route.vid.map(|x| x.0), - route.local_pref.map(|x| x.0), + route.rib_priority.map(|x| x.0), )); } @@ -1819,46 +1825,49 @@ async fn static_routes_on_switch<'a>( let mut routes_on_switch = HashMap::new(); for (location, client) in mgd_clients { - let static_routes: SwitchStaticRoutes = match client - .static_list_v4_routes() - .await - { - Ok(routes) => { - let mut flattened = HashSet::new(); - for (destination, paths) in routes.iter() { - let Ok(dst) = destination.parse() else { - error!( - log, - "failed to parse static route destination: \ + let static_routes: SwitchStaticRoutes = + match client.static_list_v4_routes().await { + Ok(routes) => { + let mut flattened = HashSet::new(); + for (destination, paths) in routes.iter() { + let Ok(dst) = destination.parse() else { + error!( + log, + "failed to parse static route destination: \ {destination}" - ); - continue; - }; - for p in paths.iter() { - let nh = match p.nexthop { - IpAddr::V4(addr) => addr, - IpAddr::V6(addr) => { - error!( - log, - "ipv6 nexthops not supported: {addr}" - ); - continue; - } + ); + continue; }; - flattened.insert((nh, dst, p.vlan_id, p.local_pref)); + for p in paths.iter() { + let nh = match p.nexthop { + IpAddr::V4(addr) => addr, + IpAddr::V6(addr) => { + error!( + log, + "ipv6 nexthops not supported: {addr}" + ); + continue; + } + }; + flattened.insert(( + nh, + dst, + p.vlan_id, + Some(p.rib_priority), + )); + } } + flattened } - flattened - } - Err(_) => { - error!( - &log, - "unable to retrieve routes from switch"; - "switch_location" => ?location, - ); - continue; - } - }; + Err(_) => { + error!( + &log, + "unable to retrieve routes from switch"; + "switch_location" => ?location, + ); + continue; + } + }; routes_on_switch.insert(*location, static_routes); } routes_on_switch diff --git a/nexus/src/app/background/tasks/vpc_routes.rs b/nexus/src/app/background/tasks/vpc_routes.rs index fe7dc6f8d1..5c326dc733 100644 --- a/nexus/src/app/background/tasks/vpc_routes.rs +++ b/nexus/src/app/background/tasks/vpc_routes.rs @@ -15,7 +15,8 @@ use nexus_types::{ identity::Resource, }; use omicron_common::api::internal::shared::{ - ResolvedVpcRoute, ResolvedVpcRouteSet, RouterId, RouterKind, RouterVersion, + ExternalIpGatewayMap, ResolvedVpcRoute, ResolvedVpcRouteSet, RouterId, + RouterKind, RouterVersion, }; use serde_json::json; use std::collections::hash_map::Entry; @@ -56,6 +57,7 @@ impl BackgroundTask for VpcRouteManager { ) -> BoxFuture<'a, serde_json::Value> { async { let log = &opctx.log; + info!(log, "VPC route manager running"); let sleds = match self .datastore @@ -93,6 +95,7 @@ impl BackgroundTask for VpcRouteManager { let mut vni_to_vpc = HashMap::new(); for (sled, client) in sled_clients { + info!(log, "VPC route manager sled {}", sled.id()); let Ok(route_sets) = client.list_vpc_routes().await else { warn!( log, @@ -102,6 +105,41 @@ impl BackgroundTask for VpcRouteManager { continue; }; + // Map each external IP in use by the sled to the Internet Gateway(s) + // which are allowed to make use of it. + // TODO: this should really not be the responsibility of this RPW. + // I would expect this belongs in a future external IPs RPW, but until + // then it lives here since it's a core part of the Internet Gateways + // system. + match self.datastore.vpc_resolve_sled_external_ips_to_gateways(opctx, sled.id()).await { + Ok(mappings) => { + info!( + log, + "computed internet gateway mappings for sled"; + "sled" => sled.serial_number(), + "assocs" => ?mappings + ); + let param = ExternalIpGatewayMap {mappings}; + if let Err(e) = client.set_eip_gateways(¶m).await { + error!( + log, + "failed to push internet gateway assignments for sled"; + "sled" => sled.serial_number(), + "err" => ?e + ); + continue; + }; + } + Err(e) => { + error!( + log, + "failed to produce EIP Internet Gateway mappings for sled"; + "sled" => sled.serial_number(), + "err" => ?e + ); + } + } + let route_sets = route_sets.into_inner(); // Lookup all VPC<->Subnet<->Router associations we might need, @@ -152,7 +190,7 @@ impl BackgroundTask for VpcRouteManager { let Ok(custom_routers) = self .datastore - .vpc_get_active_custom_routers(opctx, vpc_id) + .vpc_get_active_custom_routers_with_associated_subnets(opctx, vpc_id) .await else { error!( @@ -202,7 +240,7 @@ impl BackgroundTask for VpcRouteManager { // resolve into known_rules on an as-needed basis. for set in &route_sets { - let Some(db_router) = db_routers.get(&set.id) else { + let Some(db_router) = db_routers.get(&set.id) else { // The sled wants to know about rules for a VPC // subnet with no custom router set. Send them // the empty list, and unset its table version. @@ -221,7 +259,33 @@ impl BackgroundTask for VpcRouteManager { // number. match &set.version { Some(v) if !v.is_replaced_by(&version) => { + info!( + log, + "VPC route manager sled {} push not needed", + sled.id() + ); continue; + // Currently, this version is bumped in response to + // events that change the routes. This has to be + // done explicitly by the programmer in response to + // any event that may result in a difference in + // route resolution calculation. With the + // introduction of internet gateway targets that + // are parameterized on source IP, route resolution + // computations can change based on events that are + // not directly modifying VPC routes - like the + // linkiage of an IP pool to a silo, or the + // allocation of an external IP address and + // attachment of that IP address to a service or + // instance. This broadened context for changes + // influencing route resolution makes manual + // tracking of a router version easy to get wrong + // and I feel like it will be a bug magnet. + // + // I think we should move decisions around change + // propagation to be based on actual delta + // calculation, rather than trying to manually + // maintain a signal. } _ => {} } @@ -229,8 +293,12 @@ impl BackgroundTask for VpcRouteManager { // We may have already resolved the rules for this // router in a previous iteration. if let Some(rules) = known_rules.get(&router_id) { + info!( + log, + "VPC route manager sled {} rules already resolved", + sled.id() + ); set_rules(set.id, Some(version), rules.clone()); - continue; } match self @@ -242,15 +310,8 @@ impl BackgroundTask for VpcRouteManager { .await { Ok(rules) => { - let collapsed: HashSet<_> = rules - .into_iter() - .map(|(dest, target)| ResolvedVpcRoute { - dest, - target, - }) - .collect(); - set_rules(set.id, Some(version), collapsed.clone()); - known_rules.insert(router_id, collapsed); + set_rules(set.id, Some(version), rules.clone()); + known_rules.insert(router_id, rules); } Err(e) => { error!( diff --git a/nexus/src/app/external_endpoints.rs b/nexus/src/app/external_endpoints.rs index 18d2399eb5..9d56dad61f 100644 --- a/nexus/src/app/external_endpoints.rs +++ b/nexus/src/app/external_endpoints.rs @@ -35,11 +35,11 @@ use nexus_db_model::Certificate; use nexus_db_model::DnsGroup; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::Discoverability; -use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID; use nexus_db_queries::db::model::ServiceKind; use nexus_db_queries::db::DataStore; -use nexus_reconfigurator_execution::silo_dns_name; use nexus_types::identity::Resource; +use nexus_types::silo::silo_dns_name; +use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::DataPageParams; use omicron_common::api::external::Error; @@ -225,7 +225,7 @@ impl ExternalEndpoints { .filter(|s| { // Ignore the built-in Silo, which people are not supposed to // log into. - s.id() != *DEFAULT_SILO_ID + s.id() != DEFAULT_SILO_ID }) .find(|s| s.authentication_mode == AuthenticationMode::Local) .and_then(|s| { diff --git a/nexus/src/app/instance.rs b/nexus/src/app/instance.rs index ca8c441a41..97e7b3ffe8 100644 --- a/nexus/src/app/instance.rs +++ b/nexus/src/app/instance.rs @@ -330,7 +330,9 @@ impl super::Nexus { None => None, }; - let update = InstanceUpdate { boot_disk_id }; + let auto_restart_policy = params.auto_restart_policy.map(Into::into); + + let update = InstanceUpdate { boot_disk_id, auto_restart_policy }; self.datastore() .instance_reconfigure(opctx, &authz_instance, update) .await @@ -530,6 +532,8 @@ impl super::Nexus { } } + self.background_tasks.task_vpc_route_manager.activate(); + // TODO: This operation should return the instance as it was created. // Refetching the instance state here won't return that version of the // instance if its state changed between the time the saga finished and @@ -592,6 +596,8 @@ impl super::Nexus { saga_params, ) .await?; + + self.background_tasks.task_vpc_route_manager.activate(); Ok(()) } @@ -644,6 +650,8 @@ impl super::Nexus { ) .await?; + self.background_tasks.task_vpc_route_manager.activate(); + // TODO correctness TODO robustness TODO design // Should we lookup the instance again here? // See comment in project_create_instance. @@ -2007,10 +2015,18 @@ impl super::Nexus { ) .await?; - saga_outputs + let out = saga_outputs .lookup_node_output::("output") .map_err(|e| Error::internal_error(&format!("{:#}", &e))) - .internal_context("looking up output from ip attach saga") + .internal_context("looking up output from ip attach saga"); + + // The VPC routing RPW currently has double-duty on ensuring that + // sled-agents have up-to-date mappings between the EIPs they should + // know about and keeping routes up-to-date. + // Trigger the RPW so that OPTE can accurately select the IP ASAP. + self.vpc_needed_notify_sleds(); + + out } /// Detach an external IP from an instance. diff --git a/nexus/src/app/instance_network.rs b/nexus/src/app/instance_network.rs index 8cd0a34fbf..dc4162d353 100644 --- a/nexus/src/app/instance_network.rs +++ b/nexus/src/app/instance_network.rs @@ -261,7 +261,7 @@ pub(crate) async fn boundary_switches( pub(crate) async fn instance_ensure_dpd_config( datastore: &DataStore, log: &slog::Logger, - resolver: &internal_dns::resolver::Resolver, + resolver: &internal_dns_resolver::Resolver, opctx: &OpContext, opctx_alloc: &OpContext, instance_id: InstanceUuid, @@ -538,7 +538,7 @@ pub(crate) async fn probe_ensure_dpd_config( pub(crate) async fn instance_delete_dpd_config( datastore: &DataStore, log: &slog::Logger, - resolver: &internal_dns::resolver::Resolver, + resolver: &internal_dns_resolver::Resolver, opctx: &OpContext, opctx_alloc: &OpContext, authz_instance: &authz::Instance, @@ -574,7 +574,7 @@ pub(crate) async fn instance_delete_dpd_config( pub(crate) async fn probe_delete_dpd_config( datastore: &DataStore, log: &slog::Logger, - resolver: &internal_dns::resolver::Resolver, + resolver: &internal_dns_resolver::Resolver, opctx: &OpContext, opctx_alloc: &OpContext, probe_id: Uuid, @@ -659,7 +659,7 @@ pub(crate) async fn probe_delete_dpd_config( /// e.g. a rapid reattach or a reallocated ephemeral IP. pub(crate) async fn delete_dpd_config_by_entry( datastore: &DataStore, - resolver: &internal_dns::resolver::Resolver, + resolver: &internal_dns_resolver::Resolver, log: &slog::Logger, opctx: &OpContext, opctx_alloc: &OpContext, @@ -730,7 +730,7 @@ async fn external_ip_delete_dpd_config_inner( async fn notify_dendrite_nat_state( datastore: &DataStore, log: &slog::Logger, - resolver: &internal_dns::resolver::Resolver, + resolver: &internal_dns_resolver::Resolver, opctx_alloc: &OpContext, instance_id: Option, fail_fast: bool, diff --git a/nexus/src/app/internet_gateway.rs b/nexus/src/app/internet_gateway.rs new file mode 100644 index 0000000000..9afa0bc38c --- /dev/null +++ b/nexus/src/app/internet_gateway.rs @@ -0,0 +1,377 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Internet gateways + +use crate::external_api::params; +use nexus_auth::authz; +use nexus_auth::context::OpContext; +use nexus_db_queries::db; +use nexus_db_queries::db::lookup; +use nexus_db_queries::db::lookup::LookupPath; +use nexus_types::identity::Resource; +use omicron_common::api::external::http_pagination::PaginatedBy; +use omicron_common::api::external::CreateResult; +use omicron_common::api::external::DeleteResult; +use omicron_common::api::external::Error; +use omicron_common::api::external::ListResultVec; +use omicron_common::api::external::LookupResult; +use omicron_common::api::external::NameOrId; +use uuid::Uuid; + +impl super::Nexus { + //Internet gateways + + /// Lookup an internet gateway. + pub fn internet_gateway_lookup<'a>( + &'a self, + opctx: &'a OpContext, + igw_selector: params::InternetGatewaySelector, + ) -> LookupResult> { + match igw_selector { + params::InternetGatewaySelector { + gateway: NameOrId::Id(id), + vpc: None, + project: None + } => { + let gw = LookupPath::new(opctx, &self.db_datastore) + .internet_gateway_id(id); + Ok(gw) + } + params::InternetGatewaySelector { + gateway: NameOrId::Name(name), + vpc: Some(vpc), + project + } => { + let gw = self + .vpc_lookup(opctx, params::VpcSelector { project, vpc })? + .internet_gateway_name_owned(name.into()); + Ok(gw) + } + params::InternetGatewaySelector { + gateway: NameOrId::Id(_), + .. + } => Err(Error::invalid_request( + "when providing gateway as an ID vpc and project should not be specified", + )), + _ => Err(Error::invalid_request( + "gateway should either be an ID or vpc should be specified", + )), + } + } + + /// Create an internet gateway. + pub(crate) async fn internet_gateway_create( + &self, + opctx: &OpContext, + vpc_lookup: &lookup::Vpc<'_>, + params: ¶ms::InternetGatewayCreate, + ) -> CreateResult { + let (.., authz_vpc) = + vpc_lookup.lookup_for(authz::Action::CreateChild).await?; + let id = Uuid::new_v4(); + let router = + db::model::InternetGateway::new(id, authz_vpc.id(), params.clone()); + let (_, router) = self + .db_datastore + .vpc_create_internet_gateway(&opctx, &authz_vpc, router) + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(router) + } + + /// List internet gateways within a VPC. + pub(crate) async fn internet_gateway_list( + &self, + opctx: &OpContext, + vpc_lookup: &lookup::Vpc<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_vpc) = + vpc_lookup.lookup_for(authz::Action::ListChildren).await?; + let igws = self + .db_datastore + .internet_gateway_list(opctx, &authz_vpc, pagparams) + .await?; + Ok(igws) + } + + /// Delete an internet gateway. + /// + /// If there are routes that reference the gateway being deleted and + /// `cascade` is true all referencing routes are deleted, otherwise an + /// error is returned. + pub(crate) async fn internet_gateway_delete( + &self, + opctx: &OpContext, + lookup: &lookup::InternetGateway<'_>, + cascade: bool, + ) -> DeleteResult { + let (.., authz_vpc, authz_igw, _db_igw) = + lookup.fetch_for(authz::Action::Delete).await?; + + let out = self + .db_datastore + .vpc_delete_internet_gateway( + opctx, + &authz_igw, + authz_vpc.id(), + cascade, + ) + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) + } + + /// List IP pools associated with an internet gateway. + pub(crate) async fn internet_gateway_ip_pool_list( + &self, + opctx: &OpContext, + gateway_lookup: &lookup::InternetGateway<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_vpc) = + gateway_lookup.lookup_for(authz::Action::ListChildren).await?; + let pools = self + .db_datastore + .internet_gateway_list_ip_pools(opctx, &authz_vpc, pagparams) + .await?; + Ok(pools) + } + + /// Lookup an IP pool associated with an internet gateway. + pub fn internet_gateway_ip_pool_lookup<'a>( + &'a self, + opctx: &'a OpContext, + pool_selector: params::InternetGatewayIpPoolSelector, + ) -> LookupResult> { + match pool_selector { + params::InternetGatewayIpPoolSelector { + pool: NameOrId::Id(id), + gateway: None, + vpc: None, + project: None, + } => { + let route = LookupPath::new(opctx, &self.db_datastore) + .internet_gateway_ip_pool_id(id); + Ok(route) + } + params::InternetGatewayIpPoolSelector { + pool: NameOrId::Name(name), + gateway: Some(gateway), + vpc, + project, + } => { + let route = self + .internet_gateway_lookup( + opctx, + params::InternetGatewaySelector { project, vpc, gateway }, + )? + .internet_gateway_ip_pool_name_owned(name.into()); + Ok(route) + } + params::InternetGatewayIpPoolSelector { + pool: NameOrId::Id(_), + .. + } => Err(Error::invalid_request( + "when providing pool as an ID gateway, subnet, vpc, and project should not be specified", + )), + _ => Err(Error::invalid_request( + "pool should either be an ID or gateway should be specified", + )), + } + } + + /// Attach an IP pool to an internet gateway. + pub(crate) async fn internet_gateway_ip_pool_attach( + &self, + opctx: &OpContext, + lookup: &lookup::InternetGateway<'_>, + params: ¶ms::InternetGatewayIpPoolCreate, + ) -> CreateResult { + let (.., authz_igw, _) = + lookup.fetch_for(authz::Action::CreateChild).await?; + + // need to use this method so it works for non-fleet users + let (authz_pool, ..) = + self.silo_ip_pool_fetch(&opctx, ¶ms.ip_pool).await?; + + let id = Uuid::new_v4(); + let route = db::model::InternetGatewayIpPool::new( + id, + authz_pool.id(), + authz_igw.id(), + params.identity.clone(), + ); + let route = self + .db_datastore + .internet_gateway_attach_ip_pool(&opctx, &authz_igw, route) + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(route) + } + + /// Detach an IP pool from an internet gateway. + /// + /// If there are routes that depend on the IP pool being detached and + /// `cascade` is true then those routes are deleted, otherwise an + /// error is returned. + pub(crate) async fn internet_gateway_ip_pool_detach( + &self, + opctx: &OpContext, + lookup: &lookup::InternetGatewayIpPool<'_>, + cascade: bool, + ) -> DeleteResult { + let (.., authz_vpc, _authz_igw, authz_pool, db_pool) = + lookup.fetch_for(authz::Action::Delete).await?; + + let (.., igw) = LookupPath::new(opctx, &self.db_datastore) + .internet_gateway_id(db_pool.internet_gateway_id) + .fetch() + .await?; + + let out = self + .db_datastore + .internet_gateway_detach_ip_pool( + opctx, + igw.name().to_string(), + &authz_pool, + db_pool.ip_pool_id, + authz_vpc.id(), + cascade, + ) + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) + } + + /// List IP addresses attached to an internet gateway. + pub(crate) async fn internet_gateway_ip_address_list( + &self, + opctx: &OpContext, + gateway_lookup: &lookup::InternetGateway<'_>, + pagparams: &PaginatedBy<'_>, + ) -> ListResultVec { + let (.., authz_igw) = + gateway_lookup.lookup_for(authz::Action::ListChildren).await?; + let pools = self + .db_datastore + .internet_gateway_list_ip_addresses(opctx, &authz_igw, pagparams) + .await?; + Ok(pools) + } + + /// Lookup an IP address attached to an internet gateway. + pub fn internet_gateway_ip_address_lookup<'a>( + &'a self, + opctx: &'a OpContext, + address_selector: params::InternetGatewayIpAddressSelector, + ) -> LookupResult> { + match address_selector { + params::InternetGatewayIpAddressSelector { + address: NameOrId::Id(id), + gateway: None, + vpc: None, + project: None, + } => { + let route = LookupPath::new(opctx, &self.db_datastore) + .internet_gateway_ip_address_id(id); + Ok(route) + } + params::InternetGatewayIpAddressSelector { + address: NameOrId::Name(name), + gateway: Some(gateway), + vpc, + project, + } => { + let route = self + .internet_gateway_lookup( + opctx, + params::InternetGatewaySelector { project, vpc, gateway }, + )? + .internet_gateway_ip_address_name_owned(name.into()); + Ok(route) + } + params::InternetGatewayIpAddressSelector { + address: NameOrId::Id(_), + .. + } => Err(Error::invalid_request( + "when providing address as an ID gateway, subnet, vpc, and project should not be specified", + )), + _ => Err(Error::invalid_request( + "address should either be an ID or gateway should be specified", + )), + } + } + + /// Attach an IP address to an internet gateway. + pub(crate) async fn internet_gateway_ip_address_attach( + &self, + opctx: &OpContext, + lookup: &lookup::InternetGateway<'_>, + params: ¶ms::InternetGatewayIpAddressCreate, + ) -> CreateResult { + let (.., authz_igw, _) = + lookup.fetch_for(authz::Action::CreateChild).await?; + + let id = Uuid::new_v4(); + let route = db::model::InternetGatewayIpAddress::new( + id, + authz_igw.id(), + params.clone(), + ); + let route = self + .db_datastore + .internet_gateway_attach_ip_address(&opctx, &authz_igw, route) + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(route) + } + + /// Detach an IP pool from an internet gateway. + /// + /// If there are routes that depend on the IP address being detached and + /// `cascade` is true then those routes are deleted, otherwise an + /// error is returned. + pub(crate) async fn internet_gateway_ip_address_detach( + &self, + opctx: &OpContext, + lookup: &lookup::InternetGatewayIpAddress<'_>, + cascade: bool, + ) -> DeleteResult { + let (.., authz_vpc, _authz_igw, authz_addr, db_addr) = + lookup.fetch_for(authz::Action::Delete).await?; + + let (.., igw) = LookupPath::new(opctx, &self.db_datastore) + .internet_gateway_id(db_addr.internet_gateway_id) + .fetch() + .await?; + + let out = self + .db_datastore + .internet_gateway_detach_ip_address( + opctx, + igw.name().to_string(), + &authz_addr, + db_addr.address.ip(), + authz_vpc.id(), + cascade, + ) + .await?; + + self.vpc_needed_notify_sleds(); + + Ok(out) + } +} diff --git a/nexus/src/app/ip_pool.rs b/nexus/src/app/ip_pool.rs index d3179f9299..4e276731a2 100644 --- a/nexus/src/app/ip_pool.rs +++ b/nexus/src/app/ip_pool.rs @@ -92,13 +92,18 @@ impl super::Nexus { self.db_datastore.silo_ip_pool_list(opctx, &authz_silo, pagparams).await } - // Look up pool by name or ID, but only return it if it's linked to the - // current silo + /// Look up linked pool by name or ID. 404 on pools that exist but aren't + /// linked to the current silo. Special logic to make sure non-fleet users + /// can read the pool. pub async fn silo_ip_pool_fetch<'a>( &'a self, opctx: &'a OpContext, pool: &'a NameOrId, - ) -> LookupResult<(db::model::IpPool, db::model::IpPoolResource)> { + ) -> LookupResult<( + authz::IpPool, + db::model::IpPool, + db::model::IpPoolResource, + )> { let (authz_pool, pool) = self .ip_pool_lookup(opctx, pool)? // TODO-robustness: https://github.com/oxidecomputer/omicron/issues/3995 @@ -117,7 +122,7 @@ impl super::Nexus { // 404 if no link is found in the current silo let link = self.db_datastore.ip_pool_fetch_link(opctx, pool.id()).await; match link { - Ok(link) => Ok((pool, link)), + Ok(link) => Ok((authz_pool, pool, link)), Err(_) => Err(authz_pool.not_found()), } } diff --git a/nexus/src/app/mod.rs b/nexus/src/app/mod.rs index c004a14c7d..daee9714d4 100644 --- a/nexus/src/app/mod.rs +++ b/nexus/src/app/mod.rs @@ -15,7 +15,7 @@ use crate::populate::PopulateStatus; use crate::DropshotServer; use ::oximeter::types::ProducerRegistry; use anyhow::anyhow; -use internal_dns::ServiceName; +use internal_dns_types::names::ServiceName; use nexus_config::NexusConfig; use nexus_config::RegionAllocationStrategy; use nexus_config::Tunables; @@ -32,6 +32,8 @@ use omicron_common::api::external::Error; use omicron_common::api::internal::shared::SwitchLocation; use omicron_uuid_kinds::OmicronZoneUuid; use oximeter_producer::Server as ProducerServer; +use sagas::common_storage::make_pantry_connection_pool; +use sagas::common_storage::PooledPantryClient; use slog::Logger; use std::collections::HashMap; use std::net::SocketAddrV6; @@ -60,6 +62,7 @@ mod iam; mod image; mod instance; mod instance_network; +mod internet_gateway; mod ip_pool; mod metrics; mod network_interface; @@ -185,8 +188,11 @@ pub struct Nexus { // Nexus to not all fail. samael_max_issue_delay: std::sync::Mutex>, + /// Conection pool for Crucible pantries + pantry_connection_pool: qorb::pool::Pool, + /// DNS resolver for internal services - internal_resolver: internal_dns::resolver::Resolver, + internal_resolver: internal_dns_resolver::Resolver, /// DNS resolver Nexus uses to resolve an external host external_resolver: Arc, @@ -213,10 +219,12 @@ pub struct Nexus { impl Nexus { /// Create a new Nexus instance for the given rack id `rack_id` // TODO-polish revisit rack metadata + #[allow(clippy::too_many_arguments)] pub(crate) async fn new_with_id( rack_id: Uuid, log: Logger, - resolver: internal_dns::resolver::Resolver, + resolver: internal_dns_resolver::Resolver, + qorb_resolver: internal_dns_resolver::QorbResolver, pool: db::Pool, producer_registry: &ProducerRegistry, config: &NexusConfig, @@ -472,6 +480,7 @@ impl Nexus { as Arc, ), samael_max_issue_delay: std::sync::Mutex::new(None), + pantry_connection_pool: make_pantry_connection_pool(&qorb_resolver), internal_resolver: resolver.clone(), external_resolver, external_dns_servers: config @@ -931,10 +940,16 @@ impl Nexus { *mid } - pub fn resolver(&self) -> &internal_dns::resolver::Resolver { + pub fn resolver(&self) -> &internal_dns_resolver::Resolver { &self.internal_resolver } + pub(crate) fn pantry_connection_pool( + &self, + ) -> &qorb::pool::Pool { + &self.pantry_connection_pool + } + pub(crate) async fn dpd_clients( &self, ) -> Result, String> { @@ -992,7 +1007,7 @@ pub enum Unimpl { } pub(crate) async fn dpd_clients( - resolver: &internal_dns::resolver::Resolver, + resolver: &internal_dns_resolver::Resolver, log: &slog::Logger, ) -> Result, String> { let mappings = switch_zone_address_mappings(resolver, log).await?; @@ -1019,7 +1034,7 @@ pub(crate) async fn dpd_clients( } async fn switch_zone_address_mappings( - resolver: &internal_dns::resolver::Resolver, + resolver: &internal_dns_resolver::Resolver, log: &slog::Logger, ) -> Result, String> { let switch_zone_addresses = match resolver diff --git a/nexus/src/app/oximeter.rs b/nexus/src/app/oximeter.rs index 0c7ec3a016..770b5ac61b 100644 --- a/nexus/src/app/oximeter.rs +++ b/nexus/src/app/oximeter.rs @@ -7,14 +7,13 @@ use crate::external_api::params::ResourceMetrics; use crate::internal_api::params::OximeterInfo; use dropshot::PaginationParams; -use internal_dns::resolver::{ResolveError, Resolver}; -use internal_dns::ServiceName; +use internal_dns_resolver::{ResolveError, Resolver}; +use internal_dns_types::names::ServiceName; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; use nexus_db_queries::db::DataStore; use omicron_common::address::CLICKHOUSE_HTTP_PORT; -use omicron_common::api::external::Error; -use omicron_common::api::external::{DataPageParams, ListResultVec}; +use omicron_common::api::external::{DataPageParams, Error, ListResultVec}; use omicron_common::api::internal::nexus::{self, ProducerEndpoint}; use oximeter_client::Client as OximeterClient; use oximeter_db::query::Timestamp; @@ -113,9 +112,18 @@ impl super::Nexus { opctx: &OpContext, producer_info: nexus::ProducerEndpoint, ) -> Result<(), Error> { - let (collector, id) = self.next_collector(opctx).await?; - let db_info = db::model::ProducerEndpoint::new(&producer_info, id); - self.db_datastore.producer_endpoint_create(opctx, &db_info).await?; + let collector_info = self + .db_datastore + .producer_endpoint_upsert_and_assign(opctx, &producer_info) + .await?; + + let address = SocketAddr::from(( + collector_info.ip.ip(), + collector_info.port.try_into().unwrap(), + )); + let collector = + build_oximeter_client(&self.log, &collector_info.id, address); + collector .producers_post(&oximeter_client::types::ProducerEndpoint::from( &producer_info, @@ -125,9 +133,10 @@ impl super::Nexus { info!( self.log, "assigned collector to new producer"; - "producer_id" => ?producer_info.id, - "collector_id" => ?id, + "producer_id" => %producer_info.id, + "collector_id" => %collector_info.id, ); + Ok(()) } @@ -239,27 +248,6 @@ impl super::Nexus { ) .unwrap()) } - - // Return an oximeter collector to assign a newly-registered producer - async fn next_collector( - &self, - opctx: &OpContext, - ) -> Result<(OximeterClient, Uuid), Error> { - // TODO-robustness Replace with a real load-balancing strategy. - let page_params = DataPageParams { - marker: None, - direction: dropshot::PaginationOrder::Ascending, - limit: std::num::NonZeroU32::new(1).unwrap(), - }; - let oxs = self.db_datastore.oximeter_list(opctx, &page_params).await?; - let info = oxs.first().ok_or_else(|| Error::ServiceUnavailable { - internal_message: String::from("no oximeter collectors available"), - })?; - let address = - SocketAddr::from((info.ip.ip(), info.port.try_into().unwrap())); - let id = info.id; - Ok((build_oximeter_client(&self.log, &id, address), id)) - } } /// Idempotently un-assign a producer from an oximeter collector. diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index bc980bb543..9a361d564d 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -9,6 +9,7 @@ use crate::external_api::params::CertificateCreate; use crate::external_api::shared::ServiceUsingCertificate; use crate::internal_api::params::RackInitializationRequest; use gateway_client::types::SpType; +use internal_dns_types::names::DNS_ZONE; use ipnetwork::{IpNetwork, Ipv6Network}; use nexus_db_model::DnsGroup; use nexus_db_model::InitialDnsGroup; @@ -20,7 +21,6 @@ use nexus_db_queries::db::datastore::DnsVersionUpdateBuilder; use nexus_db_queries::db::datastore::RackInit; use nexus_db_queries::db::datastore::SledUnderlayAllocationResult; use nexus_db_queries::db::lookup::LookupPath; -use nexus_reconfigurator_execution::silo_dns_name; use nexus_types::deployment::blueprint_zone_type; use nexus_types::deployment::BlueprintZoneFilter; use nexus_types::deployment::BlueprintZoneType; @@ -48,6 +48,7 @@ use nexus_types::external_api::shared::SiloRole; use nexus_types::external_api::shared::UninitializedSled; use nexus_types::external_api::views; use nexus_types::internal_api::params::DnsRecord; +use nexus_types::silo::silo_dns_name; use omicron_common::address::{get_64_subnet, Ipv6Subnet, RACK_PREFIX}; use omicron_common::api::external::AddressLotKind; use omicron_common::api::external::BgpPeer; @@ -177,7 +178,7 @@ impl super::Nexus { .internal_dns_zone_config .zones .into_iter() - .find(|z| z.zone_name == internal_dns::DNS_ZONE) + .find(|z| z.zone_name == DNS_ZONE) .ok_or_else(|| { Error::invalid_request( "expected initial DNS config to include control plane zone", @@ -592,7 +593,7 @@ impl super::Nexus { dst: r.destination, gw: r.nexthop, vid: r.vlan_id, - local_pref: r.local_pref, + rib_priority: r.rib_priority, }) .collect(); diff --git a/nexus/src/app/sagas/common_storage.rs b/nexus/src/app/sagas/common_storage.rs index d37370506c..28e0bc4771 100644 --- a/nexus/src/app/sagas/common_storage.rs +++ b/nexus/src/app/sagas/common_storage.rs @@ -8,7 +8,6 @@ use super::*; use crate::Nexus; use crucible_pantry_client::types::VolumeConstructionRequest; -use internal_dns::ServiceName; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; @@ -16,19 +15,26 @@ use nexus_db_queries::db::lookup::LookupPath; use omicron_common::api::external::Error; use omicron_common::retry_until_known_result; use slog::Logger; +use slog_error_chain::InlineErrorChain; use std::net::SocketAddrV6; +mod pantry_pool; + +pub(crate) use pantry_pool::make_pantry_connection_pool; +pub(crate) use pantry_pool::PooledPantryClient; + // Common Pantry operations pub(crate) async fn get_pantry_address( nexus: &Arc, ) -> Result { - nexus - .resolver() - .lookup_socket_v6(ServiceName::CruciblePantry) - .await - .map_err(|e| e.to_string()) - .map_err(ActionError::action_failed) + let client = nexus.pantry_connection_pool().claim().await.map_err(|e| { + ActionError::action_failed(format!( + "failed to claim pantry client from pool: {}", + InlineErrorChain::new(&e) + )) + })?; + Ok(client.address()) } pub(crate) async fn call_pantry_attach_for_disk( diff --git a/nexus/src/app/sagas/common_storage/pantry_pool.rs b/nexus/src/app/sagas/common_storage/pantry_pool.rs new file mode 100644 index 0000000000..9d1e76d27d --- /dev/null +++ b/nexus/src/app/sagas/common_storage/pantry_pool.rs @@ -0,0 +1,92 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! `qorb` support for Crucible pantry connection pooling. + +use anyhow::anyhow; +use anyhow::Context; +use internal_dns_resolver::QorbResolver; +use internal_dns_types::names::ServiceName; +use qorb::backend; +use qorb::pool; +use std::net::SocketAddr; +use std::net::SocketAddrV6; +use std::sync::Arc; + +/// Wrapper around a Crucible pantry client that also remembers its address. +/// +/// In most cases when Nexus wants to pick a pantry, it doesn't actually want a +/// client right then, but instead wants to write down its address for subsequent +/// use (and reuse) later. This type carries around a `client` only to perform +/// health checks as supported by `qorb`; the rest of Nexus only accesses its +/// `address`. +#[derive(Debug)] +pub(crate) struct PooledPantryClient { + client: crucible_pantry_client::Client, + address: SocketAddrV6, +} + +impl PooledPantryClient { + pub(crate) fn address(&self) -> SocketAddrV6 { + self.address + } +} + +/// A [`backend::Connector`] for [`PooledPantryClient`]s. +#[derive(Debug)] +struct PantryConnector; + +#[async_trait::async_trait] +impl backend::Connector for PantryConnector { + type Connection = PooledPantryClient; + + async fn connect( + &self, + backend: &backend::Backend, + ) -> Result { + let address = match backend.address { + SocketAddr::V6(addr) => addr, + SocketAddr::V4(addr) => { + return Err(backend::Error::Other(anyhow!( + "unexpected IPv4 address for Crucible pantry: {addr}" + ))); + } + }; + let client = + crucible_pantry_client::Client::new(&format!("http://{address}")); + Ok(PooledPantryClient { client, address }) + } + + async fn is_valid( + &self, + conn: &mut Self::Connection, + ) -> Result<(), backend::Error> { + conn.client + .pantry_status() + .await + .with_context(|| { + format!("failed to fetch pantry status from {}", conn.address()) + }) + .map_err(backend::Error::Other)?; + + Ok(()) + } + + async fn on_acquire( + &self, + conn: &mut Self::Connection, + ) -> Result<(), backend::Error> { + self.is_valid(conn).await + } +} + +pub(crate) fn make_pantry_connection_pool( + qorb_resolver: &QorbResolver, +) -> pool::Pool { + pool::Pool::new( + qorb_resolver.for_service(ServiceName::CruciblePantry), + Arc::new(PantryConnector), + qorb::policy::Policy::default(), + ) +} diff --git a/nexus/src/app/sagas/instance_create.rs b/nexus/src/app/sagas/instance_create.rs index c8680701b1..07f7911ef5 100644 --- a/nexus/src/app/sagas/instance_create.rs +++ b/nexus/src/app/sagas/instance_create.rs @@ -44,7 +44,6 @@ pub(crate) struct Params { pub create_params: params::InstanceCreate, pub boundary_switches: HashSet, } - // Several nodes in this saga are wrapped in their own subsaga so that they can // have a parameter that denotes which node they are (e.g., which NIC or which // external IP). They also need the outer saga's parameters. @@ -1077,11 +1076,8 @@ async fn sic_set_boot_disk( .await .map_err(ActionError::action_failed)?; - let initial_configuration = - nexus_db_model::InstanceUpdate { boot_disk_id: Some(authz_disk.id()) }; - datastore - .instance_reconfigure(&opctx, &authz_instance, initial_configuration) + .instance_set_boot_disk(&opctx, &authz_instance, Some(authz_disk.id())) .await .map_err(ActionError::action_failed)?; @@ -1109,11 +1105,8 @@ async fn sic_set_boot_disk_undo( // If there was a boot disk, clear it. If there was not a boot disk, // this is a no-op. - let undo_configuration = - nexus_db_model::InstanceUpdate { boot_disk_id: None }; - datastore - .instance_reconfigure(&opctx, &authz_instance, undo_configuration) + .instance_set_boot_disk(&opctx, &authz_instance, None) .await .map_err(ActionError::action_failed)?; diff --git a/nexus/src/app/sagas/region_replacement_drive.rs b/nexus/src/app/sagas/region_replacement_drive.rs index d95591848e..4e6335435f 100644 --- a/nexus/src/app/sagas/region_replacement_drive.rs +++ b/nexus/src/app/sagas/region_replacement_drive.rs @@ -309,6 +309,32 @@ async fn srrd_drive_region_replacement_check( ¶ms.serialized_authn, ); + // It doesn't make sense to perform any of this saga if the volume was soft + // or hard deleted: for example, this happens if the higher level resource + // like the disk was deleted. Volume deletion potentially results in the + // clean-up of Crucible resources, so it wouldn't even be valid to attempt + // to drive forward any type of live repair or reconciliation. + // + // Setting Done here will cause this saga to transition the replacement + // request to ReplacementDone. + + let volume_deleted = osagactx + .datastore() + .volume_deleted(params.request.volume_id) + .await + .map_err(ActionError::action_failed)?; + + if volume_deleted { + info!( + log, + "volume was soft or hard deleted!"; + "region replacement id" => %params.request.id, + "volume id" => %params.request.volume_id, + ); + + return Ok(DriveCheck::Done); + } + let last_request_step = osagactx .datastore() .current_region_replacement_request_step(&opctx, params.request.id) @@ -1009,11 +1035,6 @@ async fn srrd_drive_region_replacement_prepare( "disk id" => ?disk.id(), ); - // XXX: internal-dns does not randomize the order of addresses - // in its responses: if the first Pantry in the list of - // addresses returned by DNS isn't responding, the drive saga - // will still continually try to use it. - let pantry_address = get_pantry_address(nexus).await?; DriveAction::Pantry { diff --git a/nexus/src/app/sagas/region_replacement_finish.rs b/nexus/src/app/sagas/region_replacement_finish.rs index 8d8e75ea91..8ea77f4e97 100644 --- a/nexus/src/app/sagas/region_replacement_finish.rs +++ b/nexus/src/app/sagas/region_replacement_finish.rs @@ -275,8 +275,8 @@ pub(crate) mod test { opts: CrucibleOpts { id: old_region_volume_id, target: vec![ - // XXX if you put something here, you'll need a - // synthetic dataset record + // if you put something here, you'll need a synthetic + // dataset record ], lossy: false, flush_timeout: None, diff --git a/nexus/src/app/sagas/region_snapshot_replacement_garbage_collect.rs b/nexus/src/app/sagas/region_snapshot_replacement_garbage_collect.rs index e3c5143a68..9ebc6f0271 100644 --- a/nexus/src/app/sagas/region_snapshot_replacement_garbage_collect.rs +++ b/nexus/src/app/sagas/region_snapshot_replacement_garbage_collect.rs @@ -250,8 +250,8 @@ pub(crate) mod test { opts: CrucibleOpts { id: old_snapshot_volume_id, target: vec![ - // XXX if you put something here, you'll need a - // synthetic dataset record + // if you put something here, you'll need a synthetic + // dataset record ], lossy: false, flush_timeout: None, diff --git a/nexus/src/app/sagas/region_snapshot_replacement_step_garbage_collect.rs b/nexus/src/app/sagas/region_snapshot_replacement_step_garbage_collect.rs index dedfdb213e..b83f917a70 100644 --- a/nexus/src/app/sagas/region_snapshot_replacement_step_garbage_collect.rs +++ b/nexus/src/app/sagas/region_snapshot_replacement_step_garbage_collect.rs @@ -163,8 +163,8 @@ pub(crate) mod test { opts: CrucibleOpts { id: old_snapshot_volume_id, target: vec![ - // XXX if you put something here, you'll need a - // synthetic dataset record + // if you put something here, you'll need a synthetic + // dataset record ], lossy: false, flush_timeout: None, diff --git a/nexus/src/app/sagas/vpc_create.rs b/nexus/src/app/sagas/vpc_create.rs index 2bdae167b0..6e1cbfa9ab 100644 --- a/nexus/src/app/sagas/vpc_create.rs +++ b/nexus/src/app/sagas/vpc_create.rs @@ -8,15 +8,13 @@ use super::NexusSaga; use super::ACTION_GENERATE_ID; use crate::app::sagas::declare_saga_actions; use crate::external_api::params; +use nexus_db_model::InternetGatewayIpPool; use nexus_db_queries::db::queries::vpc_subnet::InsertVpcSubnetError; use nexus_db_queries::{authn, authz, db}; use nexus_defaults as defaults; use omicron_common::api::external; use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::LookupType; -use omicron_common::api::external::RouteDestination; -use omicron_common::api::external::RouteTarget; -use omicron_common::api::external::RouterRouteKind; use oxnet::IpNet; use serde::Deserialize; use serde::Serialize; @@ -45,6 +43,10 @@ declare_saga_actions! { + svc_create_router - svc_create_router_undo } + VPC_CREATE_GATEWAY -> "gateway" { + + svc_create_gateway + - svc_create_gateway_undo + } VPC_CREATE_V4_ROUTE -> "route4" { + svc_create_v4_route - svc_create_v4_route_undo @@ -98,12 +100,18 @@ pub fn create_dag( "GenerateDefaultSubnetId", ACTION_GENERATE_ID.as_ref(), )); + builder.append(Node::action( + "default_internet_gateway_id", + "GenerateDefaultInternetGatewayId", + ACTION_GENERATE_ID.as_ref(), + )); builder.append(vpc_create_vpc_action()); builder.append(vpc_create_router_action()); builder.append(vpc_create_v4_route_action()); builder.append(vpc_create_v6_route_action()); builder.append(vpc_create_subnet_action()); builder.append(vpc_update_firewall_action()); + builder.append(vpc_create_gateway_action()); builder.append(vpc_notify_sleds_action()); Ok(builder.build()?) @@ -280,14 +288,16 @@ async fn svc_create_route( let route = db::model::RouterRoute::new( route_id, system_router_id, - RouterRouteKind::Default, + external::RouterRouteKind::Default, params::RouterRouteCreate { identity: IdentityMetadataCreateParams { name: name.parse().unwrap(), description: "The default route of a vpc".to_string(), }, - target: RouteTarget::InternetGateway("outbound".parse().unwrap()), - destination: RouteDestination::IpNet(default_net), + target: external::RouteTarget::InternetGateway( + "default".parse().unwrap(), + ), + destination: external::RouteDestination::IpNet(default_net), }, ); @@ -460,6 +470,93 @@ async fn svc_update_firewall_undo( Ok(()) } +async fn svc_create_gateway( + sagactx: NexusActionContext, +) -> Result { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let vpc_id = sagactx.lookup::("vpc_id")?; + let default_igw_id = + sagactx.lookup::("default_internet_gateway_id")?; + let (authz_vpc, _) = + sagactx.lookup::<(authz::Vpc, db::model::Vpc)>("vpc")?; + + let igw = db::model::InternetGateway::new( + default_igw_id, + vpc_id, + params::InternetGatewayCreate { + identity: IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: "Automatically created default VPC gateway".into(), + }, + }, + ); + + let (authz_igw, _) = osagactx + .datastore() + .vpc_create_internet_gateway(&opctx, &authz_vpc, igw) + .await + .map_err(ActionError::action_failed)?; + + match osagactx.datastore().ip_pools_fetch_default(&opctx).await { + Ok((authz_ip_pool, _db_ip_pool)) => { + // Attach the default IP pool to the default gateway. + // Failure of this saga takes out the gateway with a cascading delete and + // thus this ip pool. + osagactx + .datastore() + .internet_gateway_attach_ip_pool( + &opctx, + &authz_igw, + InternetGatewayIpPool::new( + Uuid::new_v4(), + authz_ip_pool.id(), + authz_igw.id(), + IdentityMetadataCreateParams { + name: "default".parse().unwrap(), + description: + "Automatically attached default IP pool".into(), + }, + ), + ) + .await + .map_err(ActionError::action_failed)?; + } + Err(e) => { + warn!( + opctx.log, + "Default ip pool lookup failed: {e}. \ + Default gateway has no ip pool association", + ); + } + }; + + Ok(authz_igw) +} + +async fn svc_create_gateway_undo( + sagactx: NexusActionContext, +) -> Result<(), anyhow::Error> { + let osagactx = sagactx.user_data(); + let params = sagactx.saga_params::()?; + let opctx = crate::context::op_context_for_saga_action( + &sagactx, + ¶ms.serialized_authn, + ); + let vpc_id = sagactx.lookup::("vpc_id")?; + let authz_igw = sagactx.lookup::("gateway")?; + + osagactx + .datastore() + .vpc_delete_internet_gateway(&opctx, &authz_igw, vpc_id, true) + .await?; + Ok(()) +} + async fn svc_notify_sleds( sagactx: NexusActionContext, ) -> Result<(), ActionError> { @@ -493,6 +590,7 @@ pub(crate) mod test { ExpressionMethods, OptionalExtension, QueryDsl, SelectableHelper, }; use dropshot::test_util::ClientTestContext; + use nexus_db_queries::db::fixed_data::vpc::SERVICES_INTERNET_GATEWAY_ID; use nexus_db_queries::{ authn::saga::Serialized, authz, context::OpContext, db::datastore::DataStore, db::fixed_data::vpc::SERVICES_VPC_ID, @@ -623,6 +721,25 @@ pub(crate) mod test { .await .expect("Failed to delete system router"); + // Default gateway + let (.., authz_vpc, authz_igw, _igw) = + LookupPath::new(&opctx, &datastore) + .project_id(project_id) + .vpc_name(&default_name.clone().into()) + .internet_gateway_name(&default_name.clone().into()) + .fetch() + .await + .expect("Failed to fetch default gateway"); + datastore + .vpc_delete_internet_gateway( + &opctx, + &authz_igw, + authz_vpc.id(), + true, + ) + .await + .expect("Failed to delete default gateway"); + // Default VPC & Firewall Rules let (.., authz_vpc, vpc) = LookupPath::new(&opctx, &datastore) .project_id(project_id) @@ -645,6 +762,8 @@ pub(crate) mod test { assert!(no_routers_exist(datastore).await); assert!(no_routes_exist(datastore).await); assert!(no_subnets_exist(datastore).await); + assert!(no_gateways_exist(datastore).await); + assert!(no_gateway_links_exist(datastore).await); assert!(no_firewall_rules_exist(datastore).await); } @@ -690,6 +809,49 @@ pub(crate) mod test { .is_none() } + async fn no_gateways_exist(datastore: &DataStore) -> bool { + use nexus_db_queries::db::model::InternetGateway; + use nexus_db_queries::db::schema::internet_gateway::dsl; + + dsl::internet_gateway + .filter(dsl::time_deleted.is_null()) + // ignore built-in services VPC + .filter(dsl::vpc_id.ne(*SERVICES_VPC_ID)) + .select(InternetGateway::as_select()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .optional() + .unwrap() + .map(|igw| { + eprintln!("Internet gateway exists: {igw:?}"); + }) + .is_none() + } + + async fn no_gateway_links_exist(datastore: &DataStore) -> bool { + use nexus_db_queries::db::model::InternetGatewayIpPool; + use nexus_db_queries::db::schema::internet_gateway_ip_pool::dsl; + + dsl::internet_gateway_ip_pool + .filter(dsl::time_deleted.is_null()) + .filter(dsl::internet_gateway_id.ne(*SERVICES_INTERNET_GATEWAY_ID)) + .select(InternetGatewayIpPool::as_select()) + .first_async::( + &*datastore.pool_connection_for_tests().await.unwrap(), + ) + .await + .optional() + .unwrap() + .map(|igw_ip_pool| { + eprintln!( + "Internet gateway ip pool links exists: {igw_ip_pool:?}" + ); + }) + .is_none() + } + async fn no_routes_exist(datastore: &DataStore) -> bool { use nexus_db_queries::db::model::RouterRoute; use nexus_db_queries::db::schema::router_route::dsl; diff --git a/nexus/src/app/silo.rs b/nexus/src/app/silo.rs index efde55cbd1..b8e3de8866 100644 --- a/nexus/src/app/silo.rs +++ b/nexus/src/app/silo.rs @@ -16,9 +16,9 @@ use nexus_db_queries::db::identity::{Asset, Resource}; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::{self, lookup}; use nexus_db_queries::{authn, authz}; -use nexus_reconfigurator_execution::blueprint_nexus_external_ips; -use nexus_reconfigurator_execution::silo_dns_name; +use nexus_types::deployment::execution::blueprint_nexus_external_ips; use nexus_types::internal_api::params::DnsRecord; +use nexus_types::silo::silo_dns_name; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::ListResultVec; use omicron_common::api::external::LookupResult; diff --git a/nexus/src/app/vpc_router.rs b/nexus/src/app/vpc_router.rs index b15edf124d..f7f27765f2 100644 --- a/nexus/src/app/vpc_router.rs +++ b/nexus/src/app/vpc_router.rs @@ -341,7 +341,6 @@ impl super::Nexus { /// - mixed explicit v4 and v6 are disallowed. /// - users cannot specify 'Vpc' as a custom/default router dest/target. /// - users cannot specify 'Subnet' as a custom/default router target. -/// - the only internet gateway we support today is 'outbound'. fn validate_user_route_targets( dest: &RouteDestination, target: &RouteTarget, @@ -367,10 +366,6 @@ fn validate_user_route_targets( format!("subnets cannot be used as a target in {route_type} routers") )), - (_, RouteTarget::InternetGateway(n)) if n.as_str() != "outbound" => return Err(Error::invalid_request( - "'outbound' is currently the only valid internet gateway" - )), - _ => {}, }; diff --git a/nexus/src/context.rs b/nexus/src/context.rs index a2a50958e4..1b2089f550 100644 --- a/nexus/src/context.rs +++ b/nexus/src/context.rs @@ -210,8 +210,9 @@ impl ServerContext { // like console index.html. leaving that out for now so we don't break // nexus in dev for everyone - // Set up DNS Client - let (resolver, dns_addrs) = match config.deployment.internal_dns { + // Set up DNS Client (both traditional and qorb-based, until we've moved + // every consumer over to qorb) + let (resolver, qorb_resolver) = match config.deployment.internal_dns { nexus_config::InternalDns::FromSubnet { subnet } => { let az_subnet = Ipv6Subnet::::new(subnet.net().addr()); @@ -221,20 +222,20 @@ impl ServerContext { az_subnet ); let resolver = - internal_dns::resolver::Resolver::new_from_subnet( + internal_dns_resolver::Resolver::new_from_subnet( log.new(o!("component" => "DnsResolver")), az_subnet, ) .map_err(|e| { format!("Failed to create DNS resolver: {}", e) })?; - - ( - resolver, - internal_dns::resolver::Resolver::servers_from_subnet( + let qorb_resolver = internal_dns_resolver::QorbResolver::new( + internal_dns_resolver::Resolver::servers_from_subnet( az_subnet, ), - ) + ); + + (resolver, qorb_resolver) } nexus_config::InternalDns::FromAddress { address } => { info!( @@ -242,30 +243,35 @@ impl ServerContext { "Setting up resolver using DNS address: {:?}", address ); - let resolver = - internal_dns::resolver::Resolver::new_from_addrs( - log.new(o!("component" => "DnsResolver")), - &[address], - ) - .map_err(|e| { - format!("Failed to create DNS resolver: {}", e) - })?; + let resolver = internal_dns_resolver::Resolver::new_from_addrs( + log.new(o!("component" => "DnsResolver")), + &[address], + ) + .map_err(|e| format!("Failed to create DNS resolver: {}", e))?; + let qorb_resolver = + internal_dns_resolver::QorbResolver::new(vec![address]); - (resolver, vec![address]) + (resolver, qorb_resolver) } }; let pool = match &config.deployment.database { nexus_config::Database::FromUrl { url } => { - info!(log, "Setting up qorb pool from a single host"; "url" => #?url); + info!( + log, "Setting up qorb database pool from a single host"; + "url" => #?url, + ); db::Pool::new_single_host( &log, &db::Config { url: url.clone() }, ) } nexus_config::Database::FromDns => { - info!(log, "Setting up qorb pool from DNS"; "dns_addrs" => #?dns_addrs); - db::Pool::new(&log, dns_addrs) + info!( + log, "Setting up qorb database pool from DNS"; + "dns_addrs" => ?qorb_resolver.bootstrap_dns_ips(), + ); + db::Pool::new(&log, &qorb_resolver) } }; @@ -273,6 +279,7 @@ impl ServerContext { rack_id, log.new(o!("component" => "nexus")), resolver, + qorb_resolver, pool, &producer_registry, config, diff --git a/nexus/src/external_api/console_api.rs b/nexus/src/external_api/console_api.rs index 87ee15fc91..fd22c22d0e 100644 --- a/nexus/src/external_api/console_api.rs +++ b/nexus/src/external_api/console_api.rs @@ -8,19 +8,6 @@ //! external API, but in order to avoid CORS issues for now, we are serving //! these routes directly from the external API. -// `HeaderName` and `HeaderValue` contain `bytes::Bytes`, which trips -// the `declare_interior_mutable_const` lint. But in a `const fn` -// context, the `AtomicPtr` that is used in `Bytes` only ever points -// to a `&'static str`, so does not have interior mutability in that -// context. -// -// A Clippy bug means that even if you ignore interior mutability of -// `Bytes` (the default behavior), it will still not ignore it for types -// where the only interior mutability is through `Bytes`. This is fixed -// in rust-lang/rust-clippy#12691, which should land in the Rust 1.80 -// toolchain; we can remove this attribute then. -#![allow(clippy::declare_interior_mutable_const)] - use crate::context::ApiContext; use anyhow::Context; use camino::{Utf8Path, Utf8PathBuf}; diff --git a/nexus/src/external_api/http_entrypoints.rs b/nexus/src/external_api/http_entrypoints.rs index 33667a3da0..df8f1aa24e 100644 --- a/nexus/src/external_api/http_entrypoints.rs +++ b/nexus/src/external_api/http_entrypoints.rs @@ -1044,7 +1044,7 @@ impl NexusExternalApi for NexusExternalApiImpl { crate::context::op_context_for_external_api(&rqctx).await?; let nexus = &apictx.context.nexus; let pool_selector = path_params.into_inner().pool; - let (pool, silo_link) = + let (.., pool, silo_link) = nexus.silo_ip_pool_fetch(&opctx, &pool_selector).await?; Ok(HttpResponseOk(views::SiloIpPool { identity: pool.identity(), @@ -4689,6 +4689,344 @@ impl NexusExternalApi for NexusExternalApiImpl { .await } + // Internet gateways + + /// List internet gateways + async fn internet_gateway_list( + rqctx: RequestContext, + query_params: Query>, + ) -> Result>, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let vpc_lookup = + nexus.vpc_lookup(&opctx, scan_params.selector.clone())?; + let results = nexus + .internet_gateway_list(&opctx, &vpc_lookup, &paginated_by) + .await? + .into_iter() + .map(|s| s.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + results, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// Fetch internet gateway + async fn internet_gateway_view( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let selector = params::InternetGatewaySelector { + project: query.project, + vpc: query.vpc, + gateway: path.gateway, + }; + let (.., internet_gateway) = nexus + .internet_gateway_lookup(&opctx, selector)? + .fetch() + .await?; + Ok(HttpResponseOk(internet_gateway.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// Create VPC internet gateway + async fn internet_gateway_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError> { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let create = create_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let query = query_params.into_inner(); + let vpc_lookup = nexus.vpc_lookup(&opctx, query)?; + let result = nexus + .internet_gateway_create(&opctx, &vpc_lookup, &create) + .await?; + Ok(HttpResponseCreated(result.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// Delete internet gateway + async fn internet_gateway_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let selector = params::InternetGatewaySelector { + project: query.project, + vpc: query.vpc, + gateway: path.gateway, + }; + let lookup = nexus.internet_gateway_lookup(&opctx, selector)?; + nexus + .internet_gateway_delete(&opctx, &lookup, query.cascade) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// List IP pools attached to an internet gateway. + async fn internet_gateway_ip_pool_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result< + HttpResponseOk>, + HttpError, + > { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let lookup = nexus.internet_gateway_lookup( + &opctx, + scan_params.selector.clone(), + )?; + let results = nexus + .internet_gateway_ip_pool_list(&opctx, &lookup, &paginated_by) + .await? + .into_iter() + .map(|route| route.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + results, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// Attach an IP pool to an internet gateway + async fn internet_gateway_ip_pool_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let create = create_params.into_inner(); + let lookup = nexus.internet_gateway_lookup(&opctx, query)?; + let result = nexus + .internet_gateway_ip_pool_attach(&opctx, &lookup, &create) + .await?; + Ok(HttpResponseCreated(result.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// Detach an IP pool from an internet gateway + async fn internet_gateway_ip_pool_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let selector = params::InternetGatewayIpPoolSelector { + project: query.project, + vpc: query.vpc, + gateway: query.gateway, + pool: path.pool, + }; + let lookup = + nexus.internet_gateway_ip_pool_lookup(&opctx, selector)?; + nexus + .internet_gateway_ip_pool_detach(&opctx, &lookup, query.cascade) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// List addresses attached to an internet gateway. + async fn internet_gateway_ip_address_list( + rqctx: RequestContext, + query_params: Query< + PaginatedByNameOrId, + >, + ) -> Result< + HttpResponseOk>, + HttpError, + > { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let pag_params = data_page_params_for(&rqctx, &query)?; + let scan_params = ScanByNameOrId::from_query(&query)?; + let paginated_by = name_or_id_pagination(&pag_params, scan_params)?; + let lookup = nexus.internet_gateway_lookup( + &opctx, + scan_params.selector.clone(), + )?; + let results = nexus + .internet_gateway_ip_address_list( + &opctx, + &lookup, + &paginated_by, + ) + .await? + .into_iter() + .map(|route| route.into()) + .collect(); + Ok(HttpResponseOk(ScanByNameOrId::results_page( + &query, + results, + &marker_for_name_or_id, + )?)) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// Attach an IP address to an internet gateway + async fn internet_gateway_ip_address_create( + rqctx: RequestContext, + query_params: Query, + create_params: TypedBody, + ) -> Result, HttpError> + { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let query = query_params.into_inner(); + let create = create_params.into_inner(); + let lookup = nexus.internet_gateway_lookup(&opctx, query)?; + let route = nexus + .internet_gateway_ip_address_attach(&opctx, &lookup, &create) + .await?; + Ok(HttpResponseCreated(route.into())) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + + /// Detach an IP address from an internet gateway + async fn internet_gateway_ip_address_delete( + rqctx: RequestContext, + path_params: Path, + query_params: Query, + ) -> Result { + let apictx = rqctx.context(); + let handler = async { + let opctx = + crate::context::op_context_for_external_api(&rqctx).await?; + let nexus = &apictx.context.nexus; + let path = path_params.into_inner(); + let query = query_params.into_inner(); + let selector = params::InternetGatewayIpAddressSelector { + project: query.project, + vpc: query.vpc, + gateway: query.gateway, + address: path.address, + }; + let lookup = + nexus.internet_gateway_ip_address_lookup(&opctx, selector)?; + nexus + .internet_gateway_ip_address_detach( + &opctx, + &lookup, + query.cascade, + ) + .await?; + Ok(HttpResponseDeleted()) + }; + apictx + .context + .external_latencies + .instrument_dropshot_handler(&rqctx, handler) + .await + } + // Racks async fn rack_list( diff --git a/nexus/test-utils/Cargo.toml b/nexus/test-utils/Cargo.toml index aa5a3096c4..c22a03a2f9 100644 --- a/nexus/test-utils/Cargo.toml +++ b/nexus/test-utils/Cargo.toml @@ -25,7 +25,8 @@ http.workspace = true http-body-util.workspace = true hyper.workspace = true illumos-utils.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true nexus-client.workspace = true nexus-config.workspace = true nexus-db-queries.workspace = true diff --git a/nexus/test-utils/src/background.rs b/nexus/test-utils/src/background.rs index 58792e547d..859796f556 100644 --- a/nexus/test-utils/src/background.rs +++ b/nexus/test-utils/src/background.rs @@ -8,14 +8,15 @@ use crate::http_testing::NexusRequest; use dropshot::test_util::ClientTestContext; use nexus_client::types::BackgroundTask; use nexus_client::types::CurrentStatus; -use nexus_client::types::CurrentStatusRunning; use nexus_client::types::LastResult; use nexus_client::types::LastResultCompleted; +use nexus_types::internal_api::background::*; use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use std::time::Duration; -/// Return the most recent start time for a background task -fn most_recent_start_time( +/// Return the most recent activate time for a background task, returning None +/// if it has never been started or is currently running. +fn most_recent_activate_time( task: &BackgroundTask, ) -> Option> { match task.current { @@ -23,11 +24,11 @@ fn most_recent_start_time( LastResult::Completed(LastResultCompleted { start_time, .. }) => Some(start_time), + LastResult::NeverCompleted => None, }, - CurrentStatus::Running(CurrentStatusRunning { start_time, .. }) => { - Some(start_time) - } + + CurrentStatus::Running(..) => None, } } @@ -44,7 +45,7 @@ pub async fn activate_background_task( .execute_and_parse_unwrap::() .await; - let last_start = most_recent_start_time(&task); + let last_activate = most_recent_activate_time(&task); internal_client .make_request( @@ -68,10 +69,45 @@ pub async fn activate_background_task( .execute_and_parse_unwrap::() .await; - if matches!(&task.current, CurrentStatus::Idle) - && most_recent_start_time(&task) > last_start - { - Ok(task) + // Wait until the task has actually run and then is idle + if matches!(&task.current, CurrentStatus::Idle) { + let current_activate = most_recent_activate_time(&task); + match (current_activate, last_activate) { + (None, None) => { + // task is idle but it hasn't started yet, and it was + // never previously activated + Err(CondCheckError::<()>::NotYet) + } + + (Some(_), None) => { + // task was activated for the first time by this + // function call, and it's done now (because the task is + // idle) + Ok(task) + } + + (None, Some(_)) => { + // the task is idle (due to the check above) but + // `most_recent_activate_time` returned None, implying + // that the LastResult is NeverCompleted? the Some in + // the second part of the tuple means this ran before, + // so panic here. + panic!("task is idle, but there's no activate time?!"); + } + + (Some(current_activation), Some(last_activation)) => { + // the task is idle, it started ok, and it was + // previously activated: compare times to make sure we + // didn't observe the same BackgroundTask object + if current_activation > last_activation { + Ok(task) + } else { + // the task hasn't started yet, we observed the same + // BackgroundTask object + Err(CondCheckError::<()>::NotYet) + } + } + } } else { Err(CondCheckError::<()>::NotYet) } @@ -84,3 +120,220 @@ pub async fn activate_background_task( last_task_poll } + +/// Run the region_replacement background task, returning how many actions +/// were taken +pub async fn run_region_replacement( + internal_client: &ClientTestContext, +) -> usize { + let last_background_task = + activate_background_task(&internal_client, "region_replacement").await; + + let LastResult::Completed(last_result_completed) = + last_background_task.last + else { + panic!( + "unexpected {:?} returned from region_replacement task", + last_background_task.last, + ); + }; + + let status = serde_json::from_value::( + last_result_completed.details, + ) + .unwrap(); + + assert!(status.errors.is_empty()); + + status.requests_created_ok.len() + + status.start_invoked_ok.len() + + status.requests_completed_ok.len() +} + +/// Run the region_replacement_driver background task, returning how many actions +/// were taken +pub async fn run_region_replacement_driver( + internal_client: &ClientTestContext, +) -> usize { + let last_background_task = + activate_background_task(&internal_client, "region_replacement_driver") + .await; + + let LastResult::Completed(last_result_completed) = + last_background_task.last + else { + panic!( + "unexpected {:?} returned from region_replacement_driver task", + last_background_task.last, + ); + }; + + let status = serde_json::from_value::( + last_result_completed.details, + ) + .unwrap(); + + assert!(status.errors.is_empty()); + + status.drive_invoked_ok.len() + status.finish_invoked_ok.len() +} + +/// Run the region_snapshot_replacement_start background task, returning how many +/// actions were taken +pub async fn run_region_snapshot_replacement_start( + internal_client: &ClientTestContext, +) -> usize { + let last_background_task = activate_background_task( + &internal_client, + "region_snapshot_replacement_start", + ) + .await; + + let LastResult::Completed(last_result_completed) = + last_background_task.last + else { + panic!( + "unexpected {:?} returned from region_snapshot_replacement_start \ + task", + last_background_task.last, + ); + }; + + let status = + serde_json::from_value::( + last_result_completed.details, + ) + .unwrap(); + + assert!(status.errors.is_empty()); + + status.requests_created_ok.len() + status.start_invoked_ok.len() +} + +/// Run the region_snapshot_replacement_garbage_collection background task, +/// returning how many actions were taken +pub async fn run_region_snapshot_replacement_garbage_collection( + internal_client: &ClientTestContext, +) -> usize { + let last_background_task = activate_background_task( + &internal_client, + "region_snapshot_replacement_garbage_collection", + ) + .await; + + let LastResult::Completed(last_result_completed) = + last_background_task.last + else { + panic!( + "unexpected {:?} returned from \ + region_snapshot_replacement_garbage_collection task", + last_background_task.last, + ); + }; + + let status = serde_json::from_value::< + RegionSnapshotReplacementGarbageCollectStatus, + >(last_result_completed.details) + .unwrap(); + + assert!(status.errors.is_empty()); + + status.garbage_collect_requested.len() +} + +/// Run the region_snapshot_replacement_step background task, returning how many +/// actions were taken +pub async fn run_region_snapshot_replacement_step( + internal_client: &ClientTestContext, +) -> usize { + let last_background_task = activate_background_task( + &internal_client, + "region_snapshot_replacement_step", + ) + .await; + + let LastResult::Completed(last_result_completed) = + last_background_task.last + else { + panic!( + "unexpected {:?} returned from region_snapshot_replacement_step \ + task", + last_background_task.last, + ); + }; + + let status = serde_json::from_value::( + last_result_completed.details, + ) + .unwrap(); + + eprintln!("{:?}", &status.errors); + + assert!(status.errors.is_empty()); + + status.step_records_created_ok.len() + + status.step_garbage_collect_invoked_ok.len() + + status.step_invoked_ok.len() +} + +/// Run the region_snapshot_replacement_finish background task, returning how many +/// actions were taken +pub async fn run_region_snapshot_replacement_finish( + internal_client: &ClientTestContext, +) -> usize { + let last_background_task = activate_background_task( + &internal_client, + "region_snapshot_replacement_finish", + ) + .await; + + let LastResult::Completed(last_result_completed) = + last_background_task.last + else { + panic!( + "unexpected {:?} returned from region_snapshot_replacement_finish \ + task", + last_background_task.last, + ); + }; + + let status = + serde_json::from_value::( + last_result_completed.details, + ) + .unwrap(); + + assert!(status.errors.is_empty()); + + status.records_set_to_done.len() +} + +/// Run all replacement related background tasks until they aren't doing +/// anything anymore. +pub async fn run_replacement_tasks_to_completion( + internal_client: &ClientTestContext, +) { + wait_for_condition( + || async { + let actions_taken = + // region replacement related + run_region_replacement(internal_client).await + + run_region_replacement_driver(internal_client).await + + // region snapshot replacement related + run_region_snapshot_replacement_start(internal_client).await + + run_region_snapshot_replacement_garbage_collection(internal_client).await + + run_region_snapshot_replacement_step(internal_client).await + + run_region_snapshot_replacement_finish(internal_client).await; + + if actions_taken > 0 { + Err(CondCheckError::<()>::NotYet) + } else { + Ok(()) + } + }, + &Duration::from_secs(1), + &Duration::from_secs(10), + ) + .await + .unwrap(); +} diff --git a/nexus/test-utils/src/lib.rs b/nexus/test-utils/src/lib.rs index 432478b8c2..d69abbd93e 100644 --- a/nexus/test-utils/src/lib.rs +++ b/nexus/test-utils/src/lib.rs @@ -9,7 +9,6 @@ use anyhow::Context; use anyhow::Result; use camino::Utf8Path; use chrono::Utc; -use dns_service_client::types::DnsConfigParams; use dropshot::test_util::ClientTestContext; use dropshot::test_util::LogContext; use dropshot::ConfigLogging; @@ -23,6 +22,9 @@ use hickory_resolver::config::Protocol; use hickory_resolver::config::ResolverConfig; use hickory_resolver::config::ResolverOpts; use hickory_resolver::TokioAsyncResolver; +use internal_dns_types::config::DnsConfigBuilder; +use internal_dns_types::names::ServiceName; +use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use nexus_config::Database; use nexus_config::DpdConfig; use nexus_config::InternalDns; @@ -45,6 +47,7 @@ use nexus_types::deployment::OmicronZoneExternalFloatingIp; use nexus_types::external_api::views::SledState; use nexus_types::internal_api::params::DatasetCreateRequest; use nexus_types::internal_api::params::DatasetPutRequest; +use nexus_types::internal_api::params::DnsConfigParams; use omicron_common::address::DNS_OPTE_IPV4_SUBNET; use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; use omicron_common::api::external::Generation; @@ -203,7 +206,7 @@ pub async fn test_setup( struct RackInitRequestBuilder { datasets: Vec, - internal_dns_config: internal_dns::DnsConfigBuilder, + internal_dns_config: DnsConfigBuilder, mac_addrs: Box + Send>, } @@ -211,7 +214,7 @@ impl RackInitRequestBuilder { fn new() -> Self { Self { datasets: vec![], - internal_dns_config: internal_dns::DnsConfigBuilder::new(), + internal_dns_config: DnsConfigBuilder::new(), mac_addrs: Box::new(MacAddr::iter_system()), } } @@ -220,7 +223,7 @@ impl RackInitRequestBuilder { &mut self, zone_id: OmicronZoneUuid, address: SocketAddrV6, - service_name: internal_dns::ServiceName, + service_name: ServiceName, ) { let zone = self .internal_dns_config @@ -240,7 +243,7 @@ impl RackInitRequestBuilder { dataset_id: Uuid, address: SocketAddrV6, kind: DatasetKind, - service_name: internal_dns::ServiceName, + service_name: ServiceName, ) { self.datasets.push(DatasetCreateRequest { zpool_id: zpool_id.into_untyped_uuid(), @@ -280,7 +283,7 @@ impl RackInitRequestBuilder { .host_zone_clickhouse( OmicronZoneUuid::from_untyped_uuid(dataset_id), *address.ip(), - internal_dns::ServiceName::Clickhouse, + ServiceName::Clickhouse, address.port(), ) .expect("Failed to setup ClickHouse DNS"); @@ -448,7 +451,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { dataset_id, address, DatasetKind::Cockroach, - internal_dns::ServiceName::Cockroach, + ServiceName::Cockroach, ); let pool_name = illumos_utils::zpool::ZpoolName::new_external(zpool_id) .to_string() @@ -692,7 +695,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { self.rack_init_builder.add_service_to_dns( nexus_id, address, - internal_dns::ServiceName::Nexus, + ServiceName::Nexus, ); self.blueprint_zones.push(BlueprintZoneConfig { @@ -778,8 +781,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { ); // Create a recovery silo - let external_dns_zone_name = - internal_dns::names::DNS_ZONE_EXTERNAL_TESTING.to_string(); + let external_dns_zone_name = DNS_ZONE_EXTERNAL_TESTING.to_string(); let silo_name: Name = "test-suite-silo".parse().unwrap(); let user_name = UserId::try_from("test-privileged".to_string()).unwrap(); @@ -834,6 +836,9 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { cockroachdb_fingerprint: String::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, + // Clickhouse clusters are not generated by RSS. One must run + // reconfigurator for that. + clickhouse_cluster_config: None, time_created: Utc::now(), creator: "nexus-test-utils".to_string(), comment: "initial test blueprint".to_string(), @@ -1016,7 +1021,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { self.rack_init_builder.add_service_to_dns( zone_id, address, - internal_dns::ServiceName::CruciblePantry, + ServiceName::CruciblePantry, ); self.blueprint_zones.push(BlueprintZoneConfig { disposition: BlueprintZoneDisposition::InService, @@ -1052,7 +1057,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { self.rack_init_builder.add_service_to_dns( zone_id, dropshot_address, - internal_dns::ServiceName::ExternalDns, + ServiceName::ExternalDns, ); let zpool_id = ZpoolUuid::new_v4(); @@ -1115,7 +1120,7 @@ impl<'a, N: NexusServer> ControlPlaneTestContextBuilder<'a, N> { self.rack_init_builder.add_service_to_dns( zone_id, http_address, - internal_dns::ServiceName::InternalDns, + ServiceName::InternalDns, ); let zpool_id = ZpoolUuid::new_v4(); diff --git a/nexus/test-utils/src/resource_helpers.rs b/nexus/test-utils/src/resource_helpers.rs index 8b1d0e5f6b..36f40be256 100644 --- a/nexus/test-utils/src/resource_helpers.rs +++ b/nexus/test-utils/src/resource_helpers.rs @@ -22,6 +22,9 @@ use nexus_types::external_api::shared::IpRange; use nexus_types::external_api::views; use nexus_types::external_api::views::Certificate; use nexus_types::external_api::views::FloatingIp; +use nexus_types::external_api::views::InternetGateway; +use nexus_types::external_api::views::InternetGatewayIpAddress; +use nexus_types::external_api::views::InternetGatewayIpPool; use nexus_types::external_api::views::IpPool; use nexus_types::external_api::views::IpPoolRange; use nexus_types::external_api::views::User; @@ -36,6 +39,7 @@ use omicron_common::api::external::IdentityMetadataCreateParams; use omicron_common::api::external::Instance; use omicron_common::api::external::InstanceAutoRestartPolicy; use omicron_common::api::external::InstanceCpuCount; +use omicron_common::api::external::Name; use omicron_common::api::external::NameOrId; use omicron_common::api::external::RouteDestination; use omicron_common::api::external::RouteTarget; @@ -742,6 +746,160 @@ pub async fn create_route_with_error( .unwrap() } +pub async fn create_internet_gateway( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + internet_gateway_name: &str, +) -> InternetGateway { + NexusRequest::objects_post( + &client, + format!( + "/v1/internet-gateways?project={}&vpc={}", + &project_name, &vpc_name + ) + .as_str(), + ¶ms::VpcRouterCreate { + identity: IdentityMetadataCreateParams { + name: internet_gateway_name.parse().unwrap(), + description: String::from("internet gateway description"), + }, + }, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + +pub async fn delete_internet_gateway( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + internet_gateway_name: &str, + cascade: bool, +) { + NexusRequest::object_delete( + &client, + format!( + "/v1/internet-gateways/{}?project={}&vpc={}&cascade={}", + &internet_gateway_name, &project_name, &vpc_name, cascade + ) + .as_str(), + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); +} + +pub async fn attach_ip_pool_to_igw( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, + ip_pool_name: &str, + attachment_name: &str, +) -> InternetGatewayIpPool { + let url = format!( + "/v1/internet-gateway-ip-pools?project={}&vpc={}&gateway={}", + project_name, vpc_name, igw_name, + ); + + let ip_pool: Name = ip_pool_name.parse().unwrap(); + NexusRequest::objects_post( + &client, + url.as_str(), + ¶ms::InternetGatewayIpPoolCreate { + identity: IdentityMetadataCreateParams { + name: attachment_name.parse().unwrap(), + description: String::from("attached pool descriptoion"), + }, + ip_pool: NameOrId::Name(ip_pool), + }, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + +pub async fn detach_ip_pool_from_igw( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, + ip_pool_name: &str, + cascade: bool, +) { + let url = format!( + "/v1/internet-gateway-ip-pools/{}?project={}&vpc={}&gateway={}&cascade={}", + ip_pool_name, project_name, vpc_name, igw_name, cascade, + ); + + NexusRequest::object_delete(&client, url.as_str()) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); +} + +pub async fn attach_ip_address_to_igw( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, + address: IpAddr, + attachment_name: &str, +) -> InternetGatewayIpAddress { + let url = format!( + "/v1/internet-gateway-ip-addresses?project={}&vpc={}&gateway={}", + project_name, vpc_name, igw_name, + ); + + NexusRequest::objects_post( + &client, + url.as_str(), + ¶ms::InternetGatewayIpAddressCreate { + identity: IdentityMetadataCreateParams { + name: attachment_name.parse().unwrap(), + description: String::from("attached pool descriptoion"), + }, + address, + }, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + +pub async fn detach_ip_address_from_igw( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, + attachment_name: &str, + cascade: bool, +) { + let url = format!( + "/v1/internet-gateway-ip-addresses/{}?project={}&vpc={}&gateway={}&cascade={}", + attachment_name, project_name, vpc_name, igw_name, cascade + ); + + NexusRequest::object_delete(&client, url.as_str()) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap(); +} + pub async fn assert_ip_pool_utilization( client: &ClientTestContext, pool_name: &str, diff --git a/nexus/tests/config.test.toml b/nexus/tests/config.test.toml index c8775aace9..94d22d491f 100644 --- a/nexus/tests/config.test.toml +++ b/nexus/tests/config.test.toml @@ -112,12 +112,12 @@ blueprints.period_secs_execute = 600 blueprints.period_secs_collect_crdb_node_ids = 600 sync_service_zone_nat.period_secs = 30 switch_port_settings_manager.period_secs = 30 -region_replacement.period_secs = 30 +region_replacement.period_secs = 60 # The driver task should wake up frequently, something like every 10 seconds. # however, if it's this low it affects the test_omdb_success_cases test output. -# keep this 30 seconds, so that the test shows "triggered by an explicit +# keep this 60 seconds, so that the test shows "triggered by an explicit # signal" instead of "triggered by a periodic timer firing" -region_replacement_driver.period_secs = 30 +region_replacement_driver.period_secs = 60 instance_watcher.period_secs = 30 service_firewall_propagation.period_secs = 300 v2p_mapping_propagation.period_secs = 30 @@ -147,10 +147,10 @@ instance_updater.period_secs = 60 # removing them from the set of instances eligible for reincarnation. Thus, set # the period much longer than the default for test purposes. instance_reincarnation.period_secs = 600 -region_snapshot_replacement_start.period_secs = 30 -region_snapshot_replacement_garbage_collection.period_secs = 30 -region_snapshot_replacement_step.period_secs = 30 -region_snapshot_replacement_finish.period_secs = 30 +region_snapshot_replacement_start.period_secs = 60 +region_snapshot_replacement_garbage_collection.period_secs = 60 +region_snapshot_replacement_step.period_secs = 60 +region_snapshot_replacement_finish.period_secs = 60 [default_region_allocation_strategy] # we only have one sled in the test environment, so we need to use the diff --git a/nexus/tests/integration_tests/authn_http.rs b/nexus/tests/integration_tests/authn_http.rs index 73066220c4..cb98b45878 100644 --- a/nexus/tests/integration_tests/authn_http.rs +++ b/nexus/tests/integration_tests/authn_http.rs @@ -26,7 +26,7 @@ use nexus_db_queries::authn::external::spoof::SPOOF_SCHEME_NAME; use nexus_db_queries::authn::external::AuthenticatorContext; use nexus_db_queries::authn::external::HttpAuthnScheme; use nexus_db_queries::authn::external::SiloUserSilo; -use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID; +use nexus_types::silo::DEFAULT_SILO_ID; use std::collections::HashMap; use std::sync::Mutex; use uuid::Uuid; @@ -342,7 +342,7 @@ impl SiloUserSilo for WhoamiServerState { silo_user_id.to_string(), "7f927c86-3371-4295-c34a-e3246a4b9c02" ); - Ok(*DEFAULT_SILO_ID) + Ok(DEFAULT_SILO_ID) } } diff --git a/nexus/tests/integration_tests/certificates.rs b/nexus/tests/integration_tests/certificates.rs index e855a7e57b..ea3979c7b9 100644 --- a/nexus/tests/integration_tests/certificates.rs +++ b/nexus/tests/integration_tests/certificates.rs @@ -10,7 +10,7 @@ use dropshot::HttpErrorResponseBody; use futures::TryStreamExt; use http::method::Method; use http::StatusCode; -use internal_dns::names::DNS_ZONE_EXTERNAL_TESTING; +use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::load_test_config; diff --git a/nexus/tests/integration_tests/console_api.rs b/nexus/tests/integration_tests/console_api.rs index 479baf2fec..505821abe2 100644 --- a/nexus/tests/integration_tests/console_api.rs +++ b/nexus/tests/integration_tests/console_api.rs @@ -12,7 +12,7 @@ use std::env::current_dir; use crate::integration_tests::saml::SAML_RESPONSE_IDP_DESCRIPTOR; use base64::Engine; -use internal_dns::names::DNS_ZONE_EXTERNAL_TESTING; +use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use nexus_db_queries::authn::{USER_TEST_PRIVILEGED, USER_TEST_UNPRIVILEGED}; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_db_queries::db::identity::{Asset, Resource}; diff --git a/nexus/tests/integration_tests/crucible_replacements.rs b/nexus/tests/integration_tests/crucible_replacements.rs new file mode 100644 index 0000000000..2f5317e249 --- /dev/null +++ b/nexus/tests/integration_tests/crucible_replacements.rs @@ -0,0 +1,483 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Tests related to region and region snapshot replacement + +use dropshot::test_util::ClientTestContext; +use nexus_db_model::PhysicalDiskPolicy; +use nexus_db_model::RegionReplacementState; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::lookup::LookupPath; +use nexus_db_queries::db::DataStore; +use nexus_test_utils::background::*; +use nexus_test_utils::http_testing::AuthnMode; +use nexus_test_utils::http_testing::NexusRequest; +use nexus_test_utils::resource_helpers::create_default_ip_pool; +use nexus_test_utils::resource_helpers::create_disk; +use nexus_test_utils::resource_helpers::create_project; +use nexus_test_utils_macros::nexus_test; +use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; +use omicron_uuid_kinds::GenericUuid; +use slog::Logger; +use std::sync::Arc; +use uuid::Uuid; + +type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + +type DiskTest<'a> = + nexus_test_utils::resource_helpers::DiskTest<'a, omicron_nexus::Server>; + +type DiskTestBuilder<'a> = nexus_test_utils::resource_helpers::DiskTestBuilder< + 'a, + omicron_nexus::Server, +>; + +const PROJECT_NAME: &str = "now-this-is-pod-racing"; + +fn get_disk_url(disk_name: &str) -> String { + format!("/v1/disks/{disk_name}?project={}", PROJECT_NAME) +} + +async fn create_project_and_pool(client: &ClientTestContext) -> Uuid { + create_default_ip_pool(client).await; + let project = create_project(client, PROJECT_NAME).await; + project.identity.id +} + +/// Assert that the first part of region replacement does not create a freed +/// crucible region (that would be picked up by a volume delete saga) +#[nexus_test] +async fn test_region_replacement_does_not_create_freed_region( + cptestctx: &ControlPlaneTestContext, +) { + let nexus = &cptestctx.server.server_context().nexus; + let datastore = nexus.datastore(); + let opctx = + OpContext::for_tests(cptestctx.logctx.log.new(o!()), datastore.clone()); + + // Create four zpools, each with one dataset. This is required for region + // and region snapshot replacement to have somewhere to move the data. + let sled_id = cptestctx.first_sled(); + let disk_test = DiskTestBuilder::new(&cptestctx) + .on_specific_sled(sled_id) + .with_zpool_count(4) + .build() + .await; + + // Create a disk + let client = &cptestctx.external_client; + let _project_id = create_project_and_pool(client).await; + + let disk = create_disk(&client, PROJECT_NAME, "disk").await; + + // Before expunging the physical disk, save the DB model + let (.., db_disk) = LookupPath::new(&opctx, &datastore) + .disk_id(disk.identity.id) + .fetch() + .await + .unwrap(); + + assert_eq!(db_disk.id(), disk.identity.id); + + // Next, expunge a physical disk that contains a region + + let disk_allocated_regions = + datastore.get_allocated_regions(db_disk.volume_id).await.unwrap(); + let (dataset, _) = &disk_allocated_regions[0]; + let zpool = disk_test + .zpools() + .find(|x| *x.id.as_untyped_uuid() == dataset.pool_id) + .expect("Expected at least one zpool"); + + let (_, db_zpool) = LookupPath::new(&opctx, datastore) + .zpool_id(zpool.id.into_untyped_uuid()) + .fetch() + .await + .unwrap(); + + datastore + .physical_disk_update_policy( + &opctx, + db_zpool.physical_disk_id, + PhysicalDiskPolicy::Expunged, + ) + .await + .unwrap(); + + // Now, run the first part of region replacement: this will move the deleted + // region into a temporary volume. + + let internal_client = &cptestctx.internal_client; + + let _ = + activate_background_task(&internal_client, "region_replacement").await; + + // Assert there are no freed crucible regions that result from that + assert!(datastore.find_deleted_volume_regions().await.unwrap().is_empty()); +} + +struct RegionReplacementDeletedVolumeTest<'a> { + log: Logger, + datastore: Arc, + disk_test: DiskTest<'a>, + client: ClientTestContext, + internal_client: ClientTestContext, + replacement_request_id: Uuid, +} + +#[derive(Debug)] +struct ExpectedEndState(pub RegionReplacementState); + +#[derive(Debug)] +struct ExpectedIntermediateState(pub RegionReplacementState); + +impl<'a> RegionReplacementDeletedVolumeTest<'a> { + pub async fn new(cptestctx: &'a ControlPlaneTestContext) -> Self { + let nexus = &cptestctx.server.server_context().nexus; + + // Create four zpools, each with one dataset. This is required for + // region and region snapshot replacement to have somewhere to move the + // data. + let disk_test = DiskTestBuilder::new(&cptestctx) + .on_specific_sled(cptestctx.first_sled()) + .with_zpool_count(4) + .build() + .await; + + let client = &cptestctx.external_client; + let internal_client = &cptestctx.internal_client; + let datastore = nexus.datastore().clone(); + + let opctx = OpContext::for_tests( + cptestctx.logctx.log.new(o!()), + datastore.clone(), + ); + + // Create a disk + let _project_id = create_project_and_pool(client).await; + + let disk = create_disk(&client, PROJECT_NAME, "disk").await; + + // Manually create the region replacement request for the first + // allocated region of that disk + + let (.., db_disk) = LookupPath::new(&opctx, &datastore) + .disk_id(disk.identity.id) + .fetch() + .await + .unwrap(); + + assert_eq!(db_disk.id(), disk.identity.id); + + let disk_allocated_regions = + datastore.get_allocated_regions(db_disk.volume_id).await.unwrap(); + let (_, region) = &disk_allocated_regions[0]; + + let replacement_request_id = datastore + .create_region_replacement_request_for_region(&opctx, ®ion) + .await + .unwrap(); + + // Assert the request is in state Requested + + let region_replacement = datastore + .get_region_replacement_request_by_id( + &opctx, + replacement_request_id, + ) + .await + .unwrap(); + + assert_eq!( + region_replacement.replacement_state, + RegionReplacementState::Requested, + ); + + RegionReplacementDeletedVolumeTest { + log: cptestctx.logctx.log.new(o!()), + datastore, + disk_test, + client: client.clone(), + internal_client: internal_client.clone(), + replacement_request_id, + } + } + + pub fn opctx(&self) -> OpContext { + OpContext::for_tests(self.log.clone(), self.datastore.clone()) + } + + pub async fn delete_the_disk(&self) { + let disk_url = get_disk_url("disk"); + NexusRequest::object_delete(&self.client, &disk_url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("failed to delete disk"); + } + + /// Make sure: + /// + /// - all region replacement related background tasks run to completion + /// - this harness' region replacement request has transitioned to Complete + /// - no Crucible resources are leaked + pub async fn finish_test(&self) { + // Make sure that all the background tasks can run to completion. + + run_replacement_tasks_to_completion(&self.internal_client).await; + + // Assert the request is in state Complete + + let region_replacement = self + .datastore + .get_region_replacement_request_by_id( + &self.opctx(), + self.replacement_request_id, + ) + .await + .unwrap(); + + assert_eq!( + region_replacement.replacement_state, + RegionReplacementState::Complete, + ); + + // Assert there are no more Crucible resources + + assert!(self.disk_test.crucible_resources_deleted().await); + } + + async fn wait_for_request_state( + &self, + expected_end_state: ExpectedEndState, + expected_intermediate_state: ExpectedIntermediateState, + ) { + wait_for_condition( + || { + let datastore = self.datastore.clone(); + let opctx = self.opctx(); + let replacement_request_id = self.replacement_request_id; + + async move { + let region_replacement = datastore + .get_region_replacement_request_by_id( + &opctx, + replacement_request_id, + ) + .await + .unwrap(); + + let state = region_replacement.replacement_state; + + if state == expected_end_state.0 { + // The saga transitioned the request ok + Ok(()) + } else if state == expected_intermediate_state.0 { + // The saga is still running + Err(CondCheckError::<()>::NotYet) + } else { + // Any other state is not expected + panic!("unexpected state {state:?}!"); + } + } + }, + &std::time::Duration::from_millis(500), + &std::time::Duration::from_secs(60), + ) + .await + .expect("request transitioned to expected state"); + + // Assert the request state + + let region_replacement = self + .datastore + .get_region_replacement_request_by_id( + &self.opctx(), + self.replacement_request_id, + ) + .await + .unwrap(); + + assert_eq!(region_replacement.replacement_state, expected_end_state.0); + } + + /// Run the "region replacement" task to transition the request to Running. + pub async fn transition_request_to_running(&self) { + // Activate the "region replacement" background task + + run_region_replacement(&self.internal_client).await; + + // The activation above could only have started the associated saga, so + // wait until the request is in state Running. + + self.wait_for_request_state( + ExpectedEndState(RegionReplacementState::Running), + ExpectedIntermediateState(RegionReplacementState::Allocating), + ) + .await; + } + + /// Call the region replacement drive task to attach the associated volume + /// to the simulated pantry, ostensibly for reconciliation + pub async fn attach_request_volume_to_pantry(&self) { + // Run the "region replacement driver" task to attach the associated + // volume to the simulated pantry. + + run_region_replacement_driver(&self.internal_client).await; + + // The activation above could only have started the associated saga, so + // wait until the request is in the expected end state. + + self.wait_for_request_state( + ExpectedEndState(RegionReplacementState::Running), + ExpectedIntermediateState(RegionReplacementState::Driving), + ) + .await; + + // Additionally, assert that the drive saga recorded that it sent the + // attachment request to the simulated pantry + + let most_recent_step = self + .datastore + .current_region_replacement_request_step( + &self.opctx(), + self.replacement_request_id, + ) + .await + .unwrap() + .unwrap(); + + assert!(most_recent_step.pantry_address().is_some()); + } + + /// Manually activate the background attachment for the request volume + pub async fn manually_activate_attached_volume( + &self, + cptestctx: &'a ControlPlaneTestContext, + ) { + let pantry = + cptestctx.sled_agent.pantry_server.as_ref().unwrap().pantry.clone(); + + let region_replacement = self + .datastore + .get_region_replacement_request_by_id( + &self.opctx(), + self.replacement_request_id, + ) + .await + .unwrap(); + + pantry + .activate_background_attachment( + region_replacement.volume_id.to_string(), + ) + .await + .unwrap(); + } + + /// Transition request to ReplacementDone via the region replacement drive + /// saga + pub async fn transition_request_to_replacement_done(&self) { + // Run the "region replacement driver" task + + run_region_replacement_driver(&self.internal_client).await; + + // The activation above could only have started the associated saga, so + // wait until the request is in the expected end state. + + self.wait_for_request_state( + ExpectedEndState(RegionReplacementState::ReplacementDone), + ExpectedIntermediateState(RegionReplacementState::Driving), + ) + .await; + } +} + +/// Assert that a region replacement request in state "Requested" can have its +/// volume deleted and still transition to Complete +#[nexus_test] +async fn test_delete_volume_region_replacement_state_requested( + cptestctx: &ControlPlaneTestContext, +) { + let test_harness = RegionReplacementDeletedVolumeTest::new(cptestctx).await; + + // The request leaves the `new` function in state Requested: delete the + // disk, then finish the test. + + test_harness.delete_the_disk().await; + + test_harness.finish_test().await; +} + +/// Assert that a region replacement request in state "Running" can have its +/// volume deleted and still transition to Complete +#[nexus_test] +async fn test_delete_volume_region_replacement_state_running( + cptestctx: &ControlPlaneTestContext, +) { + let test_harness = RegionReplacementDeletedVolumeTest::new(cptestctx).await; + + // The request leaves the `new` function in state Requested: + // - transition the request to "Running" + // - delete the disk, then finish the test. + + test_harness.transition_request_to_running().await; + + test_harness.delete_the_disk().await; + + test_harness.finish_test().await; +} + +/// Assert that a region replacement request in state "Running" that has +/// additionally had its volume attached to a Pantry can have its volume deleted +/// and still transition to Complete +#[nexus_test] +async fn test_delete_volume_region_replacement_state_running_on_pantry( + cptestctx: &ControlPlaneTestContext, +) { + let test_harness = RegionReplacementDeletedVolumeTest::new(cptestctx).await; + + // The request leaves the `new` function in state Requested: + // - transition the request to "Running" + // - call the drive task to attach the volume to the simulated pantry + // - delete the disk, then finish the test. + + test_harness.transition_request_to_running().await; + + test_harness.attach_request_volume_to_pantry().await; + + test_harness.delete_the_disk().await; + + test_harness.finish_test().await; +} + +/// Assert that a region replacement request in state "ReplacementDone" can have +/// its volume deleted and still transition to Complete +#[nexus_test] +async fn test_delete_volume_region_replacement_state_replacement_done( + cptestctx: &ControlPlaneTestContext, +) { + let test_harness = RegionReplacementDeletedVolumeTest::new(cptestctx).await; + + // The request leaves the `new` function in state Requested: + // - transition the request to "Running" + // - call the drive task to attach the volume to the simulated pantry + // - simulate that the volume activated ok + // - call the drive task again, which will observe that activation and + // transition the request to "ReplacementDone" + // - delete the disk, then finish the test. + + test_harness.transition_request_to_running().await; + + test_harness.attach_request_volume_to_pantry().await; + + test_harness.manually_activate_attached_volume(&cptestctx).await; + + test_harness.transition_request_to_replacement_done().await; + + test_harness.delete_the_disk().await; + + test_harness.finish_test().await; +} diff --git a/nexus/tests/integration_tests/disks.rs b/nexus/tests/integration_tests/disks.rs index c9659d6bb8..d6d7cb58ea 100644 --- a/nexus/tests/integration_tests/disks.rs +++ b/nexus/tests/integration_tests/disks.rs @@ -17,7 +17,7 @@ use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::RegionAllocationFor; use nexus_db_queries::db::datastore::RegionAllocationParameters; use nexus_db_queries::db::datastore::REGION_REDUNDANCY_THRESHOLD; -use nexus_db_queries::db::fixed_data::{silo::DEFAULT_SILO_ID, FLEET_ID}; +use nexus_db_queries::db::fixed_data::FLEET_ID; use nexus_db_queries::db::lookup::LookupPath; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::Collection; @@ -33,6 +33,7 @@ use nexus_test_utils::resource_helpers::objects_list_page_authz; use nexus_test_utils::SLED_AGENT_UUID; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::params; +use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Disk; use omicron_common::api::external::DiskState; @@ -1108,7 +1109,7 @@ async fn test_disk_virtual_provisioning_collection( 0 ); let virtual_provisioning_collection = datastore - .virtual_provisioning_collection_get(&opctx, *DEFAULT_SILO_ID) + .virtual_provisioning_collection_get(&opctx, DEFAULT_SILO_ID) .await .unwrap(); assert_eq!( @@ -1172,7 +1173,7 @@ async fn test_disk_virtual_provisioning_collection( 0 ); let virtual_provisioning_collection = datastore - .virtual_provisioning_collection_get(&opctx, *DEFAULT_SILO_ID) + .virtual_provisioning_collection_get(&opctx, DEFAULT_SILO_ID) .await .unwrap(); assert_eq!( @@ -1229,7 +1230,7 @@ async fn test_disk_virtual_provisioning_collection( disk_size ); let virtual_provisioning_collection = datastore - .virtual_provisioning_collection_get(&opctx, *DEFAULT_SILO_ID) + .virtual_provisioning_collection_get(&opctx, DEFAULT_SILO_ID) .await .unwrap(); assert_eq!( @@ -1267,7 +1268,7 @@ async fn test_disk_virtual_provisioning_collection( 0 ); let virtual_provisioning_collection = datastore - .virtual_provisioning_collection_get(&opctx, *DEFAULT_SILO_ID) + .virtual_provisioning_collection_get(&opctx, DEFAULT_SILO_ID) .await .unwrap(); assert_eq!( diff --git a/nexus/tests/integration_tests/endpoints.rs b/nexus/tests/integration_tests/endpoints.rs index ff4c5a8712..d6f83063a0 100644 --- a/nexus/tests/integration_tests/endpoints.rs +++ b/nexus/tests/integration_tests/endpoints.rs @@ -10,7 +10,7 @@ use crate::integration_tests::unauthorized::HTTP_SERVER; use chrono::Utc; use http::method::Method; -use internal_dns::names::DNS_ZONE_EXTERNAL_TESTING; +use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use nexus_db_queries::authn; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_db_queries::db::identity::Resource; @@ -253,6 +253,81 @@ pub static DEMO_ROUTER_ROUTE_CREATE: Lazy = destination: RouteDestination::Subnet("loopback".parse().unwrap()), }); +// Internet Gateway used for testing +pub static DEMO_INTERNET_GATEWAY_NAME: Lazy = + Lazy::new(|| "demo-internet-gateway".parse().unwrap()); +pub static DEMO_INTERNET_GATEWAYS_URL: Lazy = Lazy::new(|| { + format!( + "/v1/internet-gateways?project={}&vpc={}", + *DEMO_PROJECT_NAME, *DEMO_VPC_NAME + ) +}); +pub static DEMO_INTERNET_GATEWAY_URL: Lazy = Lazy::new(|| { + format!( + "/v1/internet-gateways/{}?project={}&vpc={}", + *DEMO_INTERNET_GATEWAY_NAME, *DEMO_PROJECT_NAME, *DEMO_VPC_NAME + ) +}); +pub static DEMO_INTERNET_GATEWAY_CREATE: Lazy = + Lazy::new(|| params::InternetGatewayCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_INTERNET_GATEWAY_NAME.clone(), + description: String::from(""), + }, + }); +pub static DEMO_INTERNET_GATEWAY_IP_POOL_CREATE: Lazy< + params::InternetGatewayIpPoolCreate, +> = Lazy::new(|| params::InternetGatewayIpPoolCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_INTERNET_GATEWAY_NAME.clone(), + description: String::from(""), + }, + ip_pool: NameOrId::Id(uuid::Uuid::new_v4()), +}); +pub static DEMO_INTERNET_GATEWAY_IP_ADDRESS_CREATE: Lazy< + params::InternetGatewayIpAddressCreate, +> = Lazy::new(|| params::InternetGatewayIpAddressCreate { + identity: IdentityMetadataCreateParams { + name: DEMO_INTERNET_GATEWAY_NAME.clone(), + description: String::from(""), + }, + address: IpAddr::V4(Ipv4Addr::UNSPECIFIED), +}); +pub static DEMO_INTERNET_GATEWAY_IP_POOLS_URL: Lazy = Lazy::new(|| { + format!( + "/v1/internet-gateway-ip-pools?project={}&vpc={}&gateway={}", + *DEMO_PROJECT_NAME, *DEMO_VPC_NAME, *DEMO_INTERNET_GATEWAY_NAME, + ) +}); +pub static DEMO_INTERNET_GATEWAY_IP_ADDRS_URL: Lazy = Lazy::new(|| { + format!( + "/v1/internet-gateway-ip-addresses?project={}&vpc={}&gateway={}", + *DEMO_PROJECT_NAME, *DEMO_VPC_NAME, *DEMO_INTERNET_GATEWAY_NAME, + ) +}); +pub static DEMO_INTERNET_GATEWAY_IP_POOL_NAME: Lazy = + Lazy::new(|| "demo-igw-pool".parse().unwrap()); +pub static DEMO_INTERNET_GATEWAY_IP_ADDRESS_NAME: Lazy = + Lazy::new(|| "demo-igw-address".parse().unwrap()); +pub static DEMO_INTERNET_GATEWAY_IP_POOL_URL: Lazy = Lazy::new(|| { + format!( + "/v1/internet-gateway-ip-pools/{}?project={}&vpc={}&gateway={}", + *DEMO_INTERNET_GATEWAY_IP_POOL_NAME, + *DEMO_PROJECT_NAME, + *DEMO_VPC_NAME, + *DEMO_INTERNET_GATEWAY_NAME, + ) +}); +pub static DEMO_INTERNET_GATEWAY_IP_ADDR_URL: Lazy = Lazy::new(|| { + format!( + "/v1/internet-gateway-ip-addresses/{}?project={}&vpc={}&gateway={}", + *DEMO_INTERNET_GATEWAY_IP_ADDRESS_NAME, + *DEMO_PROJECT_NAME, + *DEMO_VPC_NAME, + *DEMO_INTERNET_GATEWAY_NAME, + ) +}); + // Disk used for testing pub static DEMO_DISK_NAME: Lazy = Lazy::new(|| "demo-disk".parse().unwrap()); @@ -434,7 +509,10 @@ pub static DEMO_INSTANCE_CREATE: Lazy = auto_restart_policy: Default::default(), }); pub static DEMO_INSTANCE_UPDATE: Lazy = - Lazy::new(|| params::InstanceUpdate { boot_disk: None }); + Lazy::new(|| params::InstanceUpdate { + boot_disk: None, + auto_restart_policy: None, + }); // The instance needs a network interface, too. pub static DEMO_INSTANCE_NIC_NAME: Lazy = @@ -1606,6 +1684,72 @@ pub static VERIFY_ENDPOINTS: Lazy> = Lazy::new(|| { ], }, + /* Internet Gateways */ + + VerifyEndpoint { + url: &DEMO_INTERNET_GATEWAYS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + AllowedMethod::Post( + serde_json::to_value(&*DEMO_INTERNET_GATEWAY_CREATE).unwrap() + ), + ], + }, + + VerifyEndpoint { + url: &DEMO_INTERNET_GATEWAY_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + AllowedMethod::Delete, + ], + }, + + VerifyEndpoint { + url: &DEMO_INTERNET_GATEWAY_IP_POOLS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + AllowedMethod::Post( + serde_json::to_value(&*DEMO_INTERNET_GATEWAY_IP_POOL_CREATE).unwrap() + ), + ], + }, + + VerifyEndpoint { + url: &DEMO_INTERNET_GATEWAY_IP_POOL_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Delete, + ], + }, + + VerifyEndpoint { + url: &DEMO_INTERNET_GATEWAY_IP_ADDRS_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::GetNonexistent, + AllowedMethod::Post( + serde_json::to_value(&*DEMO_INTERNET_GATEWAY_IP_ADDRESS_CREATE).unwrap() + ), + ], + }, + + VerifyEndpoint { + url: &DEMO_INTERNET_GATEWAY_IP_ADDR_URL, + visibility: Visibility::Protected, + unprivileged_access: UnprivilegedAccess::None, + allowed_methods: vec![ + AllowedMethod::Delete, + ], + }, + /* Disks */ VerifyEndpoint { diff --git a/nexus/tests/integration_tests/instances.rs b/nexus/tests/integration_tests/instances.rs index 6a94a189bd..9866c761ce 100644 --- a/nexus/tests/integration_tests/instances.rs +++ b/nexus/tests/integration_tests/instances.rs @@ -16,7 +16,6 @@ use http::StatusCode; use itertools::Itertools; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; -use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID; use nexus_db_queries::db::lookup::LookupPath; use nexus_db_queries::db::DataStore; use nexus_test_interface::NexusServer; @@ -49,6 +48,7 @@ use nexus_types::external_api::views::SshKey; use nexus_types::external_api::{params, views}; use nexus_types::identity::Resource; use nexus_types::internal_api::params::InstanceMigrateRequest; +use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::api::external::ByteCount; use omicron_common::api::external::Disk; use omicron_common::api::external::DiskState; @@ -1283,29 +1283,51 @@ async fn test_instance_failed_when_on_expunged_sled( let nexus = &apictx.nexus; let instance1_name = "romeo"; let instance2_name = "juliet"; + let instance3_name = "mercutio"; create_project_and_pool(&client).await; // Create and start the test instances. - let mk_instance = |name: &'static str| async move { - let instance_url = get_instance_url(name); - let instance = create_instance(client, PROJECT_NAME, name).await; - let instance_id = InstanceUuid::from_untyped_uuid(instance.identity.id); - instance_simulate(nexus, &instance_id).await; - let instance_next = instance_get(&client, &instance_url).await; - assert_eq!(instance_next.runtime.run_state, InstanceState::Running); - - slog::info!( - &cptestctx.logctx.log, - "test instance is running"; - "instance_name" => %name, - "instance_id" => %instance_id, - ); + let mk_instance = + |name: &'static str, auto_restart: InstanceAutoRestartPolicy| async move { + let instance_url = get_instance_url(name); + let instance = create_instance_with( + client, + PROJECT_NAME, + name, + ¶ms::InstanceNetworkInterfaceAttachment::Default, + // Disks= + Vec::::new(), + // External IPs= + Vec::::new(), + true, + Some(auto_restart), + ) + .await; + let instance_id = + InstanceUuid::from_untyped_uuid(instance.identity.id); + instance_simulate(nexus, &instance_id).await; + let instance_next = instance_get(&client, &instance_url).await; + assert_eq!(instance_next.runtime.run_state, InstanceState::Running); - instance_id - }; - let instance1_id = mk_instance(instance1_name).await; - let instance2_id = mk_instance(instance2_name).await; + slog::info!( + &cptestctx.logctx.log, + "test instance is running"; + "instance_name" => %name, + "instance_id" => %instance_id, + ); + + instance_id + }; + // We are going to manually attempt to delete/restart these instances when + // they go to `Failed`, so don't allow them to reincarnate. + let instance1_id = + mk_instance(instance1_name, InstanceAutoRestartPolicy::Never).await; + let instance2_id = + mk_instance(instance2_name, InstanceAutoRestartPolicy::Never).await; + let instance3_id = + mk_instance(instance3_name, InstanceAutoRestartPolicy::BestEffort) + .await; // Create a second sled for instances on the Expunged sled to be assigned // to. @@ -1351,11 +1373,17 @@ async fn test_instance_failed_when_on_expunged_sled( // ...or restartable. expect_instance_start_ok(client, instance2_name).await; - // The restarted instance shoild now transition back to `Running`, on its + // The restarted instance should now transition back to `Running`, on its // new sled. instance_wait_for_vmm_registration(cptestctx, &instance2_id).await; instance_simulate(nexus, &instance2_id).await; instance_wait_for_state(client, instance2_id, InstanceState::Running).await; + + // The auto-restartable instance should be...restarted automatically. + + instance_wait_for_vmm_registration(cptestctx, &instance3_id).await; + instance_simulate(nexus, &instance3_id).await; + instance_wait_for_state(client, instance3_id, InstanceState::Running).await; } // Verifies that the instance-watcher background task transitions an instance @@ -1613,7 +1641,7 @@ async fn assert_metrics( ram ); } - for id in &[None, Some(*DEFAULT_SILO_ID)] { + for id in &[None, Some(DEFAULT_SILO_ID)] { assert_eq!( get_latest_system_metric( cptestctx, @@ -4129,19 +4157,12 @@ async fn test_cannot_detach_boot_disk(cptestctx: &ControlPlaneTestContext) { assert_eq!(err.message, "boot disk cannot be detached"); // Change the instance's boot disk. - let url_instance_update = format!("/v1/instances/{}", instance.identity.id); - - let builder = - RequestBuilder::new(client, http::Method::PUT, &url_instance_update) - .body(Some(¶ms::InstanceUpdate { boot_disk: None })) - .expect_status(Some(http::StatusCode::OK)); - let response = NexusRequest::new(builder) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("can attempt to reconfigure the instance"); - - let instance = response.parsed_body::().unwrap(); + let instance = expect_instance_reconfigure_ok( + &client, + &instance.identity.id, + params::InstanceUpdate { boot_disk: None, auto_restart_policy: None }, + ) + .await; assert_eq!(instance.boot_disk_id, None); // Now try to detach `disks[0]` again. This should succeed. @@ -4160,14 +4181,32 @@ async fn test_cannot_detach_boot_disk(cptestctx: &ControlPlaneTestContext) { } #[nexus_test] -async fn test_updating_running_instance_is_conflict( +async fn test_updating_running_instance_boot_disk_is_conflict( cptestctx: &ControlPlaneTestContext, ) { let client = &cptestctx.external_client; let instance_name = "immediately-running"; + // Test pre-reqs + DiskTest::new(&cptestctx).await; create_project_and_pool(&client).await; + create_disk(&client, PROJECT_NAME, "probablydata").await; + create_disk(&client, PROJECT_NAME, "alsodata").await; + + // Verify disk is there and currently detached + let disks: Vec = + NexusRequest::iter_collection_authn(client, &get_disks_url(), "", None) + .await + .expect("failed to list disks") + .all_items; + assert_eq!(disks.len(), 2); + assert_eq!(disks[0].state, DiskState::Detached); + assert_eq!(disks[1].state, DiskState::Detached); + + let probablydata = Name::try_from(String::from("probablydata")).unwrap(); + let alsodata = Name::try_from(String::from("alsodata")).unwrap(); + let instance_params = params::InstanceCreate { identity: IdentityMetadataCreateParams { name: Name::try_from(String::from(instance_name)).unwrap(), @@ -4180,8 +4219,17 @@ async fn test_updating_running_instance_is_conflict( ssh_public_keys: None, network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, external_ips: vec![], - disks: vec![], - boot_disk: None, + disks: vec![ + params::InstanceDiskAttachment::Attach( + params::InstanceDiskAttach { name: probablydata.clone() }, + ), + params::InstanceDiskAttachment::Attach( + params::InstanceDiskAttach { name: alsodata.clone() }, + ), + ], + boot_disk: Some(params::InstanceDiskAttachment::Attach( + params::InstanceDiskAttach { name: probablydata.clone() }, + )), start: true, auto_restart_policy: Default::default(), }; @@ -4206,12 +4254,69 @@ async fn test_updating_running_instance_is_conflict( instance_simulate(nexus, &instance_id).await; instance_wait_for_state(client, instance_id, InstanceState::Running).await; - let url_instance_update = format!("/v1/instances/{}", instance_id); + let error = expect_instance_reconfigure_err( + &client, + &instance_id.into_untyped_uuid(), + params::InstanceUpdate { + boot_disk: Some(alsodata.clone().into()), + auto_restart_policy: None, + }, + http::StatusCode::CONFLICT, + ) + .await; + assert_eq!(error.message, "instance must be stopped to set boot disk"); - let builder = - RequestBuilder::new(client, http::Method::PUT, &url_instance_update) - .body(Some(¶ms::InstanceUpdate { boot_disk: None })) - .expect_status(Some(http::StatusCode::CONFLICT)); + // However, we can freely change the auto-restart policy of a running + // instance. + expect_instance_reconfigure_ok( + &client, + &instance_id.into_untyped_uuid(), + params::InstanceUpdate { + // Leave the boot disk the same as the one with which the instance + // was created. + boot_disk: Some(probablydata.clone().into()), + auto_restart_policy: Some(InstanceAutoRestartPolicy::BestEffort), + }, + ) + .await; +} + +#[nexus_test] +async fn test_updating_missing_instance_is_not_found( + cptestctx: &ControlPlaneTestContext, +) { + const UUID_THAT_DOESNT_EXIST: Uuid = + Uuid::from_u128(0x12341234_4321_8765_1234_432143214321); + + let client = &cptestctx.external_client; + + let error = expect_instance_reconfigure_err( + &client, + &UUID_THAT_DOESNT_EXIST, + params::InstanceUpdate { boot_disk: None, auto_restart_policy: None }, + http::StatusCode::NOT_FOUND, + ) + .await; + assert_eq!( + error.message, + format!("not found: instance with id \"{}\"", UUID_THAT_DOESNT_EXIST) + ); +} + +async fn expect_instance_reconfigure_ok( + external_client: &ClientTestContext, + instance_id: &Uuid, + update: params::InstanceUpdate, +) -> Instance { + let url_instance_update = format!("/v1/instances/{instance_id}"); + + let builder = RequestBuilder::new( + external_client, + http::Method::PUT, + &url_instance_update, + ) + .body(Some(&update)) + .expect_status(Some(http::StatusCode::OK)); let response = NexusRequest::new(builder) .authn_as(AuthnMode::PrivilegedUser) @@ -4219,37 +4324,105 @@ async fn test_updating_running_instance_is_conflict( .await .expect("can attempt to reconfigure the instance"); - let error = response.parsed_body::().unwrap(); - assert_eq!(error.message, "instance must be stopped to update"); + response + .parsed_body::() + .expect("response should be parsed as an instance") } +async fn expect_instance_reconfigure_err( + external_client: &ClientTestContext, + instance_id: &Uuid, + update: params::InstanceUpdate, + status: http::StatusCode, +) -> HttpErrorResponseBody { + let url_instance_update = format!("/v1/instances/{instance_id}"); + + let builder = RequestBuilder::new( + external_client, + http::Method::PUT, + &url_instance_update, + ) + .body(Some(&update)) + .expect_status(Some(status)); + + let response = NexusRequest::new(builder) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .expect("can attempt to reconfigure the instance"); + + response + .parsed_body::() + .expect("error response should parse successfully") +} +// Test reconfiguring an instance's auto-restart policy. #[nexus_test] -async fn test_updating_missing_instance_is_not_found( +async fn test_auto_restart_policy_can_be_changed( cptestctx: &ControlPlaneTestContext, ) { - const UUID_THAT_DOESNT_EXIST: Uuid = - Uuid::from_u128(0x12341234_4321_8765_1234_432143214321); - let url_instance_update = - format!("/v1/instances/{}", UUID_THAT_DOESNT_EXIST); - let client = &cptestctx.external_client; + let instance_name = "reincarnation-station"; - let builder = - RequestBuilder::new(client, http::Method::PUT, &url_instance_update) - .body(Some(¶ms::InstanceUpdate { boot_disk: None })) - .expect_status(Some(http::StatusCode::NOT_FOUND)); + create_project_and_pool(&client).await; + let instance_params = params::InstanceCreate { + identity: IdentityMetadataCreateParams { + name: instance_name.parse().unwrap(), + description: String::from("stuff"), + }, + ncpus: InstanceCpuCount::try_from(2).unwrap(), + memory: ByteCount::from_gibibytes_u32(4), + hostname: instance_name.parse().unwrap(), + user_data: vec![], + ssh_public_keys: None, + network_interfaces: params::InstanceNetworkInterfaceAttachment::Default, + external_ips: vec![], + boot_disk: None, + disks: Vec::new(), + start: true, + // Start out with None + auto_restart_policy: None, + }; + + let builder = + RequestBuilder::new(client, http::Method::POST, &get_instances_url()) + .body(Some(&instance_params)) + .expect_status(Some(http::StatusCode::CREATED)); let response = NexusRequest::new(builder) .authn_as(AuthnMode::PrivilegedUser) .execute() .await - .expect("can attempt to reconfigure the instance"); + .expect("Expected instance creation to work!"); - let error = response.parsed_body::().unwrap(); - assert_eq!( - error.message, - format!("not found: instance with id \"{}\"", UUID_THAT_DOESNT_EXIST) - ); + let instance = response.parsed_body::().unwrap(); + + // Starts out as None. + assert_eq!(instance.auto_restart_status.policy, None); + + let assert_reconfigured = |auto_restart_policy| async move { + let instance = expect_instance_reconfigure_ok( + client, + &instance.identity.id, + dbg!(params::InstanceUpdate { + auto_restart_policy, + boot_disk: None, + }), + ) + .await; + assert_eq!( + dbg!(instance).auto_restart_status.policy, + auto_restart_policy, + ); + }; + + // Reconfigure to Never. + assert_reconfigured(Some(InstanceAutoRestartPolicy::Never)).await; + + // Reconfigure to BestEffort + assert_reconfigured(Some(InstanceAutoRestartPolicy::BestEffort)).await; + + // Reconfigure back to None. + assert_reconfigured(None).await; } // Create an instance with boot disk set to one of its attached disks, then set @@ -4318,23 +4491,17 @@ async fn test_boot_disk_can_be_changed(cptestctx: &ControlPlaneTestContext) { assert_eq!(instance.boot_disk_id, Some(disks[0].identity.id)); - // Change the instance's boot disk. - let url_instance_update = format!("/v1/instances/{}", instance.identity.id); - - let builder = - RequestBuilder::new(client, http::Method::PUT, &url_instance_update) - .body(Some(¶ms::InstanceUpdate { - boot_disk: Some(disks[1].identity.id.into()), - })) - .expect_status(Some(http::StatusCode::OK)); - - let response = NexusRequest::new(builder) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("can attempt to reconfigure the instance"); + // Change the instance's boot disk.ity.id); - let instance = response.parsed_body::().unwrap(); + let instance = expect_instance_reconfigure_ok( + &client, + &instance.identity.id, + params::InstanceUpdate { + boot_disk: Some(disks[1].identity.id.into()), + auto_restart_policy: None, + }, + ) + .await; assert_eq!(instance.boot_disk_id, Some(disks[1].identity.id)); } @@ -4390,22 +4557,16 @@ async fn test_boot_disk_must_be_attached(cptestctx: &ControlPlaneTestContext) { let instance = response.parsed_body::().unwrap(); // Update the instance's boot disk to the unattached disk. This should fail. - let url_instance_update = format!("/v1/instances/{}", instance.identity.id); - - let builder = - RequestBuilder::new(client, http::Method::PUT, &url_instance_update) - .body(Some(¶ms::InstanceUpdate { - boot_disk: Some(disks[0].identity.id.into()), - })) - .expect_status(Some(http::StatusCode::CONFLICT)); - let response = NexusRequest::new(builder) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("can attempt to reconfigure the instance"); - - let error = - response.parsed_body::().unwrap(); + let error = expect_instance_reconfigure_err( + &client, + &instance.identity.id, + params::InstanceUpdate { + boot_disk: Some(disks[0].identity.id.into()), + auto_restart_policy: None, + }, + http::StatusCode::CONFLICT, + ) + .await; assert_eq!(error.message, format!("boot disk must be attached")); @@ -4427,19 +4588,15 @@ async fn test_boot_disk_must_be_attached(cptestctx: &ControlPlaneTestContext) { .expect("can attempt to detach boot disk"); // And now it can be made the boot disk. - let builder = - RequestBuilder::new(client, http::Method::PUT, &url_instance_update) - .body(Some(¶ms::InstanceUpdate { - boot_disk: Some(disks[0].identity.id.into()), - })) - .expect_status(Some(http::StatusCode::OK)); - let response = NexusRequest::new(builder) - .authn_as(AuthnMode::PrivilegedUser) - .execute() - .await - .expect("can attempt to reconfigure the instance"); - - let instance = response.parsed_body::().unwrap(); + let instance = expect_instance_reconfigure_ok( + &client, + &instance.identity.id, + params::InstanceUpdate { + boot_disk: Some(disks[0].identity.id.into()), + auto_restart_policy: None, + }, + ) + .await; assert_eq!(instance.boot_disk_id, Some(disks[0].identity.id)); } @@ -6156,7 +6313,6 @@ pub async fn assert_sled_vpc_routes( .await .unwrap() .into_iter() - .map(|(dest, target)| ResolvedVpcRoute { dest, target }) .collect() } else { Default::default() @@ -6173,38 +6329,69 @@ pub async fn assert_sled_vpc_routes( .await .unwrap() .into_iter() - .map(|(dest, target)| ResolvedVpcRoute { dest, target }) .collect(); assert!(!system_routes.is_empty()); let condition = || async { + let sys_key = RouterId { vni, kind: RouterKind::System }; + let custom_key = RouterId { + vni, + kind: RouterKind::Custom(db_subnet.ipv4_block.0.into()), + }; + let vpc_routes = sled_agent.vpc_routes.lock().await; - let sys_routes_found = vpc_routes.iter().any(|(id, set)| { - *id == RouterId { vni, kind: RouterKind::System } - && set.routes == system_routes - }); - let custom_routes_found = vpc_routes.iter().any(|(id, set)| { - *id == RouterId { - vni, - kind: RouterKind::Custom(db_subnet.ipv4_block.0.into()), - } && set.routes == custom_routes - }); + let sys_routes_found = vpc_routes + .iter() + .any(|(id, set)| *id == sys_key && set.routes == system_routes); + let custom_routes_found = vpc_routes + .iter() + .any(|(id, set)| *id == custom_key && set.routes == custom_routes); if sys_routes_found && custom_routes_found { Ok(()) } else { + let found_system = + vpc_routes.get(&sys_key).cloned().unwrap_or_default(); + let found_custom = + vpc_routes.get(&custom_key).cloned().unwrap_or_default(); + + println!("unexpected route setup"); + println!("vni: {vni:?}"); + println!("sled: {}", sled_agent.id); + println!("subnet: {}", db_subnet.ipv4_block.0); + println!("expected system: {system_routes:?}"); + println!("expected custom {custom_routes:?}"); + println!("found: {vpc_routes:?}"); + println!( + "\n-----\nsystem diff (-): {:?}", + system_routes.difference(&found_system.routes) + ); + println!( + "system diff (+): {:?}", + found_system.routes.difference(&system_routes) + ); + println!( + "custom diff (-): {:?}", + custom_routes.difference(&found_custom.routes) + ); + println!( + "custom diff (+): {:?}\n-----", + found_custom.routes.difference(&custom_routes) + ); Err(CondCheckError::NotYet::<()>) } }; wait_for_condition( condition, &Duration::from_secs(1), - &Duration::from_secs(30), + &Duration::from_secs(60), ) .await .expect("matching vpc routes should be present"); + println!("success! VPC routes as expected for sled {}", sled_agent.id); + (system_routes, custom_routes) } diff --git a/nexus/tests/integration_tests/internet_gateway.rs b/nexus/tests/integration_tests/internet_gateway.rs new file mode 100644 index 0000000000..11c3addee5 --- /dev/null +++ b/nexus/tests/integration_tests/internet_gateway.rs @@ -0,0 +1,620 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use dropshot::{test_util::ClientTestContext, ResultsPage}; +use http::{Method, StatusCode}; +use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; +use nexus_test_utils::{ + http_testing::{AuthnMode, NexusRequest}, + resource_helpers::{ + attach_ip_address_to_igw, attach_ip_pool_to_igw, create_floating_ip, + create_instance_with, create_internet_gateway, create_ip_pool, + create_local_user, create_project, create_route, create_router, + create_vpc, delete_internet_gateway, detach_ip_address_from_igw, + detach_ip_pool_from_igw, link_ip_pool, objects_list_page_authz, + }, +}; +use nexus_test_utils_macros::nexus_test; +use nexus_types::external_api::{ + params::{ + self, ExternalIpCreate, InstanceNetworkInterfaceAttachment, + InstanceNetworkInterfaceCreate, + }, + shared::SiloRole, + views::{InternetGateway, InternetGatewayIpAddress, InternetGatewayIpPool}, +}; +use nexus_types::identity::Resource; +use omicron_common::{ + address::{IpRange, Ipv4Range}, + api::external::{ + IdentityMetadataCreateParams, NameOrId, RouteDestination, RouteTarget, + }, +}; + +type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + +const PROJECT_NAME: &str = "delta-quadrant"; +const VPC_NAME: &str = "dominion"; +const IGW_NAME: &str = "wormhole"; +const IP_POOL_NAME: &str = "ds9"; +const IP_POOL_ATTACHMENT_NAME: &str = "runabout"; +const IP_ADDRESS_ATTACHMENT_NAME: &str = "defiant"; +const IP_ADDRESS_ATTACHMENT: &str = "198.51.100.47"; +const IP_ADDRESS_ATTACHMENT_FROM_POOL: &str = "203.0.113.1"; +const INSTANCE_NAME: &str = "odo"; +const FLOATING_IP_NAME: &str = "floater"; +const ROUTER_NAME: &str = "deepspace"; +const ROUTE_NAME: &str = "subspace"; + +#[nexus_test] +async fn test_internet_gateway_basic_crud(ctx: &ControlPlaneTestContext) { + let c = &ctx.external_client; + test_setup(c).await; + + // should start with just default gateway + let igws = list_internet_gateways(c, PROJECT_NAME, VPC_NAME).await; + assert_eq!(igws.len(), 1, "should start with zero internet gateways"); + expect_igw_not_found(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + + // create an internet gateway + let gw = create_internet_gateway(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + let igws = list_internet_gateways(c, PROJECT_NAME, VPC_NAME).await; + assert_eq!(igws.len(), 2, "should now have two internet gateways"); + + // should be able to get the gateway just created + let same_igw = get_igw(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + assert_eq!(&gw.identity, &same_igw.identity); + + // a new igw should have zero ip pools + let igw_pools = + list_internet_gateway_ip_pools(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!(igw_pools.len(), 0, "a new igw should have no pools"); + + // a new igw should have zero ip addresses + let igw_addrs = + list_internet_gateway_ip_addresses(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!(igw_addrs.len(), 0, "a new igw should have no addresses"); + + // attach an ip pool + attach_ip_pool_to_igw( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_POOL_NAME, + IP_POOL_ATTACHMENT_NAME, + ) + .await; + let igw_pools = + list_internet_gateway_ip_pools(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!(igw_pools.len(), 1, "should now have one attached ip pool"); + + // ensure we cannot delete the IP gateway without cascading + expect_igw_delete_fail(c, PROJECT_NAME, VPC_NAME, IGW_NAME, false).await; + + // ensure we cannot detach the igw ip pool without cascading + expect_igw_ip_pool_detach_fail( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_POOL_ATTACHMENT_NAME, + false, + ) + .await; + + // attach an ip address + attach_ip_address_to_igw( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_ADDRESS_ATTACHMENT.parse().unwrap(), + IP_ADDRESS_ATTACHMENT_NAME, + ) + .await; + let igw_pools = + list_internet_gateway_ip_addresses(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!(igw_pools.len(), 1, "should now have one attached address"); + + // detach an ip pool, note we need to cascade here since a running instance + // has a route that uses the ip pool association + detach_ip_pool_from_igw( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_POOL_ATTACHMENT_NAME, + true, + ) + .await; + let igw_addrs = + list_internet_gateway_ip_pools(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!(igw_addrs.len(), 0, "should now have zero attached ip pools"); + + // detach an ip address + detach_ip_address_from_igw( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_ADDRESS_ATTACHMENT_NAME, + false, + ) + .await; + let igw_addrs = + list_internet_gateway_ip_addresses(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!( + igw_addrs.len(), + 0, + "should now have zero attached ip addresses" + ); + + // delete internet gateay + delete_internet_gateway(c, PROJECT_NAME, VPC_NAME, IGW_NAME, false).await; + let igws = list_internet_gateways(c, PROJECT_NAME, VPC_NAME).await; + assert_eq!(igws.len(), 1, "should now just have default gateway"); + + // looking for gateway should return 404 + expect_igw_not_found(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + + // looking for gateway pools should return 404 + expect_igw_pools_not_found(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + + // looking for gateway addresses should return 404 + expect_igw_addresses_not_found(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; +} + +#[nexus_test] +async fn test_internet_gateway_address_detach(ctx: &ControlPlaneTestContext) { + let c = &ctx.external_client; + test_setup(c).await; + + create_internet_gateway(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + attach_ip_address_to_igw( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_ADDRESS_ATTACHMENT_FROM_POOL.parse().unwrap(), + IP_ADDRESS_ATTACHMENT_NAME, + ) + .await; + + // ensure we cannot detach the igw address without cascading + expect_igw_ip_address_detach_fail( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_ADDRESS_ATTACHMENT_NAME, + false, + ) + .await; + + // ensure that we can detach the igw address with cascading + detach_ip_address_from_igw( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_ADDRESS_ATTACHMENT_NAME, + true, + ) + .await; + + // should be no addresses attached to the igw + let igw_addrs = + list_internet_gateway_ip_addresses(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!( + igw_addrs.len(), + 0, + "should now have zero attached ip addresses" + ); +} + +#[nexus_test] +async fn test_internet_gateway_delete_cascade(ctx: &ControlPlaneTestContext) { + let c = &ctx.external_client; + test_setup(c).await; + + create_internet_gateway(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + attach_ip_address_to_igw( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_ADDRESS_ATTACHMENT_FROM_POOL.parse().unwrap(), + IP_ADDRESS_ATTACHMENT_NAME, + ) + .await; + attach_ip_pool_to_igw( + c, + PROJECT_NAME, + VPC_NAME, + IGW_NAME, + IP_POOL_NAME, + IP_POOL_ATTACHMENT_NAME, + ) + .await; + + delete_internet_gateway(c, PROJECT_NAME, VPC_NAME, IGW_NAME, true).await; + + // looking for gateway should return 404 + expect_igw_not_found(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + // looking for gateway pools should return 404 + expect_igw_pools_not_found(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + // looking for gateway addresses should return 404 + expect_igw_addresses_not_found(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; +} + +#[nexus_test] +async fn test_igw_ip_pool_attach_silo_user(ctx: &ControlPlaneTestContext) { + let c = &ctx.external_client; + test_setup(c).await; + + // Create a non-admin user + let silo_url = format!("/v1/system/silos/{}", DEFAULT_SILO.name()); + let silo: nexus_types::external_api::views::Silo = + nexus_test_utils::resource_helpers::object_get(c, &silo_url).await; + + let user = create_local_user( + c, + &silo, + &"user".parse().unwrap(), + params::UserPassword::LoginDisallowed, + ) + .await; + + // Grant the user Collaborator role + nexus_test_utils::resource_helpers::grant_iam( + c, + &silo_url, + SiloRole::Collaborator, + user.id, + AuthnMode::PrivilegedUser, + ) + .await; + + // Create an internet gateway + create_internet_gateway(c, PROJECT_NAME, VPC_NAME, IGW_NAME).await; + + // Verify the IP pool is not attached + let igw_pools = + list_internet_gateway_ip_pools(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!(igw_pools.len(), 0, "IP pool should not be attached"); + + // Attach an IP pool as non-admin user + let url = format!( + "/v1/internet-gateway-ip-pools?project={}&vpc={}&gateway={}", + PROJECT_NAME, VPC_NAME, IGW_NAME + ); + let params = params::InternetGatewayIpPoolCreate { + identity: IdentityMetadataCreateParams { + name: IP_POOL_ATTACHMENT_NAME.parse().unwrap(), + description: "Test attachment".to_string(), + }, + ip_pool: NameOrId::Name(IP_POOL_NAME.parse().unwrap()), + }; + + let _result: InternetGatewayIpPool = + NexusRequest::objects_post(&c, &url, ¶ms) + .authn_as(AuthnMode::SiloUser(user.id)) + .execute_and_parse_unwrap() + .await; + + // Verify the non-admin user can list the attached IP pool + let igw_pools: ResultsPage = + NexusRequest::object_get(c, &url) + .authn_as(AuthnMode::SiloUser(user.id)) + .execute_and_parse_unwrap() + .await; + + assert_eq!(igw_pools.items.len(), 1); + assert_eq!(igw_pools.items[0].identity.name, IP_POOL_ATTACHMENT_NAME); + + // detach doesn't have the authz complication that attach has, but test it anyway + let url = format!( + "/v1/internet-gateway-ip-pools/{}?project={}&vpc={}&gateway={}&cascade=true", + IP_POOL_ATTACHMENT_NAME, PROJECT_NAME, VPC_NAME, IGW_NAME, + ); + NexusRequest::object_delete(&c, &url) + .authn_as(AuthnMode::SiloUser(user.id)) + .execute() + .await + .unwrap(); + + // it's gone + let igw_pools = + list_internet_gateway_ip_pools(c, PROJECT_NAME, VPC_NAME, IGW_NAME) + .await; + assert_eq!(igw_pools.len(), 0, "IP pool should not be attached"); +} + +async fn test_setup(c: &ClientTestContext) { + // create a project and vpc to test with + let _proj = create_project(&c, PROJECT_NAME).await; + let _vpc = create_vpc(&c, PROJECT_NAME, VPC_NAME).await; + let _pool = create_ip_pool( + c, + IP_POOL_NAME, + Some(IpRange::V4(Ipv4Range { + first: "203.0.113.1".parse().unwrap(), + last: "203.0.113.254".parse().unwrap(), + })), + ) + .await; + link_ip_pool(&c, IP_POOL_NAME, &DEFAULT_SILO.id(), true).await; + let _floater = create_floating_ip( + c, + FLOATING_IP_NAME, + PROJECT_NAME, + None, + Some(IP_POOL_NAME), + ) + .await; + let nic_attach = InstanceNetworkInterfaceAttachment::Create(vec![ + InstanceNetworkInterfaceCreate { + identity: IdentityMetadataCreateParams { + description: String::from("description"), + name: "noname".parse().unwrap(), + }, + ip: None, + subnet_name: "default".parse().unwrap(), + vpc_name: VPC_NAME.parse().unwrap(), + }, + ]); + let _inst = create_instance_with( + c, + PROJECT_NAME, + INSTANCE_NAME, + &nic_attach, + Vec::new(), + vec![ExternalIpCreate::Floating { + floating_ip: NameOrId::Name(FLOATING_IP_NAME.parse().unwrap()), + }], + true, + None, + ) + .await; + + let _router = create_router(c, PROJECT_NAME, VPC_NAME, ROUTER_NAME).await; + let route = create_route( + c, + PROJECT_NAME, + VPC_NAME, + ROUTER_NAME, + ROUTE_NAME, + RouteDestination::IpNet("0.0.0.0/0".parse().unwrap()), + RouteTarget::InternetGateway(IGW_NAME.parse().unwrap()), + ) + .await; + println!("{}", route.target); +} + +async fn list_internet_gateways( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, +) -> Vec { + let url = format!( + "/v1/internet-gateways?project={}&vpc={}", + project_name, vpc_name + ); + let out = objects_list_page_authz::(client, &url).await; + out.items +} + +async fn list_internet_gateway_ip_pools( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, +) -> Vec { + let url = format!( + "/v1/internet-gateway-ip-pools?project={}&vpc={}&gateway={}", + project_name, vpc_name, igw_name, + ); + let out = + objects_list_page_authz::(client, &url).await; + out.items +} + +async fn list_internet_gateway_ip_addresses( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, +) -> Vec { + let url = format!( + "/v1/internet-gateway-ip-addresses?project={}&vpc={}&gateway={}", + project_name, vpc_name, igw_name, + ); + let out = + objects_list_page_authz::(client, &url).await; + out.items +} + +async fn expect_igw_not_found( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, +) { + // check 404 response + let url = format!( + "/v1/internet-gateways/{}?project={}&vpc={}", + igw_name, project_name, vpc_name + ); + let error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( + client, + StatusCode::NOT_FOUND, + Method::GET, + &url, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); + assert_eq!( + error.message, + format!("not found: internet-gateway with name \"{IGW_NAME}\"") + ); +} + +async fn get_igw( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, +) -> InternetGateway { + // check 404 response + let url = format!( + "/v1/internet-gateways/{}?project={}&vpc={}", + igw_name, project_name, vpc_name + ); + NexusRequest::object_get(client, &url) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap() +} + +async fn expect_igw_delete_fail( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, + cascade: bool, +) { + let url = format!( + "/v1/internet-gateways/{}?project={}&vpc={}&cascade={}", + igw_name, project_name, vpc_name, cascade + ); + let _error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( + client, + StatusCode::BAD_REQUEST, + Method::DELETE, + &url, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); +} + +async fn expect_igw_ip_pool_detach_fail( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, + ip_pool_attachment_name: &str, + cascade: bool, +) { + let url = format!( + "/v1/internet-gateway-ip-pools/{}?project={}&vpc={}&gateway={}&cascade={}", + ip_pool_attachment_name, project_name, vpc_name, igw_name, cascade + ); + let _error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( + client, + StatusCode::BAD_REQUEST, + Method::DELETE, + &url, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); +} + +async fn expect_igw_ip_address_detach_fail( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, + ip_address_attachment_name: &str, + cascade: bool, +) { + let url = format!( + "/v1/internet-gateway-ip-addresses/{}?project={}&vpc={}&gateway={}&cascade={}", + ip_address_attachment_name, project_name, vpc_name, igw_name, cascade + ); + let _error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( + client, + StatusCode::BAD_REQUEST, + Method::DELETE, + &url, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); +} + +async fn expect_igw_pools_not_found( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, +) { + let url = format!( + "/v1/internet-gateway-ip-pools?project={}&vpc={}&gateway={}", + project_name, vpc_name, igw_name, + ); + let _error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( + client, + StatusCode::NOT_FOUND, + Method::GET, + &url, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); +} + +async fn expect_igw_addresses_not_found( + client: &ClientTestContext, + project_name: &str, + vpc_name: &str, + igw_name: &str, +) { + let url = format!( + "/v1/internet-gateway-ip-addresses?project={}&vpc={}&gateway={}", + project_name, vpc_name, igw_name, + ); + let _error: dropshot::HttpErrorResponseBody = NexusRequest::expect_failure( + client, + StatusCode::NOT_FOUND, + Method::GET, + &url, + ) + .authn_as(AuthnMode::PrivilegedUser) + .execute() + .await + .unwrap() + .parsed_body() + .unwrap(); +} diff --git a/nexus/tests/integration_tests/ip_pools.rs b/nexus/tests/integration_tests/ip_pools.rs index f56755d85c..bb2fba8be6 100644 --- a/nexus/tests/integration_tests/ip_pools.rs +++ b/nexus/tests/integration_tests/ip_pools.rs @@ -16,7 +16,6 @@ use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; use nexus_db_queries::db::datastore::SERVICE_IP_POOL_NAME; use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; -use nexus_db_queries::db::fixed_data::silo::INTERNAL_SILO_ID; use nexus_test_utils::http_testing::AuthnMode; use nexus_test_utils::http_testing::NexusRequest; use nexus_test_utils::http_testing::RequestBuilder; @@ -53,6 +52,7 @@ use nexus_types::external_api::views::IpPoolSiloLink; use nexus_types::external_api::views::Silo; use nexus_types::external_api::views::SiloIpPool; use nexus_types::identity::Resource; +use nexus_types::silo::INTERNAL_SILO_ID; use omicron_common::address::Ipv6Range; use omicron_common::api::external::IdentityMetadataUpdateParams; use omicron_common::api::external::InstanceState; @@ -431,11 +431,11 @@ async fn test_ip_pool_service_no_cud(cptestctx: &ControlPlaneTestContext) { assert_eq!(error.message, not_found_id); // unlink not allowed by name or ID - let url = format!("{}/silos/{}", internal_pool_id_url, *INTERNAL_SILO_ID); + let url = format!("{}/silos/{}", internal_pool_id_url, INTERNAL_SILO_ID); let error = object_delete_error(client, &url, StatusCode::NOT_FOUND).await; assert_eq!(error.message, not_found_id); - let url = format!("{}/silos/{}", internal_pool_name_url, *INTERNAL_SILO_ID); + let url = format!("{}/silos/{}", internal_pool_name_url, INTERNAL_SILO_ID); let error = object_delete_error(client, &url, StatusCode::NOT_FOUND).await; assert_eq!(error.message, not_found_name); } diff --git a/nexus/tests/integration_tests/metrics.rs b/nexus/tests/integration_tests/metrics.rs index 700568fe96..c6c014db69 100644 --- a/nexus/tests/integration_tests/metrics.rs +++ b/nexus/tests/integration_tests/metrics.rs @@ -11,7 +11,6 @@ use chrono::Utc; use dropshot::test_util::ClientTestContext; use dropshot::ResultsPage; use http::{Method, StatusCode}; -use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID; use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; use nexus_test_utils::resource_helpers::{ create_default_ip_pool, create_disk, create_instance, create_project, @@ -20,6 +19,7 @@ use nexus_test_utils::resource_helpers::{ use nexus_test_utils::ControlPlaneTestContext; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::views::OxqlQueryResult; +use nexus_types::silo::DEFAULT_SILO_ID; use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError}; use omicron_uuid_kinds::{GenericUuid, InstanceUuid}; use oximeter::types::Datum; @@ -190,7 +190,7 @@ async fn test_metrics( // silo metrics start out zero assert_system_metrics(&cptestctx, None, 0, 0, 0).await; - assert_system_metrics(&cptestctx, Some(*DEFAULT_SILO_ID), 0, 0, 0).await; + assert_system_metrics(&cptestctx, Some(DEFAULT_SILO_ID), 0, 0, 0).await; assert_silo_metrics(&cptestctx, None, 0, 0, 0).await; let project1_id = create_project(&client, "p-1").await.identity.id; @@ -206,7 +206,7 @@ async fn test_metrics( "/v1/metrics/cpus_provisioned?start_time={:?}&end_time={:?}&order=descending&limit=1&project={}", cptestctx.start_time, Utc::now(), - *DEFAULT_SILO_ID, + DEFAULT_SILO_ID, ); assert_404(&cptestctx, &bad_silo_metrics_url).await; let bad_system_metrics_url = format!( @@ -222,15 +222,14 @@ async fn test_metrics( assert_silo_metrics(&cptestctx, Some(project1_id), 0, 4, GIB).await; assert_silo_metrics(&cptestctx, None, 0, 4, GIB).await; assert_system_metrics(&cptestctx, None, 0, 4, GIB).await; - assert_system_metrics(&cptestctx, Some(*DEFAULT_SILO_ID), 0, 4, GIB).await; + assert_system_metrics(&cptestctx, Some(DEFAULT_SILO_ID), 0, 4, GIB).await; // create disk in project 1 create_disk(&client, "p-1", "d-1").await; assert_silo_metrics(&cptestctx, Some(project1_id), GIB, 4, GIB).await; assert_silo_metrics(&cptestctx, None, GIB, 4, GIB).await; assert_system_metrics(&cptestctx, None, GIB, 4, GIB).await; - assert_system_metrics(&cptestctx, Some(*DEFAULT_SILO_ID), GIB, 4, GIB) - .await; + assert_system_metrics(&cptestctx, Some(DEFAULT_SILO_ID), GIB, 4, GIB).await; // project 2 metrics still empty assert_silo_metrics(&cptestctx, Some(project2_id), 0, 0, 0).await; @@ -245,7 +244,7 @@ async fn test_metrics( assert_system_metrics(&cptestctx, None, 2 * GIB, 8, 2 * GIB).await; assert_system_metrics( &cptestctx, - Some(*DEFAULT_SILO_ID), + Some(DEFAULT_SILO_ID), 2 * GIB, 8, 2 * GIB, diff --git a/nexus/tests/integration_tests/mod.rs b/nexus/tests/integration_tests/mod.rs index fdf14dbd07..dfcea79607 100644 --- a/nexus/tests/integration_tests/mod.rs +++ b/nexus/tests/integration_tests/mod.rs @@ -11,6 +11,7 @@ mod basic; mod certificates; mod commands; mod console_api; +mod crucible_replacements; mod demo_saga; mod device_auth; mod disks; @@ -19,6 +20,7 @@ mod host_phase1_updater; mod images; mod initialization; mod instances; +mod internet_gateway; mod ip_pools; mod metrics; mod oximeter; diff --git a/nexus/tests/integration_tests/router_routes.rs b/nexus/tests/integration_tests/router_routes.rs index 61ebaad617..b7ef395636 100644 --- a/nexus/tests/integration_tests/router_routes.rs +++ b/nexus/tests/integration_tests/router_routes.rs @@ -488,13 +488,6 @@ async fn test_router_routes_disallow_custom_targets( "subnet:a-sub-name-unknown".parse().unwrap(), "subnets cannot be used as a target in Custom routers", ), - // Internet gateways are not fully supported: only 'inetgw:outbound' - // is a valid choice. - ( - valid_dest.clone(), - "inetgw:not-a-real-gw".parse().unwrap(), - "'outbound' is currently the only valid internet gateway", - ), ]; _ = create_route( client, diff --git a/nexus/tests/integration_tests/silo_users.rs b/nexus/tests/integration_tests/silo_users.rs index 598d2a28a4..ca3039f19b 100644 --- a/nexus/tests/integration_tests/silo_users.rs +++ b/nexus/tests/integration_tests/silo_users.rs @@ -7,13 +7,13 @@ use http::{method::Method, StatusCode}; use nexus_db_queries::authn::USER_TEST_UNPRIVILEGED; use nexus_db_queries::authz; use nexus_db_queries::context::OpContext; -use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO_ID; use nexus_test_utils::assert_same_items; use nexus_test_utils::http_testing::{AuthnMode, NexusRequest}; use nexus_test_utils::resource_helpers::objects_list_page_authz; use nexus_test_utils_macros::nexus_test; use nexus_types::external_api::views; use nexus_types::identity::Asset; +use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::api::external::LookupType; use uuid::Uuid; @@ -46,8 +46,8 @@ async fn test_silo_group_users(cptestctx: &ControlPlaneTestContext) { let authz_silo = authz::Silo::new( authz::FLEET, - *DEFAULT_SILO_ID, - LookupType::ById(*DEFAULT_SILO_ID), + DEFAULT_SILO_ID, + LookupType::ById(DEFAULT_SILO_ID), ); // create a group diff --git a/nexus/tests/integration_tests/silos.rs b/nexus/tests/integration_tests/silos.rs index 0de4d31395..111a5eb122 100644 --- a/nexus/tests/integration_tests/silos.rs +++ b/nexus/tests/integration_tests/silos.rs @@ -9,7 +9,7 @@ use nexus_db_queries::authn::{USER_TEST_PRIVILEGED, USER_TEST_UNPRIVILEGED}; use nexus_db_queries::authz::{self}; use nexus_db_queries::context::OpContext; use nexus_db_queries::db; -use nexus_db_queries::db::fixed_data::silo::{DEFAULT_SILO, DEFAULT_SILO_ID}; +use nexus_db_queries::db::fixed_data::silo::DEFAULT_SILO; use nexus_db_queries::db::identity::Asset; use nexus_db_queries::db::lookup::LookupPath; use nexus_test_utils::http_testing::{AuthnMode, NexusRequest, RequestBuilder}; @@ -24,6 +24,7 @@ use nexus_types::external_api::views::{ self, IdentityProvider, Project, SamlIdentityProvider, Silo, }; use nexus_types::external_api::{params, shared}; +use nexus_types::silo::DEFAULT_SILO_ID; use omicron_common::address::{IpRange, Ipv4Range}; use omicron_common::api::external::{ IdentityMetadataCreateParams, LookupType, Name, @@ -919,12 +920,12 @@ async fn test_silo_users_list(cptestctx: &ControlPlaneTestContext) { views::User { id: USER_TEST_PRIVILEGED.id(), display_name: USER_TEST_PRIVILEGED.external_id.clone(), - silo_id: *DEFAULT_SILO_ID, + silo_id: DEFAULT_SILO_ID, }, views::User { id: USER_TEST_UNPRIVILEGED.id(), display_name: USER_TEST_UNPRIVILEGED.external_id.clone(), - silo_id: *DEFAULT_SILO_ID, + silo_id: DEFAULT_SILO_ID, }, ] ); @@ -953,17 +954,17 @@ async fn test_silo_users_list(cptestctx: &ControlPlaneTestContext) { views::User { id: new_silo_user_id, display_name: new_silo_user_external_id.into(), - silo_id: *DEFAULT_SILO_ID, + silo_id: DEFAULT_SILO_ID, }, views::User { id: USER_TEST_PRIVILEGED.id(), display_name: USER_TEST_PRIVILEGED.external_id.clone(), - silo_id: *DEFAULT_SILO_ID, + silo_id: DEFAULT_SILO_ID, }, views::User { id: USER_TEST_UNPRIVILEGED.id(), display_name: USER_TEST_UNPRIVILEGED.external_id.clone(), - silo_id: *DEFAULT_SILO_ID, + silo_id: DEFAULT_SILO_ID, }, ] ); diff --git a/nexus/tests/integration_tests/switch_port.rs b/nexus/tests/integration_tests/switch_port.rs index 92c44eddad..6500d7249a 100644 --- a/nexus/tests/integration_tests/switch_port.rs +++ b/nexus/tests/integration_tests/switch_port.rs @@ -148,7 +148,7 @@ async fn test_port_settings_basic_crud(ctx: &ControlPlaneTestContext) { dst: "1.2.3.0/24".parse().unwrap(), gw: "1.2.3.4".parse().unwrap(), vid: None, - local_pref: None, + rib_priority: None, }], }, ); diff --git a/nexus/tests/integration_tests/vpc_routers.rs b/nexus/tests/integration_tests/vpc_routers.rs index 9fff9f6c4b..d4d83c0ae6 100644 --- a/nexus/tests/integration_tests/vpc_routers.rs +++ b/nexus/tests/integration_tests/vpc_routers.rs @@ -606,11 +606,11 @@ async fn test_vpc_routers_custom_delivered_to_instance( assert!(last_routes[0].1.contains(&ResolvedVpcRoute { dest: "240.0.0.0/8".parse().unwrap(), - target: RouterTarget::Drop + target: RouterTarget::Drop, })); assert!(last_routes[1].1.contains(&ResolvedVpcRoute { dest: "241.0.0.0/8".parse().unwrap(), - target: RouterTarget::Drop + target: RouterTarget::Drop, })); // Adding a new route should propagate that out to sleds. @@ -637,7 +637,7 @@ async fn test_vpc_routers_custom_delivered_to_instance( assert_eq!(last_routes[0].0, new_system); assert!(new_custom.contains(&ResolvedVpcRoute { dest: "2.0.7.0/24".parse().unwrap(), - target: RouterTarget::Ip(instance_nics[INSTANCE_NAMES[1]][0].ip) + target: RouterTarget::Ip(instance_nics[INSTANCE_NAMES[1]][0].ip), })); // Swapping router should change the installed routes at that sled. diff --git a/nexus/types/Cargo.toml b/nexus/types/Cargo.toml index fdb53aaaae..5f21652feb 100644 --- a/nexus/types/Cargo.toml +++ b/nexus/types/Cargo.toml @@ -42,8 +42,8 @@ update-engine.workspace = true uuid.workspace = true api_identity.workspace = true -dns-service-client.workspace = true gateway-client.workspace = true +internal-dns-types.workspace = true nexus-sled-agent-shared.workspace = true omicron-common.workspace = true omicron-passwords.workspace = true diff --git a/nexus/types/src/deployment.rs b/nexus/types/src/deployment.rs index 55be9401a3..eabe4e5a3b 100644 --- a/nexus/types/src/deployment.rs +++ b/nexus/types/src/deployment.rs @@ -59,6 +59,7 @@ pub use network_resources::OmicronZoneExternalSnatIp; pub use network_resources::OmicronZoneNetworkResources; pub use network_resources::OmicronZoneNic; pub use network_resources::OmicronZoneNicEntry; +pub use planning_input::ClickhousePolicy; pub use planning_input::CockroachDbClusterVersion; pub use planning_input::CockroachDbPreserveDowngrade; pub use planning_input::CockroachDbSettings; @@ -166,6 +167,10 @@ pub struct Blueprint { /// Whether to set `cluster.preserve_downgrade_option` and what to set it to pub cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade, + /// Allocation of Clickhouse Servers and Keepers for replicated clickhouse + /// setups. This is set to `None` if replicated clickhouse is not in use. + pub clickhouse_cluster_config: Option, + /// when this blueprint was generated (for debugging) pub time_created: chrono::DateTime, /// identity of the component that generated the blueprint (for debugging) @@ -286,11 +291,9 @@ impl Blueprint { .collect(); let before_zones = before - .omicron_zones + .sled_agents .iter() - .map(|(sled_id, zones_found)| { - (*sled_id, zones_found.zones.clone().into()) - }) + .map(|(sled_id, sa)| (*sled_id, sa.omicron_zones.clone().into())) .collect(); let before_disks = before diff --git a/nexus/types/src/deployment/execution/dns.rs b/nexus/types/src/deployment/execution/dns.rs new file mode 100644 index 0000000000..a813452ccd --- /dev/null +++ b/nexus/types/src/deployment/execution/dns.rs @@ -0,0 +1,172 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::{ + collections::{BTreeMap, HashMap}, + net::IpAddr, +}; + +use internal_dns_types::{config::DnsConfigBuilder, names::ServiceName}; +use omicron_common::api::external::Name; +use omicron_uuid_kinds::SledUuid; + +use crate::{ + deployment::{ + blueprint_zone_type, Blueprint, BlueprintZoneFilter, BlueprintZoneType, + }, + internal_api::params::{DnsConfigZone, DnsRecord}, + silo::{default_silo_name, silo_dns_name}, +}; + +use super::{blueprint_nexus_external_ips, Overridables, Sled}; + +/// Returns the expected contents of internal DNS based on the given blueprint +pub fn blueprint_internal_dns_config( + blueprint: &Blueprint, + sleds_by_id: &BTreeMap, + overrides: &Overridables, +) -> anyhow::Result { + // The DNS names configured here should match what RSS configures for the + // same zones. It's tricky to have RSS share the same code because it uses + // Sled Agent's _internal_ `OmicronZoneConfig` (and friends), whereas we're + // using `sled-agent-client`'s version of that type. However, the + // DnsConfigBuilder's interface is high-level enough that it handles most of + // the details. + let mut dns_builder = DnsConfigBuilder::new(); + + 'all_zones: for (_, zone) in + blueprint.all_omicron_zones(BlueprintZoneFilter::ShouldBeInInternalDns) + { + let (service_name, port) = match &zone.zone_type { + BlueprintZoneType::BoundaryNtp( + blueprint_zone_type::BoundaryNtp { address, .. }, + ) => (ServiceName::BoundaryNtp, address.port()), + BlueprintZoneType::InternalNtp( + blueprint_zone_type::InternalNtp { address, .. }, + ) => (ServiceName::InternalNtp, address.port()), + BlueprintZoneType::Clickhouse( + blueprint_zone_type::Clickhouse { address, .. }, + ) + | BlueprintZoneType::ClickhouseServer( + blueprint_zone_type::ClickhouseServer { address, .. }, + ) => { + // Add the HTTP and native TCP interfaces for ClickHouse data + // replicas. This adds the zone itself, so we need to continue + // back up to the loop over all the Omicron zones, rather than + // falling through to call `host_zone_with_one_backend()`. + let http_service = if matches!( + &zone.zone_type, + BlueprintZoneType::Clickhouse(_) + ) { + ServiceName::Clickhouse + } else { + ServiceName::ClickhouseServer + }; + dns_builder.host_zone_clickhouse( + zone.id, + zone.underlay_address, + http_service, + address.port(), + )?; + continue 'all_zones; + } + BlueprintZoneType::ClickhouseKeeper( + blueprint_zone_type::ClickhouseKeeper { address, .. }, + ) => { + // Add the Clickhouse keeper service and `clickhouse-admin` + // service used for managing the keeper. We continue below so we + // don't fall through and call `host_zone_with_one_backend`. + dns_builder.host_zone_clickhouse_keeper( + zone.id, + zone.underlay_address, + ServiceName::ClickhouseKeeper, + address.port(), + )?; + continue 'all_zones; + } + BlueprintZoneType::CockroachDb( + blueprint_zone_type::CockroachDb { address, .. }, + ) => (ServiceName::Cockroach, address.port()), + BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { + internal_address, + .. + }) => (ServiceName::Nexus, internal_address.port()), + BlueprintZoneType::Crucible(blueprint_zone_type::Crucible { + address, + .. + }) => (ServiceName::Crucible(zone.id), address.port()), + BlueprintZoneType::CruciblePantry( + blueprint_zone_type::CruciblePantry { address }, + ) => (ServiceName::CruciblePantry, address.port()), + BlueprintZoneType::Oximeter(blueprint_zone_type::Oximeter { + address, + }) => (ServiceName::Oximeter, address.port()), + BlueprintZoneType::ExternalDns( + blueprint_zone_type::ExternalDns { http_address, .. }, + ) => (ServiceName::ExternalDns, http_address.port()), + BlueprintZoneType::InternalDns( + blueprint_zone_type::InternalDns { http_address, .. }, + ) => (ServiceName::InternalDns, http_address.port()), + }; + dns_builder.host_zone_with_one_backend( + zone.id, + zone.underlay_address, + service_name, + port, + )?; + } + + let scrimlets = sleds_by_id.values().filter(|sled| sled.is_scrimlet()); + for scrimlet in scrimlets { + let sled_subnet = scrimlet.subnet(); + let switch_zone_ip = + overrides.switch_zone_ip(scrimlet.id(), sled_subnet); + dns_builder.host_zone_switch( + scrimlet.id(), + switch_zone_ip, + overrides.dendrite_port(scrimlet.id()), + overrides.mgs_port(scrimlet.id()), + overrides.mgd_port(scrimlet.id()), + )?; + } + + Ok(dns_builder.build_zone()) +} + +pub fn blueprint_external_dns_config( + blueprint: &Blueprint, + silos: &[Name], + external_dns_zone_name: String, +) -> DnsConfigZone { + let nexus_external_ips = blueprint_nexus_external_ips(blueprint); + + let dns_records: Vec = nexus_external_ips + .into_iter() + .map(|addr| match addr { + IpAddr::V4(addr) => DnsRecord::A(addr), + IpAddr::V6(addr) => DnsRecord::Aaaa(addr), + }) + .collect(); + + let records = silos + .into_iter() + // We do not generate a DNS name for the "default" Silo. + // + // We use the name here rather than the id. It shouldn't really matter + // since every system will have this silo and so no other Silo could + // have this name. But callers (particularly the test suite and + // reconfigurator-cli) specify silos by name, not id, so if we used the + // id here then they'd have to apply this filter themselves (and this + // abstraction, such as it is, would be leakier). + .filter_map(|silo_name| { + (silo_name != default_silo_name()) + .then(|| (silo_dns_name(&silo_name), dns_records.clone())) + }) + .collect::>>(); + + DnsConfigZone { + zone_name: external_dns_zone_name, + records: records.clone(), + } +} diff --git a/nexus/types/src/deployment/execution/mod.rs b/nexus/types/src/deployment/execution/mod.rs new file mode 100644 index 0000000000..e217c30d00 --- /dev/null +++ b/nexus/types/src/deployment/execution/mod.rs @@ -0,0 +1,13 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +mod dns; +mod overridables; +mod spec; +mod utils; + +pub use dns::*; +pub use overridables::*; +pub use spec::*; +pub use utils::*; diff --git a/nexus/reconfigurator/execution/src/overridables.rs b/nexus/types/src/deployment/execution/overridables.rs similarity index 59% rename from nexus/reconfigurator/execution/src/overridables.rs rename to nexus/types/src/deployment/execution/overridables.rs index f59e3228f4..bc7b28a63a 100644 --- a/nexus/reconfigurator/execution/src/overridables.rs +++ b/nexus/types/src/deployment/execution/overridables.rs @@ -16,9 +16,9 @@ use std::net::Ipv6Addr; /// /// Blueprint execution assumes certain values about production systems that /// differ in the simulated testing environment and cannot be easily derived -/// from anything else in the environment. To accommodate this, this structure -/// provides access to these values. Everywhere except the test suite, this -/// structure is empty and returns the default (production) values. The test +/// from anything else in the environment. To accommodate this, this structure +/// provides access to these values. Everywhere except the test suite, this +/// structure is empty and returns the default (production) values. The test /// suite overrides these values. #[derive(Debug, Default)] pub struct Overridables { @@ -34,8 +34,7 @@ pub struct Overridables { impl Overridables { /// Specify the TCP port on which this sled's Dendrite is listening - #[cfg(test)] - fn override_dendrite_port(&mut self, sled_id: SledUuid, port: u16) { + pub fn override_dendrite_port(&mut self, sled_id: SledUuid, port: u16) { self.dendrite_ports.insert(sled_id, port); } @@ -45,8 +44,7 @@ impl Overridables { } /// Specify the TCP port on which this sled's MGS is listening - #[cfg(test)] - fn override_mgs_port(&mut self, sled_id: SledUuid, port: u16) { + pub fn override_mgs_port(&mut self, sled_id: SledUuid, port: u16) { self.mgs_ports.insert(sled_id, port); } @@ -56,8 +54,7 @@ impl Overridables { } /// Specify the TCP port on which this sled's MGD is listening - #[cfg(test)] - fn override_mgd_port(&mut self, sled_id: SledUuid, port: u16) { + pub fn override_mgd_port(&mut self, sled_id: SledUuid, port: u16) { self.mgd_ports.insert(sled_id, port); } @@ -67,8 +64,11 @@ impl Overridables { } /// Specify the IP address of this switch zone - #[cfg(test)] - fn override_switch_zone_ip(&mut self, sled_id: SledUuid, addr: Ipv6Addr) { + pub fn override_switch_zone_ip( + &mut self, + sled_id: SledUuid, + addr: Ipv6Addr, + ) { self.switch_zone_ips.insert(sled_id, addr); } @@ -83,39 +83,4 @@ impl Overridables { .copied() .unwrap_or_else(|| get_switch_zone_address(sled_subnet)) } - - /// Generates a set of overrides describing the simulated test environment. - #[cfg(test)] - pub fn for_test( - cptestctx: &nexus_test_utils::ControlPlaneTestContext< - omicron_nexus::Server, - >, - ) -> Overridables { - use omicron_common::api::external::SwitchLocation; - - let mut overrides = Overridables::default(); - let scrimlets = [ - (nexus_test_utils::SLED_AGENT_UUID, SwitchLocation::Switch0), - (nexus_test_utils::SLED_AGENT2_UUID, SwitchLocation::Switch1), - ]; - for (id_str, switch_location) in scrimlets { - let sled_id = id_str.parse().unwrap(); - let ip = Ipv6Addr::LOCALHOST; - let mgs_port = cptestctx - .gateway - .get(&switch_location) - .unwrap() - .client - .bind_address - .port(); - let dendrite_port = - cptestctx.dendrite.get(&switch_location).unwrap().port; - let mgd_port = cptestctx.mgd.get(&switch_location).unwrap().port; - overrides.override_switch_zone_ip(sled_id, ip); - overrides.override_dendrite_port(sled_id, dendrite_port); - overrides.override_mgs_port(sled_id, mgs_port); - overrides.override_mgd_port(sled_id, mgd_port); - } - overrides - } } diff --git a/nexus/types/src/deployment/execution.rs b/nexus/types/src/deployment/execution/spec.rs similarity index 99% rename from nexus/types/src/deployment/execution.rs rename to nexus/types/src/deployment/execution/spec.rs index 16bf73873a..4b64477bf2 100644 --- a/nexus/types/src/deployment/execution.rs +++ b/nexus/types/src/deployment/execution/spec.rs @@ -35,6 +35,7 @@ pub enum ExecutionComponent { DatasetRecords, Dns, Cockroach, + Clickhouse, } /// Steps for reconfigurator execution. diff --git a/nexus/types/src/deployment/execution/utils.rs b/nexus/types/src/deployment/execution/utils.rs new file mode 100644 index 0000000000..fe386910b4 --- /dev/null +++ b/nexus/types/src/deployment/execution/utils.rs @@ -0,0 +1,66 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use std::net::{IpAddr, SocketAddrV6}; + +use nexus_sled_agent_shared::inventory::SledRole; +use omicron_common::address::{Ipv6Subnet, SLED_PREFIX}; +use omicron_uuid_kinds::SledUuid; + +use crate::deployment::{ + blueprint_zone_type, Blueprint, BlueprintZoneFilter, BlueprintZoneType, +}; + +/// The minimal information needed to represent a sled in the context of +/// blueprint execution. +#[derive(Debug, Clone)] +pub struct Sled { + id: SledUuid, + sled_agent_address: SocketAddrV6, + role: SledRole, +} + +impl Sled { + pub fn new( + id: SledUuid, + sled_agent_address: SocketAddrV6, + role: SledRole, + ) -> Sled { + Sled { id, sled_agent_address, role } + } + + pub fn id(&self) -> SledUuid { + self.id + } + + pub fn sled_agent_address(&self) -> SocketAddrV6 { + self.sled_agent_address + } + + pub fn subnet(&self) -> Ipv6Subnet { + Ipv6Subnet::::new(*self.sled_agent_address.ip()) + } + + pub fn role(&self) -> SledRole { + self.role + } + + pub fn is_scrimlet(&self) -> bool { + self.role == SledRole::Scrimlet + } +} + +/// Return the Nexus external addresses according to the given blueprint +pub fn blueprint_nexus_external_ips(blueprint: &Blueprint) -> Vec { + blueprint + .all_omicron_zones(BlueprintZoneFilter::ShouldBeExternallyReachable) + .filter_map(|(_, z)| match z.zone_type { + BlueprintZoneType::Nexus(blueprint_zone_type::Nexus { + external_ip, + .. + }) => Some(external_ip.ip), + _ => None, + }) + .collect() +} diff --git a/nexus/types/src/deployment/network_resources.rs b/nexus/types/src/deployment/network_resources.rs index 4afec0989b..cdabbc7fdc 100644 --- a/nexus/types/src/deployment/network_resources.rs +++ b/nexus/types/src/deployment/network_resources.rs @@ -42,7 +42,7 @@ use thiserror::Error; /// /// So we use two separate maps for now. But a single map is always a /// possibility in the future, if required. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct OmicronZoneNetworkResources { /// external IPs allocated to Omicron zones omicron_zone_external_ips: TriMap, @@ -59,6 +59,11 @@ impl OmicronZoneNetworkResources { } } + pub fn is_empty(&self) -> bool { + self.omicron_zone_external_ips.is_empty() + && self.omicron_zone_nics.is_empty() + } + pub fn omicron_zone_external_ips( &self, ) -> impl Iterator + '_ { @@ -233,7 +238,7 @@ pub struct OmicronZoneExternalSnatIp { /// /// This is a slimmer `nexus_db_model::ServiceNetworkInterface` that only stores /// the fields necessary for blueprint planning. -#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct OmicronZoneNic { pub id: VnicUuid, pub mac: MacAddr, @@ -245,7 +250,7 @@ pub struct OmicronZoneNic { /// A pair of an Omicron zone ID and an external IP. /// /// Part of [`OmicronZoneNetworkResources`]. -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct OmicronZoneExternalIpEntry { pub zone_id: OmicronZoneUuid, pub ip: OmicronZoneExternalIp, @@ -276,7 +281,7 @@ impl TriMapEntry for OmicronZoneExternalIpEntry { /// A pair of an Omicron zone ID and a network interface. /// /// Part of [`OmicronZoneNetworkResources`]. -#[derive(Clone, Copy, Debug, Deserialize, Serialize)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)] pub struct OmicronZoneNicEntry { pub zone_id: OmicronZoneUuid, pub nic: OmicronZoneNic, diff --git a/nexus/types/src/deployment/planning_input.rs b/nexus/types/src/deployment/planning_input.rs index 169c134c5d..5541df60e6 100644 --- a/nexus/types/src/deployment/planning_input.rs +++ b/nexus/types/src/deployment/planning_input.rs @@ -22,6 +22,7 @@ use omicron_common::address::SLED_PREFIX; use omicron_common::api::external::Generation; use omicron_common::api::internal::shared::SourceNatConfigError; use omicron_common::disk::DiskIdentity; +use omicron_common::policy::SINGLE_NODE_CLICKHOUSE_REDUNDANCY; use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::PhysicalDiskUuid; use omicron_uuid_kinds::SledUuid; @@ -103,6 +104,10 @@ impl PlanningInput { self.policy.target_internal_dns_zone_count } + pub fn target_oximeter_zone_count(&self) -> usize { + self.policy.target_oximeter_zone_count + } + pub fn target_cockroachdb_zone_count(&self) -> usize { self.policy.target_cockroachdb_zone_count } @@ -113,10 +118,46 @@ impl PlanningInput { self.policy.target_cockroachdb_cluster_version } + pub fn target_crucible_pantry_zone_count(&self) -> usize { + self.policy.target_crucible_pantry_zone_count + } + + pub fn target_clickhouse_zone_count(&self) -> usize { + if let Some(policy) = &self.policy.clickhouse_policy { + if policy.deploy_with_standalone { + SINGLE_NODE_CLICKHOUSE_REDUNDANCY + } else { + 0 + } + } else { + SINGLE_NODE_CLICKHOUSE_REDUNDANCY + } + } + + pub fn target_clickhouse_server_zone_count(&self) -> usize { + self.policy + .clickhouse_policy + .as_ref() + .map(|policy| policy.target_servers) + .unwrap_or(0) + } + + pub fn target_clickhouse_keeper_zone_count(&self) -> usize { + self.policy + .clickhouse_policy + .as_ref() + .map(|policy| policy.target_keepers) + .unwrap_or(0) + } + pub fn service_ip_pool_ranges(&self) -> &[IpRange] { &self.policy.service_ip_pool_ranges } + pub fn clickhouse_cluster_enabled(&self) -> bool { + self.policy.clickhouse_policy.is_some() + } + pub fn all_sleds( &self, filter: SledFilter, @@ -813,9 +854,15 @@ pub struct Policy { /// internal DNS server on each of the expected reserved addresses). pub target_internal_dns_zone_count: usize, + /// desired total number of deployed Oximeter zones + pub target_oximeter_zone_count: usize, + /// desired total number of deployed CockroachDB zones pub target_cockroachdb_zone_count: usize, + /// desired total number of deployed CruciblePantry zones + pub target_crucible_pantry_zone_count: usize, + /// desired CockroachDB `cluster.preserve_downgrade_option` setting. /// at present this is hardcoded based on the version of CockroachDB we /// presently ship and the tick-tock pattern described in RFD 469. @@ -893,9 +940,11 @@ impl PlanningInputBuilder { target_boundary_ntp_zone_count: 0, target_nexus_zone_count: 0, target_internal_dns_zone_count: 0, + target_oximeter_zone_count: 0, target_cockroachdb_zone_count: 0, target_cockroachdb_cluster_version: CockroachDbClusterVersion::POLICY, + target_crucible_pantry_zone_count: 0, clickhouse_policy: None, }, internal_dns_version: Generation::new(), diff --git a/nexus/types/src/deployment/tri_map.rs b/nexus/types/src/deployment/tri_map.rs index e4ef320b4f..f5432759f1 100644 --- a/nexus/types/src/deployment/tri_map.rs +++ b/nexus/types/src/deployment/tri_map.rs @@ -22,13 +22,76 @@ use serde::{Deserialize, Serialize, Serializer}; #[derive_where(Clone, Debug, Default)] pub(crate) struct TriMap { entries: Vec, - // Invariant: the value (usize) in these maps are valid indexes into + // Invariant: the values (usize) in these maps are valid indexes into // `entries`, and are a 1:1 mapping. k1_to_entry: HashMap, k2_to_entry: HashMap, k3_to_entry: HashMap, } +impl PartialEq for TriMap { + fn eq(&self, other: &Self) -> bool { + // Implementing PartialEq for TriMap is tricky because TriMap is not + // semantically like an IndexMap: two maps are equivalent even if their + // entries are in a different order. In other words, any permutation of + // entries is equivalent. + // + // We also can't sort the entries because they're not necessarily Ord. + // + // So we write a custom equality check that checks that each key in one + // map points to the same entry as in the other map. + + if self.entries.len() != other.entries.len() { + return false; + } + + // Walk over all the entries in the first map and check that they point + // to the same entry in the second map. + for (ix, entry) in self.entries.iter().enumerate() { + let k1 = entry.key1(); + let k2 = entry.key2(); + let k3 = entry.key3(); + + // Check that the indexes are the same in the other map. + let Some(other_ix1) = other.k1_to_entry.get(&k1).copied() else { + return false; + }; + let Some(other_ix2) = other.k2_to_entry.get(&k2).copied() else { + return false; + }; + let Some(other_ix3) = other.k3_to_entry.get(&k3).copied() else { + return false; + }; + + if other_ix1 != other_ix2 || other_ix1 != other_ix3 { + // All the keys were present but they didn't point to the same + // entry. + return false; + } + + // Check that the other map's entry is the same as this map's + // entry. (This is what we use the `PartialEq` bound on T for.) + // + // Because we've checked that other_ix1, other_ix2 and other_ix3 + // are Some(ix), we know that ix is valid and points to the + // expected entry. + let other_entry = &other.entries[other_ix1]; + if entry != other_entry { + eprintln!( + "mismatch: ix: {}, entry: {:?}, other_entry: {:?}", + ix, entry, other_entry + ); + return false; + } + } + + true + } +} + +// The Eq bound on T ensures that the TriMap forms an equivalence class. +impl Eq for TriMap {} + // Note: Eq and PartialEq are not implemented for TriMap. Implementing them // would need to be done with care, because TriMap is not semantically like an // IndexMap: two maps are equivalent even if their entries are in a different @@ -92,6 +155,10 @@ impl TriMap { } } + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + pub(crate) fn iter(&self) -> impl Iterator { self.entries.iter() } @@ -255,6 +322,7 @@ impl std::error::Error for DuplicateEntry {} #[cfg(test)] mod tests { use super::*; + use prop::sample::SizeRange; use proptest::prelude::*; use test_strategy::{proptest, Arbitrary}; @@ -512,4 +580,213 @@ mod tests { } } } + + #[proptest(cases = 64)] + fn proptest_permutation_eq( + #[strategy(test_entry_permutation_strategy(0..256))] entries: ( + Vec, + Vec, + ), + ) { + let (entries1, entries2) = entries; + let mut map1 = TriMap::::new(); + let mut map2 = TriMap::::new(); + + for entry in entries1 { + map1.insert_no_dups(entry.clone()).unwrap(); + } + for entry in entries2 { + map2.insert_no_dups(entry.clone()).unwrap(); + } + + assert_eq_props(map1, map2); + } + + // Returns a pair of permutations of a set of unique entries. + fn test_entry_permutation_strategy( + size: impl Into, + ) -> impl Strategy, Vec)> { + prop::collection::vec(any::(), size.into()).prop_perturb( + |v, mut rng| { + // It is possible (likely even) that the input vector has + // duplicates. How can we remove them? The easiest way is to + // use the TriMap logic that already exists to check for + // duplicates. Insert all the entries one by one, then get the + // list. + let mut map = TriMap::::new(); + for entry in v { + // The error case here is expected -- we're actively + // de-duping entries right now. + _ = map.insert_no_dups(entry); + } + let v = map.entries; + + // Now shuffle the entries. This is a simple Fisher-Yates + // shuffle (Durstenfeld variant, low to high). + let mut v2 = v.clone(); + if v.len() < 2 { + return (v, v2); + } + for i in 0..v2.len() - 2 { + let j = rng.gen_range(i..v2.len()); + v2.swap(i, j); + } + + (v, v2) + }, + ) + } + + // Test various conditions for non-equality. + // + // It's somewhat hard to capture mutations in a proptest (partly because + // `TriMap` doesn't support mutating existing entries at the moment), so + // this is a small example-based test. + #[test] + fn test_permutation_eq_examples() { + let mut map1 = TriMap::::new(); + let mut map2 = TriMap::::new(); + + // Two empty maps are equal. + assert_eq!(map1, map2); + + // Insert a single entry into one map. + let entry = TestEntry { + key1: 0, + key2: 'a', + key3: "x".to_string(), + value: "v".to_string(), + }; + map1.insert_no_dups(entry.clone()).unwrap(); + + // The maps are not equal. + assert_ne_props(&map1, &map2); + + // Insert the same entry into the other map. + map2.insert_no_dups(entry.clone()).unwrap(); + + // The maps are now equal. + assert_eq_props(&map1, &map2); + + { + // Insert an entry with the same key2 and key3 but a different + // key1. + let mut map1 = map1.clone(); + map1.insert_no_dups(TestEntry { + key1: 1, + key2: 'b', + key3: "y".to_string(), + value: "v".to_string(), + }) + .unwrap(); + assert_ne_props(&map1, &map2); + + let mut map2 = map2.clone(); + map2.insert_no_dups(TestEntry { + key1: 2, + key2: 'b', + key3: "y".to_string(), + value: "v".to_string(), + }) + .unwrap(); + assert_ne_props(&map1, &map2); + } + + { + // Insert an entry with the same key1 and key3 but a different + // key2. + let mut map1 = map1.clone(); + map1.insert_no_dups(TestEntry { + key1: 1, + key2: 'b', + key3: "y".to_string(), + value: "v".to_string(), + }) + .unwrap(); + assert_ne_props(&map1, &map2); + + let mut map2 = map2.clone(); + map2.insert_no_dups(TestEntry { + key1: 1, + key2: 'c', + key3: "y".to_string(), + value: "v".to_string(), + }) + .unwrap(); + assert_ne_props(&map1, &map2); + } + + { + // Insert an entry with the same key1 and key2 but a different + // key3. + let mut map1 = map1.clone(); + map1.insert_no_dups(TestEntry { + key1: 1, + key2: 'b', + key3: "y".to_string(), + value: "v".to_string(), + }) + .unwrap(); + assert_ne_props(&map1, &map2); + + let mut map2 = map2.clone(); + map2.insert_no_dups(TestEntry { + key1: 1, + key2: 'b', + key3: "z".to_string(), + value: "v".to_string(), + }) + .unwrap(); + assert_ne_props(&map1, &map2); + } + + { + // Insert an entry where all the keys are the same, but the value is + // different. + let mut map1 = map1.clone(); + map1.insert_no_dups(TestEntry { + key1: 1, + key2: 'b', + key3: "y".to_string(), + value: "w".to_string(), + }) + .unwrap(); + assert_ne_props(&map1, &map2); + + let mut map2 = map2.clone(); + map2.insert_no_dups(TestEntry { + key1: 1, + key2: 'b', + key3: "y".to_string(), + value: "x".to_string(), + }) + .unwrap(); + assert_ne_props(&map1, &map2); + } + } + + /// Assert equality properties. + /// + /// The PartialEq algorithm is not obviously symmetric or reflexive, so we + /// must ensure in our tests that it is. + #[allow(clippy::eq_op)] + fn assert_eq_props(a: T, b: T) { + assert_eq!(a, a, "a == a"); + assert_eq!(b, b, "b == b"); + assert_eq!(a, b, "a == b"); + assert_eq!(b, a, "b == a"); + } + + /// Assert inequality properties. + /// + /// The PartialEq algorithm is not obviously symmetric or reflexive, so we + /// must ensure in our tests that it is. + #[allow(clippy::eq_op)] + fn assert_ne_props(a: T, b: T) { + // Also check reflexivity while we're here. + assert_eq!(a, a, "a == a"); + assert_eq!(b, b, "b == b"); + assert_ne!(a, b, "a != b"); + assert_ne!(b, a, "b != a"); + } } diff --git a/nexus/types/src/deployment/zone_type.rs b/nexus/types/src/deployment/zone_type.rs index a67fb06563..13d3d0bba2 100644 --- a/nexus/types/src/deployment/zone_type.rs +++ b/nexus/types/src/deployment/zone_type.rs @@ -100,6 +100,26 @@ impl BlueprintZoneType { matches!(self, BlueprintZoneType::Crucible(_)) } + /// Identifies whether this is a Crucible pantry zone + pub fn is_crucible_pantry(&self) -> bool { + matches!(self, BlueprintZoneType::CruciblePantry(_)) + } + + /// Identifies whether this is a clickhouse keeper zone + pub fn is_clickhouse_keeper(&self) -> bool { + matches!(self, BlueprintZoneType::ClickhouseKeeper(_)) + } + + /// Identifies whether this is a clickhouse server zone + pub fn is_clickhouse_server(&self) -> bool { + matches!(self, BlueprintZoneType::ClickhouseServer(_)) + } + + /// Identifies whether this is a single-node clickhouse zone + pub fn is_clickhouse(&self) -> bool { + matches!(self, BlueprintZoneType::Clickhouse(_)) + } + /// Returns the durable dataset associated with this zone, if any exists. pub fn durable_dataset(&self) -> Option> { let (dataset, kind, &address) = match self { diff --git a/nexus/types/src/external_api/params.rs b/nexus/types/src/external_api/params.rs index 8f0abd3d46..4daf61d4d0 100644 --- a/nexus/types/src/external_api/params.rs +++ b/nexus/types/src/external_api/params.rs @@ -76,6 +76,7 @@ path_param!(VpcPath, vpc, "VPC"); path_param!(SubnetPath, subnet, "subnet"); path_param!(RouterPath, router, "router"); path_param!(RoutePath, route, "route"); +path_param!(InternetGatewayPath, gateway, "gateway"); path_param!(FloatingIpPath, floating_ip, "floating IP"); path_param!(DiskPath, disk, "disk"); path_param!(SnapshotPath, snapshot, "snapshot"); @@ -83,6 +84,7 @@ path_param!(ImagePath, image, "image"); path_param!(SiloPath, silo, "silo"); path_param!(ProviderPath, provider, "SAML identity provider"); path_param!(IpPoolPath, pool, "IP pool"); +path_param!(IpAddressPath, address, "IP address"); path_param!(SshKeyPath, ssh_key, "SSH key"); path_param!(AddressLotPath, address_lot, "address lot"); path_param!(ProbePath, probe, "probe"); @@ -258,6 +260,17 @@ pub struct OptionalVpcSelector { pub vpc: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct InternetGatewayDeleteSelector { + /// Name or ID of the project, only required if `vpc` is provided as a `Name` + pub project: Option, + /// Name or ID of the VPC + pub vpc: Option, + /// Also delete routes targeting this gateway. + #[serde(default)] + pub cascade: bool, +} + #[derive(Deserialize, JsonSchema)] pub struct SubnetSelector { /// Name or ID of the project, only required if `vpc` is provided as a `Name` @@ -300,6 +313,65 @@ pub struct RouteSelector { pub route: NameOrId, } +// Internet gateways + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct InternetGatewaySelector { + /// Name or ID of the project, only required if `vpc` is provided as a `Name` + pub project: Option, + /// Name or ID of the VPC, only required if `gateway` is provided as a `Name` + pub vpc: Option, + /// Name or ID of the internet gateway + pub gateway: NameOrId, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct OptionalInternetGatewaySelector { + /// Name or ID of the project, only required if `vpc` is provided as a `Name` + pub project: Option, + /// Name or ID of the VPC, only required if `gateway` is provided as a `Name` + pub vpc: Option, + /// Name or ID of the internet gateway + pub gateway: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema, PartialEq)] +pub struct DeleteInternetGatewayElementSelector { + /// Name or ID of the project, only required if `vpc` is provided as a `Name` + pub project: Option, + /// Name or ID of the VPC, only required if `gateway` is provided as a `Name` + pub vpc: Option, + /// Name or ID of the internet gateway + pub gateway: Option, + /// Also delete routes targeting this gateway element. + #[serde(default)] + pub cascade: bool, +} + +#[derive(Deserialize, JsonSchema)] +pub struct InternetGatewayIpPoolSelector { + /// Name or ID of the project, only required if `vpc` is provided as a `Name` + pub project: Option, + /// Name or ID of the VPC, only required if `gateway` is provided as a `Name` + pub vpc: Option, + /// Name or ID of the gateway, only required if `pool` is provided as a `Name` + pub gateway: Option, + /// Name or ID of the pool + pub pool: NameOrId, +} + +#[derive(Deserialize, JsonSchema)] +pub struct InternetGatewayIpAddressSelector { + /// Name or ID of the project, only required if `vpc` is provided as a `Name` + pub project: Option, + /// Name or ID of the VPC, only required if `gateway` is provided as a `Name` + pub vpc: Option, + /// Name or ID of the gateway, only required if `address` is provided as a `Name` + pub gateway: Option, + /// Name or ID of the address + pub address: NameOrId, +} + // Silos /// Create-time parameters for a `Silo` @@ -998,8 +1070,11 @@ pub enum ExternalIpDetach { pub struct InstanceCreate { #[serde(flatten)] pub identity: IdentityMetadataCreateParams, + /// The number of vCPUs to be allocated to the instance pub ncpus: InstanceCpuCount, + /// The amount of RAM (in bytes) to be allocated to the instance pub memory: ByteCount, + /// The hostname to be assigned to the instance pub hostname: Hostname, /// User data for instance initialization systems (such as cloud-init). @@ -1076,6 +1151,11 @@ pub struct InstanceUpdate { /// /// If not provided, unset the instance's boot disk. pub boot_disk: Option, + + /// The auto-restart policy for this instance. + /// + /// If not provided, unset the instance's auto-restart policy. + pub auto_restart_policy: Option, } #[inline] @@ -1294,6 +1374,29 @@ pub struct RouterRouteUpdate { pub destination: RouteDestination, } +// INTERNET GATEWAYS + +/// Create-time parameters for an `InternetGateway` +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InternetGatewayCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InternetGatewayIpPoolCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + pub ip_pool: NameOrId, +} + +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InternetGatewayIpAddressCreate { + #[serde(flatten)] + pub identity: IdentityMetadataCreateParams, + pub address: IpAddr, +} + // DISKS #[derive(Display, Serialize, Deserialize, JsonSchema)] @@ -1406,12 +1509,12 @@ pub enum DiskSource { /// Create-time parameters for a `Disk` #[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] pub struct DiskCreate { - /// common identifying metadata + /// The common identifying metadata for the disk #[serde(flatten)] pub identity: IdentityMetadataCreateParams, - /// initial source for this disk + /// The initial source for this disk pub disk_source: DiskSource, - /// total size of the Disk in bytes + /// The total size of the Disk (in bytes) pub size: ByteCount, } @@ -1687,7 +1790,7 @@ pub struct Route { /// Local preference for route. Higher preference indictes precedence /// within and across protocols. - pub local_pref: Option, + pub rib_priority: Option, } /// Select a BGP config by a name or id. diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index e8d81b05bb..0fd45c0666 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -303,6 +303,45 @@ pub struct VpcRouter { pub vpc_id: Uuid, } +// INTERNET GATEWAYS + +/// An internet gateway provides a path between VPC networks and external +/// networks. +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InternetGateway { + #[serde(flatten)] + pub identity: IdentityMetadata, + + /// The VPC to which the gateway belongs. + pub vpc_id: Uuid, +} + +/// An IP pool that is attached to an internet gateway +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InternetGatewayIpPool { + #[serde(flatten)] + pub identity: IdentityMetadata, + + /// The associated internet gateway. + pub internet_gateway_id: Uuid, + + /// The associated IP pool. + pub ip_pool_id: Uuid, +} + +/// An IP address that is attached to an internet gateway +#[derive(ObjectIdentity, Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct InternetGatewayIpAddress { + #[serde(flatten)] + pub identity: IdentityMetadata, + + /// The associated internet gateway. + pub internet_gateway_id: Uuid, + + /// The associated IP address, + pub address: IpAddr, +} + // IP POOLS /// A collection of IP ranges. If a pool is linked to a silo, IP addresses from diff --git a/nexus/types/src/internal_api/background.rs b/nexus/types/src/internal_api/background.rs index c4f0628742..cf3d652587 100644 --- a/nexus/types/src/internal_api/background.rs +++ b/nexus/types/src/internal_api/background.rs @@ -12,6 +12,7 @@ use uuid::Uuid; pub struct RegionReplacementStatus { pub requests_created_ok: Vec, pub start_invoked_ok: Vec, + pub requests_completed_ok: Vec, pub errors: Vec, } diff --git a/nexus/types/src/internal_api/params.rs b/nexus/types/src/internal_api/params.rs index c803f003f1..77677687a3 100644 --- a/nexus/types/src/internal_api/params.rs +++ b/nexus/types/src/internal_api/params.rs @@ -180,7 +180,7 @@ pub struct RackInitializationRequest { /// x.509 Certificates used to encrypt communication with the external API. pub certs: Vec, /// initial internal DNS config - pub internal_dns_zone_config: dns_service_client::types::DnsConfigParams, + pub internal_dns_zone_config: internal_dns_types::config::DnsConfigParams, /// delegated DNS name for external DNS pub external_dns_zone_name: String, /// configuration for the initial (recovery) Silo @@ -193,10 +193,10 @@ pub struct RackInitializationRequest { pub allowed_source_ips: AllowedSourceIps, } -pub type DnsConfigParams = dns_service_client::types::DnsConfigParams; -pub type DnsConfigZone = dns_service_client::types::DnsConfigZone; -pub type DnsRecord = dns_service_client::types::DnsRecord; -pub type Srv = dns_service_client::types::Srv; +pub type DnsConfigParams = internal_dns_types::config::DnsConfigParams; +pub type DnsConfigZone = internal_dns_types::config::DnsConfigZone; +pub type DnsRecord = internal_dns_types::config::DnsRecord; +pub type Srv = internal_dns_types::config::Srv; /// Message used to notify Nexus that this oximeter instance is up and running. #[derive(Debug, Clone, Copy, JsonSchema, Serialize, Deserialize)] diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 563fd8a4d3..85958f518a 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -13,7 +13,7 @@ use crate::external_api::params::PhysicalDiskKind; use crate::external_api::params::UninitializedSledId; use chrono::DateTime; use chrono::Utc; -use clickhouse_admin_types::KeeperId; +use clickhouse_admin_types::ClickhouseKeeperClusterMembership; pub use gateway_client::types::PowerState; pub use gateway_client::types::RotImageError; pub use gateway_client::types::RotSlot; @@ -31,7 +31,6 @@ pub use omicron_common::api::internal::shared::SourceNatConfig; pub use omicron_common::zpool_name::ZpoolName; use omicron_uuid_kinds::CollectionUuid; use omicron_uuid_kinds::DatasetUuid; -use omicron_uuid_kinds::OmicronZoneUuid; use omicron_uuid_kinds::SledUuid; use omicron_uuid_kinds::ZpoolUuid; use serde::{Deserialize, Serialize}; @@ -117,14 +116,30 @@ pub struct Collection { /// Sled Agent information, by *sled* id pub sled_agents: BTreeMap, - /// Omicron zones found, by *sled* id - pub omicron_zones: BTreeMap, - /// The raft configuration (cluster membership) of the clickhouse keeper /// cluster as returned from each available keeper via `clickhouse-admin` in /// the `ClickhouseKeeper` zone + /// + /// Each clickhouse keeper is uniquely identified by its `KeeperId` + /// and deployed to a separate omicron zone. The uniqueness of IDs and + /// deployments is guaranteed by the reconfigurator. DNS is used to find + /// `clickhouse-admin-keeper` servers running in the same zone as keepers + /// and retrieve their local knowledge of the raft cluster. Each keeper + /// reports its own unique ID along with its membership information. We use + /// this information to decide upon the most up to date state (which will + /// eventually be reflected to other keepers), so that we can choose how to + /// reconfigure our keeper cluster if needed. + /// + /// All this data is directly reported from the `clickhouse-keeper-admin` + /// servers in this format. While we could also cache the zone ID + /// in the `ClickhouseKeeper` zones, return that along with the + /// `ClickhouseKeeperClusterMembership`, and map by zone ID here, the + /// information would be superfluous. It would be filtered out by the + /// reconfigurator planner downstream. It is not necessary for the planners + /// to use this since the blueprints already contain the zone ID/ KeeperId + /// mappings and guarantee unique pairs. pub clickhouse_keeper_cluster_membership: - BTreeMap, + BTreeSet, } impl Collection { @@ -152,7 +167,7 @@ impl Collection { pub fn all_omicron_zones( &self, ) -> impl Iterator { - self.omicron_zones.values().flat_map(|z| z.zones.zones.iter()) + self.sled_agents.values().flat_map(|sa| sa.omicron_zones.zones.iter()) } /// Iterate over the sled ids of sleds identified as Scrimlets @@ -167,11 +182,11 @@ impl Collection { /// there is one. pub fn latest_clickhouse_keeper_membership( &self, - ) -> Option<(OmicronZoneUuid, ClickhouseKeeperClusterMembership)> { + ) -> Option { self.clickhouse_keeper_cluster_membership .iter() - .max_by_key(|(_, membership)| membership.leader_committed_log_index) - .map(|(zone_id, membership)| (*zone_id, membership.clone())) + .max_by_key(|membership| membership.leader_committed_log_index) + .map(|membership| (membership.clone())) } } @@ -500,29 +515,8 @@ pub struct SledAgent { pub usable_hardware_threads: u32, pub usable_physical_ram: ByteCount, pub reservoir_size: ByteCount, + pub omicron_zones: OmicronZonesConfig, pub disks: Vec, pub zpools: Vec, pub datasets: Vec, } - -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -pub struct OmicronZonesFound { - pub time_collected: DateTime, - pub source: String, - pub sled_id: SledUuid, - pub zones: OmicronZonesConfig, -} - -/// The configuration of the clickhouse keeper raft cluster returned from a -/// single keeper node -/// -/// Each keeper is asked for its known raft configuration via `clickhouse-admin` -/// dropshot servers running in `ClickhouseKeeper` zones. state. We include the -/// leader committed log index known to the current keeper node (whether or not -/// it is the leader) to determine which configuration is newest. -#[derive(Clone, Debug, PartialEq, Eq, Deserialize, Serialize)] -pub struct ClickhouseKeeperClusterMembership { - pub queried_keeper: KeeperId, - pub leader_committed_log_index: u64, - pub raft_config: BTreeSet, -} diff --git a/nexus/types/src/lib.rs b/nexus/types/src/lib.rs index 8a0a3ec80e..4b9c8836a4 100644 --- a/nexus/types/src/lib.rs +++ b/nexus/types/src/lib.rs @@ -35,3 +35,4 @@ pub mod external_api; pub mod identity; pub mod internal_api; pub mod inventory; +pub mod silo; diff --git a/nexus/types/src/silo.rs b/nexus/types/src/silo.rs new file mode 100644 index 0000000000..1b34c95fb8 --- /dev/null +++ b/nexus/types/src/silo.rs @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Silo-related utilities and fixed data. + +use std::sync::LazyLock; + +use omicron_common::api::external::Name; + +/// The ID of the "default" silo. +pub static DEFAULT_SILO_ID: uuid::Uuid = + uuid::Uuid::from_u128(0x001de000_5110_4000_8000_000000000000); + +/// Return the name of the default silo. +pub fn default_silo_name() -> &'static Name { + static DEFAULT_SILO_NAME: LazyLock = + LazyLock::new(|| "default-silo".parse().unwrap()); + + &DEFAULT_SILO_NAME +} + +/// The ID of the built-in internal silo. +pub static INTERNAL_SILO_ID: uuid::Uuid = + uuid::Uuid::from_u128(0x001de000_5110_4000_8000_000000000001); + +/// Return the name of the internal silo. +pub fn internal_silo_name() -> &'static Name { + static INTERNAL_SILO_NAME: LazyLock = + LazyLock::new(|| "oxide-internal".parse().unwrap()); + + &INTERNAL_SILO_NAME +} + +/// Returns the (relative) DNS name for this Silo's API and console endpoints +/// _within_ the external DNS zone (i.e., without that zone's suffix) +/// +/// This specific naming scheme is determined under RFD 357. +pub fn silo_dns_name(name: &omicron_common::api::external::Name) -> String { + // RFD 4 constrains resource names (including Silo names) to DNS-safe + // strings, which is why it's safe to directly put the name of the + // resource into the DNS name rather than doing any kind of escaping. + format!("{}.sys", name) +} diff --git a/openapi/bootstrap-agent.json b/openapi/bootstrap-agent.json index 6b4d2093a1..54feaa7325 100644 --- a/openapi/bootstrap-agent.json +++ b/openapi/bootstrap-agent.json @@ -1257,19 +1257,19 @@ } ] }, - "local_pref": { - "nullable": true, - "description": "The local preference associated with this route.", - "default": null, - "type": "integer", - "format": "uint32", - "minimum": 0 - }, "nexthop": { "description": "The nexthop/gateway address.", "type": "string", "format": "ip" }, + "rib_priority": { + "nullable": true, + "description": "The RIB priority (i.e. Admin Distance) associated with this route.", + "default": null, + "type": "integer", + "format": "uint8", + "minimum": 0 + }, "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", diff --git a/openapi/clickhouse-admin.json b/openapi/clickhouse-admin-keeper.json similarity index 78% rename from openapi/clickhouse-admin.json rename to openapi/clickhouse-admin-keeper.json index 5d1ba8464d..3c48a082b7 100644 --- a/openapi/clickhouse-admin.json +++ b/openapi/clickhouse-admin-keeper.json @@ -1,8 +1,8 @@ { "openapi": "3.0.3", "info": { - "title": "ClickHouse Cluster Admin API", - "description": "API for interacting with the Oxide control plane's ClickHouse cluster", + "title": "ClickHouse Cluster Admin Keeper API", + "description": "API for interacting with the Oxide control plane's ClickHouse cluster keepers", "contact": { "url": "https://oxide.computer", "email": "api@oxide.computer" @@ -10,35 +10,11 @@ "version": "0.0.1" }, "paths": { - "/keeper/conf": { - "get": { - "summary": "Retrieve configuration information from a keeper node.", - "operationId": "keeper_conf", - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/KeeperConf" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/keeper/config": { + "/config": { "put": { "summary": "Generate a ClickHouse configuration file for a keeper node on a specified", - "description": "directory.", - "operationId": "generate_keeper_config", + "description": "directory and enable the SMF service if not currently enabled.\n\nNote that we cannot start the keeper service until there is an initial configuration set via this endpoint.", + "operationId": "generate_config", "requestBody": { "content": { "application/json": { @@ -69,18 +45,17 @@ } } }, - "/keeper/lgif": { + "/keeper/cluster-membership": { "get": { - "summary": "Retrieve a logically grouped information file from a keeper node.", - "description": "This information is used internally by ZooKeeper to manage snapshots and logs for consistency and recovery.", - "operationId": "lgif", + "summary": "Retrieve cluster membership information from a keeper node.", + "operationId": "keeper_cluster_membership", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Lgif" + "$ref": "#/components/schemas/ClickhouseKeeperClusterMembership" } } } @@ -94,18 +69,17 @@ } } }, - "/keeper/raft-config": { + "/keeper/conf": { "get": { - "summary": "Retrieve information from ClickHouse virtual node /keeper/config which", - "description": "contains last committed cluster configuration.", - "operationId": "raft_config", + "summary": "Retrieve configuration information from a keeper node.", + "operationId": "keeper_conf", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RaftConfig" + "$ref": "#/components/schemas/KeeperConf" } } } @@ -119,28 +93,43 @@ } } }, - "/server/config": { - "put": { - "summary": "Generate a ClickHouse configuration file for a server node on a specified", - "description": "directory.", - "operationId": "generate_server_config", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ServerConfigurableSettings" + "/keeper/lgif": { + "get": { + "summary": "Retrieve a logically grouped information file from a keeper node.", + "description": "This information is used internally by ZooKeeper to manage snapshots and logs for consistency and recovery.", + "operationId": "lgif", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Lgif" + } } } }, - "required": true - }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/keeper/raft-config": { + "get": { + "summary": "Retrieve information from ClickHouse virtual node /keeper/config which", + "description": "contains last committed cluster configuration.", + "operationId": "raft_config", "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ReplicaConfig" + "$ref": "#/components/schemas/RaftConfig" } } } @@ -199,6 +188,39 @@ } ] }, + "ClickhouseKeeperClusterMembership": { + "description": "The configuration of the clickhouse keeper raft cluster returned from a single keeper node\n\nEach keeper is asked for its known raft configuration via `clickhouse-admin` dropshot servers running in `ClickhouseKeeper` zones. state. We include the leader committed log index known to the current keeper node (whether or not it is the leader) to determine which configuration is newest.", + "type": "object", + "properties": { + "leader_committed_log_index": { + "description": "Index of the last committed log entry from the leader's perspective", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "queried_keeper": { + "description": "Keeper ID of the keeper being queried", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperId" + } + ] + }, + "raft_config": { + "description": "Keeper IDs of all keepers in the cluster", + "type": "array", + "items": { + "$ref": "#/components/schemas/KeeperId" + }, + "uniqueItems": true + } + }, + "required": [ + "leader_committed_log_index", + "queried_keeper", + "raft_config" + ] + }, "Error": { "description": "Error information from a response.", "type": "object", @@ -536,21 +558,8 @@ "tcp_port" ] }, - "KeeperConfigsForReplica": { - "type": "object", - "properties": { - "nodes": { - "type": "array", - "items": { - "$ref": "#/components/schemas/KeeperNodeConfig" - } - } - }, - "required": [ - "nodes" - ] - }, "KeeperConfigurableSettings": { + "description": "The top most type for configuring clickhouse-servers via clickhouse-admin-keeper-api", "type": "object", "properties": { "generation": { @@ -604,23 +613,6 @@ "format": "uint64", "minimum": 0 }, - "KeeperNodeConfig": { - "type": "object", - "properties": { - "host": { - "$ref": "#/components/schemas/ClickhouseHost" - }, - "port": { - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "host", - "port" - ] - }, "KeeperServerInfo": { "type": "object", "properties": { @@ -823,27 +815,6 @@ "debug" ] }, - "Macros": { - "type": "object", - "properties": { - "cluster": { - "type": "string" - }, - "replica": { - "$ref": "#/components/schemas/ServerId" - }, - "shard": { - "type": "integer", - "format": "uint64", - "minimum": 0 - } - }, - "required": [ - "cluster", - "replica", - "shard" - ] - }, "RaftConfig": { "description": "Keeper raft configuration information", "type": "object", @@ -909,204 +880,6 @@ "required": [ "servers" ] - }, - "RemoteServers": { - "type": "object", - "properties": { - "cluster": { - "type": "string" - }, - "replicas": { - "type": "array", - "items": { - "$ref": "#/components/schemas/ServerNodeConfig" - } - }, - "secret": { - "type": "string" - } - }, - "required": [ - "cluster", - "replicas", - "secret" - ] - }, - "ReplicaConfig": { - "description": "Configuration for a ClickHouse replica server", - "type": "object", - "properties": { - "data_path": { - "description": "Directory for all files generated by ClickHouse itself", - "type": "string", - "format": "Utf8PathBuf" - }, - "http_port": { - "description": "Port for HTTP connections", - "type": "integer", - "format": "uint16", - "minimum": 0 - }, - "interserver_http_port": { - "description": "Port for interserver HTTP connections", - "type": "integer", - "format": "uint16", - "minimum": 0 - }, - "keepers": { - "description": "Contains settings that allow ClickHouse servers to interact with a Keeper cluster", - "allOf": [ - { - "$ref": "#/components/schemas/KeeperConfigsForReplica" - } - ] - }, - "listen_host": { - "description": "Address the server is listening on", - "type": "string", - "format": "ipv6" - }, - "logger": { - "description": "Logging settings", - "allOf": [ - { - "$ref": "#/components/schemas/LogConfig" - } - ] - }, - "macros": { - "description": "Parameter substitutions for replicated tables", - "allOf": [ - { - "$ref": "#/components/schemas/Macros" - } - ] - }, - "remote_servers": { - "description": "Configuration of clusters used by the Distributed table engine and bythe cluster table function", - "allOf": [ - { - "$ref": "#/components/schemas/RemoteServers" - } - ] - }, - "tcp_port": { - "description": "Port for TCP connections", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "data_path", - "http_port", - "interserver_http_port", - "keepers", - "listen_host", - "logger", - "macros", - "remote_servers", - "tcp_port" - ] - }, - "ServerConfigurableSettings": { - "type": "object", - "properties": { - "generation": { - "description": "A unique identifier for the configuration generation.", - "allOf": [ - { - "$ref": "#/components/schemas/Generation" - } - ] - }, - "settings": { - "description": "Configurable settings for a ClickHouse replica server node.", - "allOf": [ - { - "$ref": "#/components/schemas/ServerSettings" - } - ] - } - }, - "required": [ - "generation", - "settings" - ] - }, - "ServerId": { - "description": "A unique ID for a Clickhouse Server", - "type": "integer", - "format": "uint64", - "minimum": 0 - }, - "ServerNodeConfig": { - "type": "object", - "properties": { - "host": { - "$ref": "#/components/schemas/ClickhouseHost" - }, - "port": { - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "host", - "port" - ] - }, - "ServerSettings": { - "description": "Configurable settings for a ClickHouse replica server node.", - "type": "object", - "properties": { - "config_dir": { - "description": "Directory for the generated server configuration XML file", - "type": "string", - "format": "Utf8PathBuf" - }, - "datastore_path": { - "description": "Directory for all files generated by ClickHouse itself", - "type": "string", - "format": "Utf8PathBuf" - }, - "id": { - "description": "Unique ID of the server node", - "allOf": [ - { - "$ref": "#/components/schemas/ServerId" - } - ] - }, - "keepers": { - "description": "Addresses for each of the individual nodes in the Keeper cluster", - "type": "array", - "items": { - "$ref": "#/components/schemas/ClickhouseHost" - } - }, - "listen_addr": { - "description": "Address the server is listening on", - "type": "string", - "format": "ipv6" - }, - "remote_servers": { - "description": "Addresses for each of the individual replica servers", - "type": "array", - "items": { - "$ref": "#/components/schemas/ClickhouseHost" - } - } - }, - "required": [ - "config_dir", - "datastore_path", - "id", - "keepers", - "listen_addr", - "remote_servers" - ] } }, "responses": { diff --git a/openapi/clickhouse-admin-server.json b/openapi/clickhouse-admin-server.json new file mode 100644 index 0000000000..9dc2506eb1 --- /dev/null +++ b/openapi/clickhouse-admin-server.json @@ -0,0 +1,423 @@ +{ + "openapi": "3.0.3", + "info": { + "title": "ClickHouse Cluster Admin Server API", + "description": "API for interacting with the Oxide control plane's ClickHouse cluster replica servers", + "contact": { + "url": "https://oxide.computer", + "email": "api@oxide.computer" + }, + "version": "0.0.1" + }, + "paths": { + "/config": { + "put": { + "summary": "Generate a ClickHouse configuration file for a server node on a specified", + "description": "directory and enable the SMF service.", + "operationId": "generate_config", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ServerConfigurableSettings" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ReplicaConfig" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "ClickhouseHost": { + "oneOf": [ + { + "type": "object", + "properties": { + "ipv6": { + "type": "string", + "format": "ipv6" + } + }, + "required": [ + "ipv6" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "ipv4": { + "type": "string", + "format": "ipv4" + } + }, + "required": [ + "ipv4" + ], + "additionalProperties": false + }, + { + "type": "object", + "properties": { + "domain_name": { + "type": "string" + } + }, + "required": [ + "domain_name" + ], + "additionalProperties": false + } + ] + }, + "Error": { + "description": "Error information from a response.", + "type": "object", + "properties": { + "error_code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "message", + "request_id" + ] + }, + "Generation": { + "description": "Generation numbers stored in the database, used for optimistic concurrency control", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "KeeperConfigsForReplica": { + "type": "object", + "properties": { + "nodes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/KeeperNodeConfig" + } + } + }, + "required": [ + "nodes" + ] + }, + "KeeperNodeConfig": { + "type": "object", + "properties": { + "host": { + "$ref": "#/components/schemas/ClickhouseHost" + }, + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "host", + "port" + ] + }, + "LogConfig": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "format": "uint", + "minimum": 0 + }, + "errorlog": { + "type": "string", + "format": "Utf8PathBuf" + }, + "level": { + "$ref": "#/components/schemas/LogLevel" + }, + "log": { + "type": "string", + "format": "Utf8PathBuf" + }, + "size": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "count", + "errorlog", + "level", + "log", + "size" + ] + }, + "LogLevel": { + "type": "string", + "enum": [ + "trace", + "debug" + ] + }, + "Macros": { + "type": "object", + "properties": { + "cluster": { + "type": "string" + }, + "replica": { + "$ref": "#/components/schemas/ServerId" + }, + "shard": { + "type": "integer", + "format": "uint64", + "minimum": 0 + } + }, + "required": [ + "cluster", + "replica", + "shard" + ] + }, + "RemoteServers": { + "type": "object", + "properties": { + "cluster": { + "type": "string" + }, + "replicas": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ServerNodeConfig" + } + }, + "secret": { + "type": "string" + } + }, + "required": [ + "cluster", + "replicas", + "secret" + ] + }, + "ReplicaConfig": { + "description": "Configuration for a ClickHouse replica server", + "type": "object", + "properties": { + "data_path": { + "description": "Directory for all files generated by ClickHouse itself", + "type": "string", + "format": "Utf8PathBuf" + }, + "http_port": { + "description": "Port for HTTP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "interserver_http_port": { + "description": "Port for interserver HTTP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + }, + "keepers": { + "description": "Contains settings that allow ClickHouse servers to interact with a Keeper cluster", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperConfigsForReplica" + } + ] + }, + "listen_host": { + "description": "Address the server is listening on", + "type": "string", + "format": "ipv6" + }, + "logger": { + "description": "Logging settings", + "allOf": [ + { + "$ref": "#/components/schemas/LogConfig" + } + ] + }, + "macros": { + "description": "Parameter substitutions for replicated tables", + "allOf": [ + { + "$ref": "#/components/schemas/Macros" + } + ] + }, + "remote_servers": { + "description": "Configuration of clusters used by the Distributed table engine and bythe cluster table function", + "allOf": [ + { + "$ref": "#/components/schemas/RemoteServers" + } + ] + }, + "tcp_port": { + "description": "Port for TCP connections", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "data_path", + "http_port", + "interserver_http_port", + "keepers", + "listen_host", + "logger", + "macros", + "remote_servers", + "tcp_port" + ] + }, + "ServerConfigurableSettings": { + "description": "The top most type for configuring clickhouse-servers via clickhouse-admin-server-api", + "type": "object", + "properties": { + "generation": { + "description": "A unique identifier for the configuration generation.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "settings": { + "description": "Configurable settings for a ClickHouse replica server node.", + "allOf": [ + { + "$ref": "#/components/schemas/ServerSettings" + } + ] + } + }, + "required": [ + "generation", + "settings" + ] + }, + "ServerId": { + "description": "A unique ID for a Clickhouse Server", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "ServerNodeConfig": { + "type": "object", + "properties": { + "host": { + "$ref": "#/components/schemas/ClickhouseHost" + }, + "port": { + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "host", + "port" + ] + }, + "ServerSettings": { + "description": "Configurable settings for a ClickHouse replica server node.", + "type": "object", + "properties": { + "config_dir": { + "description": "Directory for the generated server configuration XML file", + "type": "string", + "format": "Utf8PathBuf" + }, + "datastore_path": { + "description": "Directory for all files generated by ClickHouse itself", + "type": "string", + "format": "Utf8PathBuf" + }, + "id": { + "description": "Unique ID of the server node", + "allOf": [ + { + "$ref": "#/components/schemas/ServerId" + } + ] + }, + "keepers": { + "description": "Addresses for each of the individual nodes in the Keeper cluster", + "type": "array", + "items": { + "$ref": "#/components/schemas/ClickhouseHost" + } + }, + "listen_addr": { + "description": "Address the server is listening on", + "type": "string", + "format": "ipv6" + }, + "remote_servers": { + "description": "Addresses for each of the individual replica servers", + "type": "array", + "items": { + "$ref": "#/components/schemas/ClickhouseHost" + } + } + }, + "required": [ + "config_dir", + "datastore_path", + "id", + "keepers", + "listen_addr", + "remote_servers" + ] + } + }, + "responses": { + "Error": { + "description": "Error", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Error" + } + } + } + } + } + } +} diff --git a/openapi/nexus-internal.json b/openapi/nexus-internal.json index 9e05ff72f9..7d762ecc5b 100644 --- a/openapi/nexus-internal.json +++ b/openapi/nexus-internal.json @@ -1872,6 +1872,15 @@ "$ref": "#/components/schemas/BlueprintZonesConfig" } }, + "clickhouse_cluster_config": { + "nullable": true, + "description": "Allocation of Clickhouse Servers and Keepers for replicated clickhouse setups. This is set to `None` if replicated clickhouse is not in use.", + "allOf": [ + { + "$ref": "#/components/schemas/ClickhouseClusterConfig" + } + ] + }, "cockroachdb_fingerprint": { "description": "CockroachDB state fingerprint when this blueprint was created", "type": "string" @@ -2539,6 +2548,74 @@ "key" ] }, + "ClickhouseClusterConfig": { + "description": "Global configuration for all clickhouse servers (replicas) and keepers", + "type": "object", + "properties": { + "cluster_name": { + "description": "An arbitrary name for the Clickhouse cluster shared by all nodes", + "type": "string" + }, + "cluster_secret": { + "description": "An arbitrary string shared by all nodes used at runtime to determine whether nodes are part of the same cluster.", + "type": "string" + }, + "generation": { + "description": "The last update to the clickhouse cluster configuration\n\nThis is used by `clickhouse-admin` in the clickhouse server and keeper zones to discard old configurations.", + "allOf": [ + { + "$ref": "#/components/schemas/Generation" + } + ] + }, + "highest_seen_keeper_leader_committed_log_index": { + "description": "This is used as a marker to tell if the raft configuration in a new inventory collection is newer than the last collection. This serves as a surrogate for the log index of the last committed configuration, which clickhouse keeper doesn't expose.\n\nThis is necesssary because during inventory collection we poll multiple keeper nodes, and each returns their local knowledge of the configuration. But we may reach different nodes in different attempts, and some nodes in a following attempt may reflect stale configuration. Due to timing, we can always query old information. That is just normal polling. However, we never want to use old configuration if we have already seen and acted on newer configuration.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "keepers": { + "description": "The desired state of the clickhouse keeper cluster\n\nWe decouple deployment of zones that should contain clickhouse keeper processes from actually starting or stopping those processes, adding or removing them to/from the keeper cluster, and reconfiguring other keeper and clickhouse server nodes to reflect the new configuration.\n\nAs part of this decoupling, we keep track of the intended zone deployment in the blueprint, but that is not enough to track the desired state of the keeper cluster. We are only allowed to add or remove one keeper node at a time, and therefore we must track the desired state of the keeper cluster which may change multiple times until the keepers in the cluster match the deployed zones. An example may help:\n\n1. We start with 3 keeper nodes in 3 deployed keeper zones and need to add two to reach our desired policy of 5 keepers 2. The planner adds 2 new keeper zones to the blueprint 3. The planner will also add **one** new keeper to the `keepers` field below that matches one of the deployed zones. 4. The executor will start the new keeper process that was added to the `keepers` field, attempt to add it to the keeper cluster by pushing configuration updates to the other keepers, and then updating the clickhouse server configurations to know about the new keeper. 5. If the keeper is successfully added, as reflected in inventory, then steps 3 and 4 above will be repeated for the next keeper process. 6. If the keeper is not successfully added by the executor it will continue to retry indefinitely. 7. If the zone is expunged while the planner has it as part of its desired state in `keepers`, and the executor is trying to add it, the keeper will be removed from `keepers` in the next blueprint. If it has been added to the actual cluster by an executor in the meantime it will be removed on the next iteration of an executor.", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/KeeperId" + } + }, + "max_used_keeper_id": { + "description": "Clickhouse Keeper IDs must be unique and are handed out monotonically. Keep track of the last used one.", + "allOf": [ + { + "$ref": "#/components/schemas/KeeperId" + } + ] + }, + "max_used_server_id": { + "description": "Clickhouse Server IDs must be unique and are handed out monotonically. Keep track of the last used one.", + "allOf": [ + { + "$ref": "#/components/schemas/ServerId" + } + ] + }, + "servers": { + "description": "The desired state of clickhouse server processes on the rack\n\nClickhouse servers do not have the same limitations as keepers and can be deployed all at once.", + "type": "object", + "additionalProperties": { + "$ref": "#/components/schemas/ServerId" + } + } + }, + "required": [ + "cluster_name", + "cluster_secret", + "generation", + "highest_seen_keeper_leader_committed_log_index", + "keepers", + "max_used_keeper_id", + "max_used_server_id", + "servers" + ] + }, "CockroachDbClusterVersion": { "description": "CockroachDB cluster versions we are aware of.\n\nCockroachDB can be upgraded from one major version to the next, e.g. v22.1 -> v22.2. Each major version introduces changes in how it stores data on disk to support new features, and each major version has support for reading the previous version's data so that it can perform an upgrade. The version of the data format is called the \"cluster version\", which is distinct from but related to the software version that's being run.\n\nWhile software version v22.2 is using cluster version v22.1, it's possible to downgrade back to v22.1. Once the cluster version is upgraded, there's no going back.\n\nTo give us some time to evaluate new versions of the software while retaining a downgrade path, we currently deploy new versions of CockroachDB across two releases of the Oxide software, in a \"tick-tock\" model:\n\n- In \"tick\" releases, we upgrade the version of the CockroachDB software to a new major version, and update `CockroachDbClusterVersion::NEWLY_INITIALIZED`. On upgraded racks, the new version is running with the previous cluster version; on newly-initialized racks, the new version is running with the new cluser version. - In \"tock\" releases, we change `CockroachDbClusterVersion::POLICY` to the major version we upgraded to in the last \"tick\" release. This results in a new blueprint that upgrades the cluster version, destroying the downgrade path but allowing us to eventually upgrade to the next release.\n\nThese presently describe major versions of CockroachDB. The order of these must be maintained in the correct order (the first variant must be the earliest version).", "type": "string", @@ -2985,7 +3062,6 @@ ] }, "DnsConfigParams": { - "description": "DnsConfigParams\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"generation\", \"time_created\", \"zones\" ], \"properties\": { \"generation\": { \"type\": \"integer\", \"format\": \"uint64\", \"minimum\": 0.0 }, \"time_created\": { \"type\": \"string\", \"format\": \"date-time\" }, \"zones\": { \"type\": \"array\", \"items\": { \"$ref\": \"#/components/schemas/DnsConfigZone\" } } } } ```
", "type": "object", "properties": { "generation": { @@ -3011,7 +3087,6 @@ ] }, "DnsConfigZone": { - "description": "DnsConfigZone\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"records\", \"zone_name\" ], \"properties\": { \"records\": { \"type\": \"object\", \"additionalProperties\": { \"type\": \"array\", \"items\": { \"$ref\": \"#/components/schemas/DnsRecord\" } } }, \"zone_name\": { \"type\": \"string\" } } } ```
", "type": "object", "properties": { "records": { @@ -3033,7 +3108,6 @@ ] }, "DnsRecord": { - "description": "DnsRecord\n\n
JSON schema\n\n```json { \"oneOf\": [ { \"type\": \"object\", \"required\": [ \"data\", \"type\" ], \"properties\": { \"data\": { \"type\": \"string\", \"format\": \"ipv4\" }, \"type\": { \"type\": \"string\", \"enum\": [ \"A\" ] } } }, { \"type\": \"object\", \"required\": [ \"data\", \"type\" ], \"properties\": { \"data\": { \"type\": \"string\", \"format\": \"ipv6\" }, \"type\": { \"type\": \"string\", \"enum\": [ \"AAAA\" ] } } }, { \"type\": \"object\", \"required\": [ \"data\", \"type\" ], \"properties\": { \"data\": { \"$ref\": \"#/components/schemas/Srv\" }, \"type\": { \"type\": \"string\", \"enum\": [ \"SRV\" ] } } } ] } ```
", "oneOf": [ { "type": "object", @@ -3305,6 +3379,15 @@ "description": "`true` if this instance's auto-restart policy will permit the control plane to automatically restart it if it enters the `Failed` state.", "type": "boolean" }, + "auto_restart_policy": { + "nullable": true, + "description": "The auto-restart policy configured for this instance, or `None` if no explicit policy is configured.\n\nIf this is not present, then this instance uses the default auto-restart policy, which may or may not allow it to be restarted. The `auto_restart_enabled` field indicates whether the instance will be automatically restarted.", + "allOf": [ + { + "$ref": "#/components/schemas/InstanceAutoRestartPolicy" + } + ] + }, "boot_disk_id": { "nullable": true, "description": "the ID of the disk used to boot this Instance, if a specific one is assigned.", @@ -3392,6 +3475,25 @@ "time_run_state_updated" ] }, + "InstanceAutoRestartPolicy": { + "description": "A policy determining when an instance should be automatically restarted by the control plane.", + "oneOf": [ + { + "description": "The instance should not be automatically restarted by the control plane if it fails.", + "type": "string", + "enum": [ + "never" + ] + }, + { + "description": "If this instance is running and unexpectedly fails (e.g. due to a host software crash or unexpected host reboot), the control plane will make a best-effort attempt to restart it. The control plane may choose not to restart the instance to preserve the overall availability of the system.", + "type": "string", + "enum": [ + "best_effort" + ] + } + ] + }, "InstanceCpuCount": { "description": "The number of CPUs in an Instance", "type": "integer", @@ -3639,6 +3741,12 @@ "last" ] }, + "KeeperId": { + "description": "A unique ID for a ClickHouse Keeper", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, "LastResult": { "oneOf": [ { @@ -4712,19 +4820,19 @@ } ] }, - "local_pref": { - "nullable": true, - "description": "The local preference associated with this route.", - "default": null, - "type": "integer", - "format": "uint32", - "minimum": 0 - }, "nexthop": { "description": "The nexthop/gateway address.", "type": "string", "format": "ip" }, + "rib_priority": { + "nullable": true, + "description": "The RIB priority (i.e. Admin Distance) associated with this route.", + "default": null, + "type": "integer", + "format": "uint8", + "minimum": 0 + }, "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", @@ -4951,6 +5059,12 @@ } ] }, + "ServerId": { + "description": "A unique ID for a Clickhouse Server", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, "SledAgentInfo": { "description": "Sent by a sled agent to Nexus to inform about resources", "type": "object", @@ -5210,7 +5324,6 @@ ] }, "Srv": { - "description": "Srv\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"port\", \"prio\", \"target\", \"weight\" ], \"properties\": { \"port\": { \"type\": \"integer\", \"format\": \"uint16\", \"minimum\": 0.0 }, \"prio\": { \"type\": \"integer\", \"format\": \"uint16\", \"minimum\": 0.0 }, \"target\": { \"type\": \"string\" }, \"weight\": { \"type\": \"integer\", \"format\": \"uint16\", \"minimum\": 0.0 } } } ```
", "type": "object", "properties": { "port": { diff --git a/openapi/nexus.json b/openapi/nexus.json index 74333bef82..97f9830b7a 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "20241009.0" + "version": "20241204.0" }, "paths": { "/device/auth": { @@ -2676,14 +2676,22 @@ } } }, - "/v1/ip-pools": { + "/v1/internet-gateway-ip-addresses": { "get": { "tags": [ - "projects" + "vpcs" ], - "summary": "List IP pools", - "operationId": "project_ip_pool_list", + "summary": "List IP addresses attached to internet gateway", + "operationId": "internet_gateway_ip_address_list", "parameters": [ + { + "in": "query", + "name": "gateway", + "description": "Name or ID of the internet gateway", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "limit", @@ -2704,12 +2712,28 @@ "type": "string" } }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", "schema": { "$ref": "#/components/schemas/NameOrIdSortMode" } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "responses": { @@ -2718,7 +2742,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloIpPoolResultsPage" + "$ref": "#/components/schemas/InternetGatewayIpAddressResultsPage" } } } @@ -2731,62 +2755,41 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "gateway" + ] } - } - }, - "/v1/ip-pools/{pool}": { - "get": { + }, + "post": { "tags": [ - "projects" + "vpcs" ], - "summary": "Fetch IP pool", - "operationId": "project_ip_pool_view", + "summary": "Attach IP address to internet gateway", + "operationId": "internet_gateway_ip_address_create", "parameters": [ { - "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", + "in": "query", + "name": "gateway", + "description": "Name or ID of the internet gateway", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SiloIpPool" - } - } - } }, - "4XX": { - "$ref": "#/components/responses/Error" + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/login/{silo_name}/local": { - "post": { - "tags": [ - "login" - ], - "summary": "Authenticate a user via username and password", - "operationId": "login_local", - "parameters": [ { - "in": "path", - "name": "silo_name", - "required": true, + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", "schema": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -2794,59 +2797,19 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UsernamePasswordCredentials" + "$ref": "#/components/schemas/InternetGatewayIpAddressCreate" } } }, "required": true }, "responses": { - "204": { - "description": "resource updated" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/logout": { - "post": { - "tags": [ - "hidden" - ], - "summary": "Log user out of web console by deleting session on client and server", - "operationId": "logout", - "responses": { - "204": { - "description": "resource updated" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/me": { - "get": { - "tags": [ - "session" - ], - "summary": "Fetch user for current session", - "operationId": "current_user_view", - "responses": { - "200": { - "description": "successful operation", + "201": { + "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/CurrentUser" + "$ref": "#/components/schemas/InternetGatewayIpAddress" } } } @@ -2860,52 +2823,59 @@ } } }, - "/v1/me/groups": { - "get": { + "/v1/internet-gateway-ip-addresses/{address}": { + "delete": { "tags": [ - "session" + "vpcs" ], - "summary": "Fetch current user's groups", - "operationId": "current_user_groups", + "summary": "Detach IP address from internet gateway", + "operationId": "internet_gateway_ip_address_delete", "parameters": [ + { + "in": "path", + "name": "address", + "description": "Name or ID of the IP address", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", + "name": "cascade", + "description": "Also delete routes targeting this gateway element.", "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 + "type": "boolean" } }, { "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "name": "gateway", + "description": "Name or ID of the internet gateway", "schema": { - "nullable": true, - "type": "string" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "sort_by", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/GroupResultsPage" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -2913,21 +2883,25 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } } }, - "/v1/me/ssh-keys": { + "/v1/internet-gateway-ip-pools": { "get": { "tags": [ - "session" + "vpcs" ], - "summary": "List SSH public keys", - "description": "Lists SSH public keys for the currently authenticated user.", - "operationId": "current_user_ssh_key_list", + "summary": "List IP pools attached to internet gateway", + "operationId": "internet_gateway_ip_pool_list", "parameters": [ + { + "in": "query", + "name": "gateway", + "description": "Name or ID of the internet gateway", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "limit", @@ -2948,12 +2922,28 @@ "type": "string" } }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", "schema": { "$ref": "#/components/schemas/NameOrIdSortMode" } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "responses": { @@ -2962,7 +2952,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SshKeyResultsPage" + "$ref": "#/components/schemas/InternetGatewayIpPoolResultsPage" } } } @@ -2975,21 +2965,49 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "gateway" + ] } }, "post": { "tags": [ - "session" + "vpcs" + ], + "summary": "Attach IP pool to internet gateway", + "operationId": "internet_gateway_ip_pool_create", + "parameters": [ + { + "in": "query", + "name": "gateway", + "description": "Name or ID of the internet gateway", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } ], - "summary": "Create SSH public key", - "description": "Create an SSH public key for the currently authenticated user.", - "operationId": "current_user_ssh_key_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SshKeyCreate" + "$ref": "#/components/schemas/InternetGatewayIpPoolCreate" } } }, @@ -3001,7 +3019,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SshKey" + "$ref": "#/components/schemas/InternetGatewayIpPool" } } } @@ -3015,157 +3033,59 @@ } } }, - "/v1/me/ssh-keys/{ssh_key}": { - "get": { + "/v1/internet-gateway-ip-pools/{pool}": { + "delete": { "tags": [ - "session" + "vpcs" ], - "summary": "Fetch SSH public key", - "description": "Fetch SSH public key associated with the currently authenticated user.", - "operationId": "current_user_ssh_key_view", + "summary": "Detach IP pool from internet gateway", + "operationId": "internet_gateway_ip_pool_delete", "parameters": [ { "in": "path", - "name": "ssh_key", - "description": "Name or ID of the SSH key", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SshKey" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "session" - ], - "summary": "Delete SSH public key", - "description": "Delete an SSH public key associated with the currently authenticated user.", - "operationId": "current_user_ssh_key_delete", - "parameters": [ - { - "in": "path", - "name": "ssh_key", - "description": "Name or ID of the SSH key", + "name": "pool", + "description": "Name or ID of the IP pool", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - } - }, - "/v1/metrics/{metric_name}": { - "get": { - "tags": [ - "metrics" - ], - "summary": "View metrics", - "description": "View CPU, memory, or storage utilization metrics at the silo or project level.", - "operationId": "silo_metric", - "parameters": [ - { - "in": "path", - "name": "metric_name", - "required": true, - "schema": { - "$ref": "#/components/schemas/SystemMetricName" - } - }, - { - "in": "query", - "name": "end_time", - "description": "An exclusive end time of metrics.", - "schema": { - "type": "string", - "format": "date-time" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } }, { "in": "query", - "name": "order", - "description": "Query result order", + "name": "cascade", + "description": "Also delete routes targeting this gateway element.", "schema": { - "$ref": "#/components/schemas/PaginationOrder" + "type": "boolean" } }, { "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "name": "gateway", + "description": "Name or ID of the internet gateway", "schema": { - "nullable": true, - "type": "string" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "start_time", - "description": "An inclusive start time of metrics.", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { - "type": "string", - "format": "date-time" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "project", - "description": "Name or ID of the project", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `gateway` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/MeasurementResultsPage" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -3173,31 +3093,17 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [ - "end_time", - "start_time" - ] } } }, - "/v1/network-interfaces": { + "/v1/internet-gateways": { "get": { "tags": [ - "instances" + "vpcs" ], - "summary": "List network interfaces", - "operationId": "instance_network_interface_list", + "summary": "List internet gateways", + "operationId": "internet_gateway_list", "parameters": [ - { - "in": "query", - "name": "instance", - "description": "Name or ID of the instance", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "limit", @@ -3221,7 +3127,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -3232,6 +3138,14 @@ "schema": { "$ref": "#/components/schemas/NameOrIdSortMode" } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "responses": { @@ -3240,7 +3154,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + "$ref": "#/components/schemas/InternetGatewayResultsPage" } } } @@ -3254,30 +3168,30 @@ }, "x-dropshot-pagination": { "required": [ - "instance" + "vpc" ] } }, "post": { "tags": [ - "instances" + "vpcs" ], - "summary": "Create network interface", - "operationId": "instance_network_interface_create", + "summary": "Create VPC internet gateway", + "operationId": "internet_gateway_create", "parameters": [ { "in": "query", - "name": "instance", - "description": "Name or ID of the instance", - "required": true, + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -3287,7 +3201,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceCreate" + "$ref": "#/components/schemas/InternetGatewayCreate" } } }, @@ -3299,7 +3213,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterface" + "$ref": "#/components/schemas/InternetGateway" } } } @@ -3313,18 +3227,18 @@ } } }, - "/v1/network-interfaces/{interface}": { + "/v1/internet-gateways/{gateway}": { "get": { "tags": [ - "instances" + "vpcs" ], - "summary": "Fetch network interface", - "operationId": "instance_network_interface_view", + "summary": "Fetch internet gateway", + "operationId": "internet_gateway_view", "parameters": [ { "in": "path", - "name": "interface", - "description": "Name or ID of the network interface", + "name": "gateway", + "description": "Name or ID of the gateway", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -3332,16 +3246,16 @@ }, { "in": "query", - "name": "instance", - "description": "Name or ID of the instance", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -3353,7 +3267,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterface" + "$ref": "#/components/schemas/InternetGateway" } } } @@ -3366,17 +3280,17 @@ } } }, - "put": { + "delete": { "tags": [ - "instances" + "vpcs" ], - "summary": "Update network interface", - "operationId": "instance_network_interface_update", + "summary": "Delete internet gateway", + "operationId": "internet_gateway_delete", "parameters": [ { "in": "path", - "name": "interface", - "description": "Name or ID of the network interface", + "name": "gateway", + "description": "Name or ID of the gateway", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -3384,79 +3298,24 @@ }, { "in": "query", - "name": "instance", - "description": "Name or ID of the instance", + "name": "cascade", + "description": "Also delete routes targeting this gateway.", "schema": { - "$ref": "#/components/schemas/NameOrId" + "type": "boolean" } }, { "in": "query", "name": "project", - "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceUpdate" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterface" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "instances" - ], - "summary": "Delete network interface", - "description": "Note that the primary interface for an instance cannot be deleted if there are any secondary interfaces. A new primary interface must be designated first. The primary interface can be deleted if there are no secondary interfaces.", - "operationId": "instance_network_interface_delete", - "parameters": [ - { - "in": "path", - "name": "interface", - "description": "Name or ID of the network interface", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "instance", - "description": "Name or ID of the instance", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -3475,21 +3334,49 @@ } } }, - "/v1/ping": { + "/v1/ip-pools": { "get": { "tags": [ - "system/status" + "projects" + ], + "summary": "List IP pools", + "operationId": "project_ip_pool_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } ], - "summary": "Ping API", - "description": "Always responds with Ok if it responds at all.", - "operationId": "ping", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Ping" + "$ref": "#/components/schemas/SiloIpPoolResultsPage" } } } @@ -3500,23 +3387,37 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } } }, - "/v1/policy": { + "/v1/ip-pools/{pool}": { "get": { "tags": [ - "silos" + "projects" + ], + "summary": "Fetch IP pool", + "operationId": "project_ip_pool_view", + "parameters": [ + { + "in": "path", + "name": "pool", + "description": "Name or ID of the IP pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } ], - "summary": "Fetch current silo's IAM policy", - "operationId": "policy_view", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloRolePolicy" + "$ref": "#/components/schemas/SiloIpPool" } } } @@ -3528,30 +3429,82 @@ "$ref": "#/components/responses/Error" } } - }, - "put": { + } + }, + "/v1/login/{silo_name}/local": { + "post": { "tags": [ - "silos" + "login" + ], + "summary": "Authenticate a user via username and password", + "operationId": "login_local", + "parameters": [ + { + "in": "path", + "name": "silo_name", + "required": true, + "schema": { + "$ref": "#/components/schemas/Name" + } + } ], - "summary": "Update current silo's IAM policy", - "operationId": "policy_update", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloRolePolicy" + "$ref": "#/components/schemas/UsernamePasswordCredentials" } } }, "required": true }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/logout": { + "post": { + "tags": [ + "hidden" + ], + "summary": "Log user out of web console by deleting session on client and server", + "operationId": "logout", + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/me": { + "get": { + "tags": [ + "session" + ], + "summary": "Fetch user for current session", + "operationId": "current_user_view", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloRolePolicy" + "$ref": "#/components/schemas/CurrentUser" } } } @@ -3565,13 +3518,73 @@ } } }, - "/v1/projects": { + "/v1/me/groups": { "get": { "tags": [ - "projects" + "session" ], - "summary": "List projects", - "operationId": "project_list", + "summary": "Fetch current user's groups", + "operationId": "current_user_groups", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/GroupResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/me/ssh-keys": { + "get": { + "tags": [ + "session" + ], + "summary": "List SSH public keys", + "description": "Lists SSH public keys for the currently authenticated user.", + "operationId": "current_user_ssh_key_list", "parameters": [ { "in": "query", @@ -3607,7 +3620,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProjectResultsPage" + "$ref": "#/components/schemas/SshKeyResultsPage" } } } @@ -3625,15 +3638,16 @@ }, "post": { "tags": [ - "projects" + "session" ], - "summary": "Create project", - "operationId": "project_create", + "summary": "Create SSH public key", + "description": "Create an SSH public key for the currently authenticated user.", + "operationId": "current_user_ssh_key_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProjectCreate" + "$ref": "#/components/schemas/SshKeyCreate" } } }, @@ -3645,7 +3659,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Project" + "$ref": "#/components/schemas/SshKey" } } } @@ -3659,18 +3673,19 @@ } } }, - "/v1/projects/{project}": { + "/v1/me/ssh-keys/{ssh_key}": { "get": { "tags": [ - "projects" + "session" ], - "summary": "Fetch project", - "operationId": "project_view", + "summary": "Fetch SSH public key", + "description": "Fetch SSH public key associated with the currently authenticated user.", + "operationId": "current_user_ssh_key_view", "parameters": [ { "in": "path", - "name": "project", - "description": "Name or ID of the project", + "name": "ssh_key", + "description": "Name or ID of the SSH key", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -3683,7 +3698,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Project" + "$ref": "#/components/schemas/SshKey" } } } @@ -3696,63 +3711,18 @@ } } }, - "put": { + "delete": { "tags": [ - "projects" + "session" ], - "summary": "Update a project", - "operationId": "project_update", + "summary": "Delete SSH public key", + "description": "Delete an SSH public key associated with the currently authenticated user.", + "operationId": "current_user_ssh_key_delete", "parameters": [ { "in": "path", - "name": "project", - "description": "Name or ID of the project", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProjectUpdate" - } - } - }, - "required": true - }, - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Project" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "projects" - ], - "summary": "Delete project", - "operationId": "project_delete", - "parameters": [ - { - "in": "path", - "name": "project", - "description": "Name or ID of the project", + "name": "ssh_key", + "description": "Name or ID of the SSH key", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -3772,77 +3742,85 @@ } } }, - "/v1/projects/{project}/policy": { + "/v1/metrics/{metric_name}": { "get": { "tags": [ - "projects" + "metrics" ], - "summary": "Fetch project's IAM policy", - "operationId": "project_policy_view", + "summary": "View metrics", + "description": "View CPU, memory, or storage utilization metrics at the silo or project level.", + "operationId": "silo_metric", "parameters": [ { "in": "path", - "name": "project", - "description": "Name or ID of the project", + "name": "metric_name", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "$ref": "#/components/schemas/SystemMetricName" } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProjectRolePolicy" - } - } + }, + { + "in": "query", + "name": "end_time", + "description": "An exclusive end time of metrics.", + "schema": { + "type": "string", + "format": "date-time" } }, - "4XX": { - "$ref": "#/components/responses/Error" + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "projects" - ], - "summary": "Update project's IAM policy", - "operationId": "project_policy_update", - "parameters": [ { - "in": "path", + "in": "query", + "name": "order", + "description": "Query result order", + "schema": { + "$ref": "#/components/schemas/PaginationOrder" + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "start_time", + "description": "An inclusive start time of metrics.", + "schema": { + "type": "string", + "format": "date-time" + } + }, + { + "in": "query", "name": "project", "description": "Name or ID of the project", - "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/ProjectRolePolicy" - } - } - }, - "required": true - }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/ProjectRolePolicy" + "$ref": "#/components/schemas/MeasurementResultsPage" } } } @@ -3853,17 +3831,31 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [ + "end_time", + "start_time" + ] } } }, - "/v1/snapshots": { + "/v1/network-interfaces": { "get": { "tags": [ - "snapshots" + "instances" ], - "summary": "List snapshots", - "operationId": "snapshot_list", + "summary": "List network interfaces", + "operationId": "instance_network_interface_list", "parameters": [ + { + "in": "query", + "name": "instance", + "description": "Name or ID of the instance", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "limit", @@ -3887,7 +3879,7 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -3906,7 +3898,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SnapshotResultsPage" + "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" } } } @@ -3920,33 +3912,40 @@ }, "x-dropshot-pagination": { "required": [ - "project" + "instance" ] } }, "post": { "tags": [ - "snapshots" + "instances" ], - "summary": "Create snapshot", - "description": "Creates a point-in-time snapshot from a disk.", - "operationId": "snapshot_create", + "summary": "Create network interface", + "operationId": "instance_network_interface_create", "parameters": [ { "in": "query", - "name": "project", - "description": "Name or ID of the project", + "name": "instance", + "description": "Name or ID of the instance", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SnapshotCreate" + "$ref": "#/components/schemas/InstanceNetworkInterfaceCreate" } } }, @@ -3958,7 +3957,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Snapshot" + "$ref": "#/components/schemas/InstanceNetworkInterface" } } } @@ -3972,27 +3971,35 @@ } } }, - "/v1/snapshots/{snapshot}": { + "/v1/network-interfaces/{interface}": { "get": { "tags": [ - "snapshots" + "instances" ], - "summary": "Fetch snapshot", - "operationId": "snapshot_view", + "summary": "Fetch network interface", + "operationId": "instance_network_interface_view", "parameters": [ { "in": "path", - "name": "snapshot", - "description": "Name or ID of the snapshot", + "name": "interface", + "description": "Name or ID of the network interface", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "instance", + "description": "Name or ID of the instance", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -4004,7 +4011,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Snapshot" + "$ref": "#/components/schemas/InstanceNetworkInterface" } } } @@ -4017,34 +4024,59 @@ } } }, - "delete": { + "put": { "tags": [ - "snapshots" + "instances" ], - "summary": "Delete snapshot", - "operationId": "snapshot_delete", + "summary": "Update network interface", + "operationId": "instance_network_interface_update", "parameters": [ { "in": "path", - "name": "snapshot", - "description": "Name or ID of the snapshot", + "name": "interface", + "description": "Name or ID of the network interface", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "instance", + "description": "Name or ID of the instance", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceNetworkInterfaceUpdate" + } + } + }, + "required": true + }, "responses": { - "204": { - "description": "successful deletion" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceNetworkInterface" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -4053,54 +4085,44 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/system/hardware/disks": { - "get": { + }, + "delete": { "tags": [ - "system/hardware" + "instances" ], - "summary": "List physical disks", - "operationId": "physical_disk_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", + "summary": "Delete network interface", + "description": "Note that the primary interface for an instance cannot be deleted if there are any secondary interfaces. A new primary interface must be designated first. The primary interface can be deleted if there are no secondary interfaces.", + "operationId": "instance_network_interface_delete", + "parameters": [ + { + "in": "path", + "name": "interface", + "description": "Name or ID of the network interface", + "required": true, "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "name": "instance", + "description": "Name or ID of the instance", "schema": { - "nullable": true, - "type": "string" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "sort_by", + "name": "project", + "description": "Name or ID of the project, only required if `instance` is provided as a `Name`", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/PhysicalDiskResultsPage" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -4108,38 +4130,24 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } } }, - "/v1/system/hardware/disks/{disk_id}": { + "/v1/ping": { "get": { "tags": [ - "system/hardware" - ], - "summary": "Get a physical disk", - "operationId": "physical_disk_view", - "parameters": [ - { - "in": "path", - "name": "disk_id", - "description": "ID of the physical disk", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } - } + "system/status" ], + "summary": "Ping API", + "description": "Always responds with Ok if it responds at all.", + "operationId": "ping", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PhysicalDisk" + "$ref": "#/components/schemas/Ping" } } } @@ -4153,49 +4161,20 @@ } } }, - "/v1/system/hardware/racks": { + "/v1/policy": { "get": { "tags": [ - "system/hardware" - ], - "summary": "List racks", - "operationId": "rack_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/IdSortMode" - } - } + "silos" ], + "summary": "Fetch current silo's IAM policy", + "operationId": "policy_view", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RackResultsPage" + "$ref": "#/components/schemas/SiloRolePolicy" } } } @@ -4206,38 +4185,31 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } - } - }, - "/v1/system/hardware/racks/{rack_id}": { - "get": { + }, + "put": { "tags": [ - "system/hardware" + "silos" ], - "summary": "Fetch rack", - "operationId": "rack_view", - "parameters": [ - { - "in": "path", - "name": "rack_id", - "description": "ID of the rack", - "required": true, - "schema": { - "type": "string", - "format": "uuid" + "summary": "Update current silo's IAM policy", + "operationId": "policy_update", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SiloRolePolicy" + } } - } - ], + }, + "required": true + }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Rack" + "$ref": "#/components/schemas/SiloRolePolicy" } } } @@ -4251,13 +4223,13 @@ } } }, - "/v1/system/hardware/sleds": { + "/v1/projects": { "get": { "tags": [ - "system/hardware" + "projects" ], - "summary": "List sleds", - "operationId": "sled_list", + "summary": "List projects", + "operationId": "project_list", "parameters": [ { "in": "query", @@ -4283,7 +4255,7 @@ "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrIdSortMode" } } ], @@ -4293,7 +4265,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SledResultsPage" + "$ref": "#/components/schemas/ProjectResultsPage" } } } @@ -4311,15 +4283,15 @@ }, "post": { "tags": [ - "system/hardware" + "projects" ], - "summary": "Add sled to initialized rack", - "operationId": "sled_add", + "summary": "Create project", + "operationId": "project_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UninitializedSledId" + "$ref": "#/components/schemas/ProjectCreate" } } }, @@ -4331,7 +4303,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SledId" + "$ref": "#/components/schemas/Project" } } } @@ -4345,22 +4317,21 @@ } } }, - "/v1/system/hardware/sleds/{sled_id}": { + "/v1/projects/{project}": { "get": { "tags": [ - "system/hardware" + "projects" ], - "summary": "Fetch sled", - "operationId": "sled_view", + "summary": "Fetch project", + "operationId": "project_view", "parameters": [ { "in": "path", - "name": "sled_id", - "description": "ID of the sled", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -4370,7 +4341,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Sled" + "$ref": "#/components/schemas/Project" } } } @@ -4382,61 +4353,41 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/system/hardware/sleds/{sled_id}/disks": { - "get": { + }, + "put": { "tags": [ - "system/hardware" + "projects" ], - "summary": "List physical disks attached to sleds", - "operationId": "sled_physical_disk_list", + "summary": "Update a project", + "operationId": "project_update", "parameters": [ { "in": "path", - "name": "sled_id", - "description": "ID of the sled", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { - "type": "string", - "format": "uuid" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProjectUpdate" + } + } + }, + "required": true + }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/PhysicalDiskResultsPage" + "$ref": "#/components/schemas/Project" } } } @@ -4447,68 +4398,28 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } - } - }, - "/v1/system/hardware/sleds/{sled_id}/instances": { - "get": { + }, + "delete": { "tags": [ - "system/hardware" + "projects" ], - "summary": "List instances running on given sled", - "operationId": "sled_instance_list", + "summary": "Delete project", + "operationId": "project_delete", "parameters": [ { "in": "path", - "name": "sled_id", - "description": "ID of the sled", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { - "type": "string", - "format": "uuid" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SledInstanceResultsPage" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -4516,48 +4427,34 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } } }, - "/v1/system/hardware/sleds/{sled_id}/provision-policy": { - "put": { + "/v1/projects/{project}/policy": { + "get": { "tags": [ - "system/hardware" + "projects" ], - "summary": "Set sled provision policy", - "operationId": "sled_set_provision_policy", + "summary": "Fetch project's IAM policy", + "operationId": "project_policy_view", "parameters": [ { "in": "path", - "name": "sled_id", - "description": "ID of the sled", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SledProvisionPolicyParams" - } - } - }, - "required": true - }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SledProvisionPolicyResponse" + "$ref": "#/components/schemas/ProjectRolePolicy" } } } @@ -4569,44 +4466,41 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/system/hardware/sleds-uninitialized": { - "get": { + }, + "put": { "tags": [ - "system/hardware" + "projects" ], - "summary": "List uninitialized sleds", - "operationId": "sled_list_uninitialized", + "summary": "Update project's IAM policy", + "operationId": "project_policy_update", "parameters": [ { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "in": "path", + "name": "project", + "description": "Name or ID of the project", + "required": true, "schema": { - "nullable": true, - "type": "string" + "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ProjectRolePolicy" + } + } + }, + "required": true + }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UninitializedSledResultsPage" + "$ref": "#/components/schemas/ProjectRolePolicy" } } } @@ -4617,19 +4511,16 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } } }, - "/v1/system/hardware/switch-port": { + "/v1/snapshots": { "get": { "tags": [ - "system/hardware" + "snapshots" ], - "summary": "List switch ports", - "operationId": "networking_switch_port_list", + "summary": "List snapshots", + "operationId": "snapshot_list", "parameters": [ { "in": "query", @@ -4653,19 +4544,17 @@ }, { "in": "query", - "name": "sort_by", + "name": "project", + "description": "Name or ID of the project", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "switch_port_id", - "description": "An optional switch port id to use when listing switch ports.", + "name": "sort_by", "schema": { - "nullable": true, - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrIdSortMode" } } ], @@ -4675,7 +4564,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SwitchPortResultsPage" + "$ref": "#/components/schemas/SnapshotResultsPage" } } } @@ -4688,44 +4577,26 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "project" + ] } - } - }, - "/v1/system/hardware/switch-port/{port}/settings": { + }, "post": { "tags": [ - "system/hardware" + "snapshots" ], - "summary": "Apply switch port settings", - "operationId": "networking_switch_port_apply_settings", + "summary": "Create snapshot", + "description": "Creates a point-in-time snapshot from a disk.", + "operationId": "snapshot_create", "parameters": [ - { - "in": "path", - "name": "port", - "description": "A name to use when selecting switch ports.", - "required": true, - "schema": { - "$ref": "#/components/schemas/Name" - } - }, - { - "in": "query", - "name": "rack_id", - "description": "A rack id to use when selecting switch ports.", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } - }, { "in": "query", - "name": "switch_location", - "description": "A switch location to use when selecting switch ports.", + "name": "project", + "description": "Name or ID of the project", "required": true, "schema": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -4733,15 +4604,22 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SwitchPortApplySettings" + "$ref": "#/components/schemas/SnapshotCreate" } } }, "required": true }, "responses": { - "204": { - "description": "resource updated" + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Snapshot" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -4750,46 +4628,44 @@ "$ref": "#/components/responses/Error" } } - }, - "delete": { + } + }, + "/v1/snapshots/{snapshot}": { + "get": { "tags": [ - "system/hardware" + "snapshots" ], - "summary": "Clear switch port settings", - "operationId": "networking_switch_port_clear_settings", + "summary": "Fetch snapshot", + "operationId": "snapshot_view", "parameters": [ { "in": "path", - "name": "port", - "description": "A name to use when selecting switch ports.", - "required": true, - "schema": { - "$ref": "#/components/schemas/Name" - } - }, - { - "in": "query", - "name": "rack_id", - "description": "A rack id to use when selecting switch ports.", + "name": "snapshot", + "description": "Name or ID of the snapshot", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "switch_location", - "description": "A switch location to use when selecting switch ports.", - "required": true, + "name": "project", + "description": "Name or ID of the project", "schema": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "204": { - "description": "resource updated" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Snapshot" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -4798,55 +4674,35 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/system/hardware/switch-port/{port}/status": { - "get": { + }, + "delete": { "tags": [ - "system/hardware" + "snapshots" ], - "summary": "Get switch port status", - "operationId": "networking_switch_port_status", + "summary": "Delete snapshot", + "operationId": "snapshot_delete", "parameters": [ { "in": "path", - "name": "port", - "description": "A name to use when selecting switch ports.", - "required": true, - "schema": { - "$ref": "#/components/schemas/Name" - } - }, - { - "in": "query", - "name": "rack_id", - "description": "A rack id to use when selecting switch ports.", + "name": "snapshot", + "description": "Name or ID of the snapshot", "required": true, "schema": { - "type": "string", - "format": "uuid" + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "switch_location", - "description": "A switch location to use when selecting switch ports.", - "required": true, + "name": "project", + "description": "Name or ID of the project", "schema": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SwitchLinkState" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -4857,13 +4713,13 @@ } } }, - "/v1/system/hardware/switches": { + "/v1/system/hardware/disks": { "get": { "tags": [ "system/hardware" ], - "summary": "List switches", - "operationId": "switch_list", + "summary": "List physical disks", + "operationId": "physical_disk_list", "parameters": [ { "in": "query", @@ -4899,7 +4755,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SwitchResultsPage" + "$ref": "#/components/schemas/PhysicalDiskResultsPage" } } } @@ -4916,18 +4772,18 @@ } } }, - "/v1/system/hardware/switches/{switch_id}": { + "/v1/system/hardware/disks/{disk_id}": { "get": { "tags": [ "system/hardware" ], - "summary": "Fetch switch", - "operationId": "switch_view", + "summary": "Get a physical disk", + "operationId": "physical_disk_view", "parameters": [ { "in": "path", - "name": "switch_id", - "description": "ID of the switch", + "name": "disk_id", + "description": "ID of the physical disk", "required": true, "schema": { "type": "string", @@ -4941,7 +4797,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Switch" + "$ref": "#/components/schemas/PhysicalDisk" } } } @@ -4955,13 +4811,13 @@ } } }, - "/v1/system/identity-providers": { + "/v1/system/hardware/racks": { "get": { "tags": [ - "system/silos" + "system/hardware" ], - "summary": "List a silo's IdP's name", - "operationId": "silo_identity_provider_list", + "summary": "List racks", + "operationId": "rack_list", "parameters": [ { "in": "query", @@ -4983,19 +4839,11 @@ "type": "string" } }, - { - "in": "query", - "name": "silo", - "description": "Name or ID of the silo", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" + "$ref": "#/components/schemas/IdSortMode" } } ], @@ -5005,7 +4853,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IdentityProviderResultsPage" + "$ref": "#/components/schemas/RackResultsPage" } } } @@ -5018,48 +4866,36 @@ } }, "x-dropshot-pagination": { - "required": [ - "silo" - ] + "required": [] } } }, - "/v1/system/identity-providers/local/users": { - "post": { + "/v1/system/hardware/racks/{rack_id}": { + "get": { "tags": [ - "system/silos" + "system/hardware" ], - "summary": "Create user", - "description": "Users can only be created in Silos with `provision_type` == `Fixed`. Otherwise, Silo users are just-in-time (JIT) provisioned when a user first logs in using an external Identity Provider.", - "operationId": "local_idp_user_create", + "summary": "Fetch rack", + "operationId": "rack_view", "parameters": [ { - "in": "query", - "name": "silo", - "description": "Name or ID of the silo", + "in": "path", + "name": "rack_id", + "description": "ID of the rack", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "type": "string", + "format": "uuid" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UserCreate" - } - } - }, - "required": true - }, "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/User" + "$ref": "#/components/schemas/Rack" } } } @@ -5073,37 +4909,52 @@ } } }, - "/v1/system/identity-providers/local/users/{user_id}": { - "delete": { + "/v1/system/hardware/sleds": { + "get": { "tags": [ - "system/silos" + "system/hardware" ], - "summary": "Delete user", - "operationId": "local_idp_user_delete", + "summary": "List sleds", + "operationId": "sled_list", "parameters": [ { - "in": "path", - "name": "user_id", - "description": "The user's internal ID", - "required": true, + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "type": "string", - "format": "uuid" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 } }, { "in": "query", - "name": "silo", - "description": "Name or ID of the silo", - "required": true, + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" } } ], "responses": { - "204": { - "description": "successful deletion" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledResultsPage" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -5111,51 +4962,37 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } - } - }, - "/v1/system/identity-providers/local/users/{user_id}/set-password": { + }, "post": { "tags": [ - "system/silos" - ], - "summary": "Set or invalidate user's password", - "description": "Passwords can only be updated for users in Silos with identity mode `LocalOnly`.", - "operationId": "local_idp_user_set_password", - "parameters": [ - { - "in": "path", - "name": "user_id", - "description": "The user's internal ID", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } - }, - { - "in": "query", - "name": "silo", - "description": "Name or ID of the silo", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } + "system/hardware" ], + "summary": "Add sled to initialized rack", + "operationId": "sled_add", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UserPassword" + "$ref": "#/components/schemas/UninitializedSledId" } } }, "required": true }, "responses": { - "204": { - "description": "resource updated" + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SledId" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -5166,41 +5003,32 @@ } } }, - "/v1/system/identity-providers/saml": { - "post": { + "/v1/system/hardware/sleds/{sled_id}": { + "get": { "tags": [ - "system/silos" + "system/hardware" ], - "summary": "Create SAML IdP", - "operationId": "saml_identity_provider_create", + "summary": "Fetch sled", + "operationId": "sled_view", "parameters": [ { - "in": "query", - "name": "silo", - "description": "Name or ID of the silo", + "in": "path", + "name": "sled_id", + "description": "ID of the sled", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "type": "string", + "format": "uuid" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SamlIdentityProviderCreate" - } - } - }, - "required": true - }, "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SamlIdentityProvider" + "$ref": "#/components/schemas/Sled" } } } @@ -5214,30 +5042,49 @@ } } }, - "/v1/system/identity-providers/saml/{provider}": { + "/v1/system/hardware/sleds/{sled_id}/disks": { "get": { "tags": [ - "system/silos" + "system/hardware" ], - "summary": "Fetch SAML IdP", - "operationId": "saml_identity_provider_view", + "summary": "List physical disks attached to sleds", + "operationId": "sled_physical_disk_list", "parameters": [ { "in": "path", - "name": "provider", - "description": "Name or ID of the SAML identity provider", + "name": "sled_id", + "description": "ID of the sled", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "type": "string", + "format": "uuid" } }, { "in": "query", - "name": "silo", - "description": "Name or ID of the silo", - "required": true, + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" } } ], @@ -5247,7 +5094,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SamlIdentityProvider" + "$ref": "#/components/schemas/PhysicalDiskResultsPage" } } } @@ -5258,17 +5105,30 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } } }, - "/v1/system/ip-pools": { + "/v1/system/hardware/sleds/{sled_id}/instances": { "get": { "tags": [ - "system/ip-pools" + "system/hardware" ], - "summary": "List IP pools", - "operationId": "ip_pool_list", + "summary": "List instances running on given sled", + "operationId": "sled_instance_list", "parameters": [ + { + "in": "path", + "name": "sled_id", + "description": "ID of the sled", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, { "in": "query", "name": "limit", @@ -5293,7 +5153,7 @@ "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" + "$ref": "#/components/schemas/IdSortMode" } } ], @@ -5303,7 +5163,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolResultsPage" + "$ref": "#/components/schemas/SledInstanceResultsPage" } } } @@ -5318,30 +5178,44 @@ "x-dropshot-pagination": { "required": [] } - }, - "post": { + } + }, + "/v1/system/hardware/sleds/{sled_id}/provision-policy": { + "put": { "tags": [ - "system/ip-pools" + "system/hardware" + ], + "summary": "Set sled provision policy", + "operationId": "sled_set_provision_policy", + "parameters": [ + { + "in": "path", + "name": "sled_id", + "description": "ID of the sled", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + } ], - "summary": "Create IP pool", - "operationId": "ip_pool_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolCreate" + "$ref": "#/components/schemas/SledProvisionPolicyParams" } } }, "required": true }, "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPool" + "$ref": "#/components/schemas/SledProvisionPolicyResponse" } } } @@ -5355,77 +5229,42 @@ } } }, - "/v1/system/ip-pools/{pool}": { + "/v1/system/hardware/sleds-uninitialized": { "get": { "tags": [ - "system/ip-pools" + "system/hardware" ], - "summary": "Fetch IP pool", - "operationId": "ip_pool_view", + "summary": "List uninitialized sleds", + "operationId": "sled_list_uninitialized", "parameters": [ { - "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", - "required": true, + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IpPool" - } - } + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 } }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "put": { - "tags": [ - "system/ip-pools" - ], - "summary": "Update IP pool", - "operationId": "ip_pool_update", - "parameters": [ { - "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", - "required": true, + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "string" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IpPoolUpdate" - } - } - }, - "required": true - }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPool" + "$ref": "#/components/schemas/UninitializedSledResultsPage" } } } @@ -5436,56 +5275,20 @@ "5XX": { "$ref": "#/components/responses/Error" } - } - }, - "delete": { - "tags": [ - "system/ip-pools" - ], - "summary": "Delete IP pool", - "operationId": "ip_pool_delete", - "parameters": [ - { - "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "responses": { - "204": { - "description": "successful deletion" - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } + }, + "x-dropshot-pagination": { + "required": [] } } }, - "/v1/system/ip-pools/{pool}/ranges": { + "/v1/system/hardware/switch-port": { "get": { "tags": [ - "system/ip-pools" + "system/hardware" ], - "summary": "List ranges for IP pool", - "description": "Ranges are ordered by their first address.", - "operationId": "ip_pool_range_list", + "summary": "List switch ports", + "operationId": "networking_switch_port_list", "parameters": [ - { - "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "limit", @@ -5505,6 +5308,23 @@ "nullable": true, "type": "string" } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } + }, + { + "in": "query", + "name": "switch_port_id", + "description": "An optional switch port id to use when listing switch ports.", + "schema": { + "nullable": true, + "type": "string", + "format": "uuid" + } } ], "responses": { @@ -5513,7 +5333,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolRangeResultsPage" + "$ref": "#/components/schemas/SwitchPortResultsPage" } } } @@ -5530,46 +5350,57 @@ } } }, - "/v1/system/ip-pools/{pool}/ranges/add": { + "/v1/system/hardware/switch-port/{port}/settings": { "post": { "tags": [ - "system/ip-pools" + "system/hardware" ], - "summary": "Add range to IP pool", - "description": "IPv6 ranges are not allowed yet.", - "operationId": "ip_pool_range_add", + "summary": "Apply switch port settings", + "operationId": "networking_switch_port_apply_settings", "parameters": [ { "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", + "name": "port", + "description": "A name to use when selecting switch ports.", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IpRange" - } + "$ref": "#/components/schemas/Name" } }, - "required": true - }, - "responses": { - "201": { - "description": "successful creation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IpPoolRange" - } + { + "in": "query", + "name": "rack_id", + "description": "A rack id to use when selecting switch ports.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "switch_location", + "description": "A switch location to use when selecting switch ports.", + "required": true, + "schema": { + "$ref": "#/components/schemas/Name" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SwitchPortApplySettings" } } }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, "4XX": { "$ref": "#/components/responses/Error" }, @@ -5577,36 +5408,43 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/system/ip-pools/{pool}/ranges/remove": { - "post": { + }, + "delete": { "tags": [ - "system/ip-pools" + "system/hardware" ], - "summary": "Remove range from IP pool", - "operationId": "ip_pool_range_remove", + "summary": "Clear switch port settings", + "operationId": "networking_switch_port_clear_settings", "parameters": [ { "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", + "name": "port", + "description": "A name to use when selecting switch ports.", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "$ref": "#/components/schemas/Name" } - } - ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IpRange" - } + }, + { + "in": "query", + "name": "rack_id", + "description": "A rack id to use when selecting switch ports.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" } }, - "required": true - }, + { + "in": "query", + "name": "switch_location", + "description": "A switch location to use when selecting switch ports.", + "required": true, + "schema": { + "$ref": "#/components/schemas/Name" + } + } + ], "responses": { "204": { "description": "resource updated" @@ -5620,23 +5458,71 @@ } } }, - "/v1/system/ip-pools/{pool}/silos": { + "/v1/system/hardware/switch-port/{port}/status": { "get": { "tags": [ - "system/ip-pools" + "system/hardware" ], - "summary": "List IP pool's linked silos", - "operationId": "ip_pool_silo_list", + "summary": "Get switch port status", + "operationId": "networking_switch_port_status", "parameters": [ { "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", + "name": "port", + "description": "A name to use when selecting switch ports.", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "$ref": "#/components/schemas/Name" + } + }, + { + "in": "query", + "name": "rack_id", + "description": "A rack id to use when selecting switch ports.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "switch_location", + "description": "A switch location to use when selecting switch ports.", + "required": true, + "schema": { + "$ref": "#/components/schemas/Name" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SwitchLinkState" + } + } } }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/hardware/switches": { + "get": { + "tags": [ + "system/hardware" + ], + "summary": "List switches", + "operationId": "switch_list", + "parameters": [ { "in": "query", "name": "limit", @@ -5671,7 +5557,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolSiloLinkResultsPage" + "$ref": "#/components/schemas/SwitchResultsPage" } } } @@ -5686,42 +5572,34 @@ "x-dropshot-pagination": { "required": [] } - }, - "post": { + } + }, + "/v1/system/hardware/switches/{switch_id}": { + "get": { "tags": [ - "system/ip-pools" + "system/hardware" ], - "summary": "Link IP pool to silo", - "description": "Users in linked silos can allocate external IPs from this pool for their instances. A silo can have at most one default pool. IPs are allocated from the default pool when users ask for one without specifying a pool.", - "operationId": "ip_pool_silo_link", + "summary": "Fetch switch", + "operationId": "switch_view", "parameters": [ { "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", + "name": "switch_id", + "description": "ID of the switch", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "type": "string", + "format": "uuid" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IpPoolLinkSilo" - } - } - }, - "required": true - }, "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolSiloLink" + "$ref": "#/components/schemas/Switch" } } } @@ -5735,49 +5613,57 @@ } } }, - "/v1/system/ip-pools/{pool}/silos/{silo}": { - "put": { + "/v1/system/identity-providers": { + "get": { "tags": [ - "system/ip-pools" + "system/silos" ], - "summary": "Make IP pool default for silo", - "description": "When a user asks for an IP (e.g., at instance create time) without specifying a pool, the IP comes from the default pool if a default is configured. When a pool is made the default for a silo, any existing default will remain linked to the silo, but will no longer be the default.", - "operationId": "ip_pool_silo_update", + "summary": "List a silo's IdP's name", + "operationId": "silo_identity_provider_list", "parameters": [ { - "in": "path", - "name": "pool", - "required": true, + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 } }, { - "in": "path", + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", "name": "silo", - "required": true, + "description": "Name or ID of the silo", "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IpPoolSiloUpdate" - } - } - }, - "required": true - }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolSiloLink" + "$ref": "#/components/schemas/IdentityProviderResultsPage" } } } @@ -5788,33 +5674,143 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [ + "silo" + ] } - }, - "delete": { + } + }, + "/v1/system/identity-providers/local/users": { + "post": { "tags": [ - "system/ip-pools" + "system/silos" ], - "summary": "Unlink IP pool from silo", - "description": "Will fail if there are any outstanding IPs allocated in the silo.", - "operationId": "ip_pool_silo_unlink", - "parameters": [ - { + "summary": "Create user", + "description": "Users can only be created in Silos with `provision_type` == `Fixed`. Otherwise, Silo users are just-in-time (JIT) provisioned when a user first logs in using an external Identity Provider.", + "operationId": "local_idp_user_create", + "parameters": [ + { + "in": "query", + "name": "silo", + "description": "Name or ID of the silo", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/User" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/identity-providers/local/users/{user_id}": { + "delete": { + "tags": [ + "system/silos" + ], + "summary": "Delete user", + "operationId": "local_idp_user_delete", + "parameters": [ + { "in": "path", - "name": "pool", + "name": "user_id", + "description": "The user's internal ID", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "silo", + "description": "Name or ID of the silo", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/system/identity-providers/local/users/{user_id}/set-password": { + "post": { + "tags": [ + "system/silos" + ], + "summary": "Set or invalidate user's password", + "description": "Passwords can only be updated for users in Silos with identity mode `LocalOnly`.", + "operationId": "local_idp_user_set_password", + "parameters": [ { "in": "path", + "name": "user_id", + "description": "The user's internal ID", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", "name": "silo", + "description": "Name or ID of the silo", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserPassword" + } + } + }, + "required": true + }, "responses": { "204": { "description": "resource updated" @@ -5828,31 +5824,41 @@ } } }, - "/v1/system/ip-pools/{pool}/utilization": { - "get": { + "/v1/system/identity-providers/saml": { + "post": { "tags": [ - "system/ip-pools" + "system/silos" ], - "summary": "Fetch IP pool utilization", - "operationId": "ip_pool_utilization_view", + "summary": "Create SAML IdP", + "operationId": "saml_identity_provider_create", "parameters": [ { - "in": "path", - "name": "pool", - "description": "Name or ID of the IP pool", + "in": "query", + "name": "silo", + "description": "Name or ID of the silo", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SamlIdentityProviderCreate" + } + } + }, + "required": true + }, "responses": { - "200": { - "description": "successful operation", + "201": { + "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolUtilization" + "$ref": "#/components/schemas/SamlIdentityProvider" } } } @@ -5866,20 +5872,40 @@ } } }, - "/v1/system/ip-pools-service": { + "/v1/system/identity-providers/saml/{provider}": { "get": { "tags": [ - "system/ip-pools" + "system/silos" + ], + "summary": "Fetch SAML IdP", + "operationId": "saml_identity_provider_view", + "parameters": [ + { + "in": "path", + "name": "provider", + "description": "Name or ID of the SAML identity provider", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "silo", + "description": "Name or ID of the silo", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } ], - "summary": "Fetch Oxide service IP pool", - "operationId": "ip_pool_service_view", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPool" + "$ref": "#/components/schemas/SamlIdentityProvider" } } } @@ -5893,14 +5919,13 @@ } } }, - "/v1/system/ip-pools-service/ranges": { + "/v1/system/ip-pools": { "get": { "tags": [ "system/ip-pools" ], - "summary": "List IP ranges for the Oxide service pool", - "description": "Ranges are ordered by their first address.", - "operationId": "ip_pool_service_range_list", + "summary": "List IP pools", + "operationId": "ip_pool_list", "parameters": [ { "in": "query", @@ -5921,6 +5946,13 @@ "nullable": true, "type": "string" } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } } ], "responses": { @@ -5929,7 +5961,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolRangeResultsPage" + "$ref": "#/components/schemas/IpPoolResultsPage" } } } @@ -5944,21 +5976,18 @@ "x-dropshot-pagination": { "required": [] } - } - }, - "/v1/system/ip-pools-service/ranges/add": { + }, "post": { "tags": [ "system/ip-pools" ], - "summary": "Add IP range to Oxide service pool", - "description": "IPv6 ranges are not allowed yet.", - "operationId": "ip_pool_service_range_add", + "summary": "Create IP pool", + "operationId": "ip_pool_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpRange" + "$ref": "#/components/schemas/IpPoolCreate" } } }, @@ -5970,7 +5999,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/IpPoolRange" + "$ref": "#/components/schemas/IpPool" } } } @@ -5984,26 +6013,34 @@ } } }, - "/v1/system/ip-pools-service/ranges/remove": { - "post": { + "/v1/system/ip-pools/{pool}": { + "get": { "tags": [ "system/ip-pools" ], - "summary": "Remove IP range from Oxide service pool", - "operationId": "ip_pool_service_range_remove", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/IpRange" - } + "summary": "Fetch IP pool", + "operationId": "ip_pool_view", + "parameters": [ + { + "in": "path", + "name": "pool", + "description": "Name or ID of the IP pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" } - }, - "required": true - }, + } + ], "responses": { - "204": { - "description": "resource updated" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IpPool" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -6012,87 +6049,41 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/system/metrics/{metric_name}": { - "get": { + }, + "put": { "tags": [ - "system/metrics" + "system/ip-pools" ], - "summary": "View metrics", - "description": "View CPU, memory, or storage utilization metrics at the fleet or silo level.", - "operationId": "system_metric", + "summary": "Update IP pool", + "operationId": "ip_pool_update", "parameters": [ { "in": "path", - "name": "metric_name", + "name": "pool", + "description": "Name or ID of the IP pool", "required": true, - "schema": { - "$ref": "#/components/schemas/SystemMetricName" - } - }, - { - "in": "query", - "name": "end_time", - "description": "An exclusive end time of metrics.", - "schema": { - "type": "string", - "format": "date-time" - } - }, - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "order", - "description": "Query result order", - "schema": { - "$ref": "#/components/schemas/PaginationOrder" - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "start_time", - "description": "An inclusive start time of metrics.", - "schema": { - "type": "string", - "format": "date-time" - } - }, - { - "in": "query", - "name": "silo", - "description": "Name or ID of the silo", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IpPoolUpdate" + } + } + }, + "required": true + }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/MeasurementResultsPage" + "$ref": "#/components/schemas/IpPool" } } } @@ -6103,23 +6094,56 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [ - "end_time", - "start_time" - ] + } + }, + "delete": { + "tags": [ + "system/ip-pools" + ], + "summary": "Delete IP pool", + "operationId": "ip_pool_delete", + "parameters": [ + { + "in": "path", + "name": "pool", + "description": "Name or ID of the IP pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } } } }, - "/v1/system/networking/address-lot": { + "/v1/system/ip-pools/{pool}/ranges": { "get": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "List address lots", - "operationId": "networking_address_lot_list", + "summary": "List ranges for IP pool", + "description": "Ranges are ordered by their first address.", + "operationId": "ip_pool_range_list", "parameters": [ + { + "in": "path", + "name": "pool", + "description": "Name or ID of the IP pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "limit", @@ -6139,13 +6163,6 @@ "nullable": true, "type": "string" } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } } ], "responses": { @@ -6154,7 +6171,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AddressLotResultsPage" + "$ref": "#/components/schemas/IpPoolRangeResultsPage" } } } @@ -6169,18 +6186,32 @@ "x-dropshot-pagination": { "required": [] } - }, + } + }, + "/v1/system/ip-pools/{pool}/ranges/add": { "post": { "tags": [ - "system/networking" + "system/ip-pools" + ], + "summary": "Add range to IP pool", + "description": "IPv6 ranges are not allowed yet.", + "operationId": "ip_pool_range_add", + "parameters": [ + { + "in": "path", + "name": "pool", + "description": "Name or ID of the IP pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } ], - "summary": "Create address lot", - "operationId": "networking_address_lot_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AddressLotCreate" + "$ref": "#/components/schemas/IpRange" } } }, @@ -6192,7 +6223,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AddressLotCreateResponse" + "$ref": "#/components/schemas/IpPoolRange" } } } @@ -6206,27 +6237,37 @@ } } }, - "/v1/system/networking/address-lot/{address_lot}": { - "delete": { + "/v1/system/ip-pools/{pool}/ranges/remove": { + "post": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "Delete address lot", - "operationId": "networking_address_lot_delete", + "summary": "Remove range from IP pool", + "operationId": "ip_pool_range_remove", "parameters": [ { "in": "path", - "name": "address_lot", - "description": "Name or ID of the address lot", + "name": "pool", + "description": "Name or ID of the IP pool", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IpRange" + } + } + }, + "required": true + }, "responses": { "204": { - "description": "successful deletion" + "description": "resource updated" }, "4XX": { "$ref": "#/components/responses/Error" @@ -6237,18 +6278,18 @@ } } }, - "/v1/system/networking/address-lot/{address_lot}/blocks": { + "/v1/system/ip-pools/{pool}/silos": { "get": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "List blocks in address lot", - "operationId": "networking_address_lot_block_list", + "summary": "List IP pool's linked silos", + "operationId": "ip_pool_silo_list", "parameters": [ { "in": "path", - "name": "address_lot", - "description": "Name or ID of the address lot", + "name": "pool", + "description": "Name or ID of the IP pool", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -6288,7 +6329,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AddressLotBlockResultsPage" + "$ref": "#/components/schemas/IpPoolSiloLinkResultsPage" } } } @@ -6303,57 +6344,42 @@ "x-dropshot-pagination": { "required": [] } - } - }, - "/v1/system/networking/allow-list": { - "get": { + }, + "post": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "Get user-facing services IP allowlist", - "operationId": "networking_allow_list_view", - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/AllowList" - } - } + "summary": "Link IP pool to silo", + "description": "Users in linked silos can allocate external IPs from this pool for their instances. A silo can have at most one default pool. IPs are allocated from the default pool when users ask for one without specifying a pool.", + "operationId": "ip_pool_silo_link", + "parameters": [ + { + "in": "path", + "name": "pool", + "description": "Name or ID of the IP pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" } - } - }, - "put": { - "tags": [ - "system/networking" ], - "summary": "Update user-facing services IP allowlist", - "operationId": "networking_allow_list_update", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AllowListUpdate" + "$ref": "#/components/schemas/IpPoolLinkSilo" } } }, "required": true }, "responses": { - "200": { - "description": "successful operation", + "201": { + "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AllowList" + "$ref": "#/components/schemas/IpPoolSiloLink" } } } @@ -6367,23 +6393,86 @@ } } }, - "/v1/system/networking/bfd-disable": { - "post": { + "/v1/system/ip-pools/{pool}/silos/{silo}": { + "put": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "Disable a BFD session", - "operationId": "networking_bfd_disable", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BfdSessionDisable" + "summary": "Make IP pool default for silo", + "description": "When a user asks for an IP (e.g., at instance create time) without specifying a pool, the IP comes from the default pool if a default is configured. When a pool is made the default for a silo, any existing default will remain linked to the silo, but will no longer be the default.", + "operationId": "ip_pool_silo_update", + "parameters": [ + { + "in": "path", + "name": "pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "silo", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IpPoolSiloUpdate" } } }, "required": true }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IpPoolSiloLink" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "system/ip-pools" + ], + "summary": "Unlink IP pool from silo", + "description": "Will fail if there are any outstanding IPs allocated in the silo.", + "operationId": "ip_pool_silo_unlink", + "parameters": [ + { + "in": "path", + "name": "pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "path", + "name": "silo", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], "responses": { "204": { "description": "resource updated" @@ -6397,26 +6486,34 @@ } } }, - "/v1/system/networking/bfd-enable": { - "post": { + "/v1/system/ip-pools/{pool}/utilization": { + "get": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "Enable a BFD session", - "operationId": "networking_bfd_enable", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/BfdSessionEnable" - } + "summary": "Fetch IP pool utilization", + "operationId": "ip_pool_utilization_view", + "parameters": [ + { + "in": "path", + "name": "pool", + "description": "Name or ID of the IP pool", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" } - }, - "required": true - }, + } + ], "responses": { - "204": { - "description": "resource updated" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IpPoolUtilization" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -6427,24 +6524,20 @@ } } }, - "/v1/system/networking/bfd-status": { + "/v1/system/ip-pools-service": { "get": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "Get BFD status", - "operationId": "networking_bfd_status", + "summary": "Fetch Oxide service IP pool", + "operationId": "ip_pool_service_view", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "title": "Array_of_BfdStatus", - "type": "array", - "items": { - "$ref": "#/components/schemas/BfdStatus" - } + "$ref": "#/components/schemas/IpPool" } } } @@ -6458,13 +6551,14 @@ } } }, - "/v1/system/networking/bgp": { + "/v1/system/ip-pools-service/ranges": { "get": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "List BGP configurations", - "operationId": "networking_bgp_config_list", + "summary": "List IP ranges for the Oxide service pool", + "description": "Ranges are ordered by their first address.", + "operationId": "ip_pool_service_range_list", "parameters": [ { "in": "query", @@ -6485,13 +6579,6 @@ "nullable": true, "type": "string" } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } } ], "responses": { @@ -6500,7 +6587,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BgpConfigResultsPage" + "$ref": "#/components/schemas/IpPoolRangeResultsPage" } } } @@ -6515,18 +6602,21 @@ "x-dropshot-pagination": { "required": [] } - }, + } + }, + "/v1/system/ip-pools-service/ranges/add": { "post": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "Create new BGP configuration", - "operationId": "networking_bgp_config_create", + "summary": "Add IP range to Oxide service pool", + "description": "IPv6 ranges are not allowed yet.", + "operationId": "ip_pool_service_range_add", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BgpConfigCreate" + "$ref": "#/components/schemas/IpRange" } } }, @@ -6538,7 +6628,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BgpConfig" + "$ref": "#/components/schemas/IpPoolRange" } } } @@ -6550,24 +6640,25 @@ "$ref": "#/components/responses/Error" } } - }, - "delete": { + } + }, + "/v1/system/ip-pools-service/ranges/remove": { + "post": { "tags": [ - "system/networking" + "system/ip-pools" ], - "summary": "Delete BGP configuration", - "operationId": "networking_bgp_config_delete", - "parameters": [ - { - "in": "query", - "name": "name_or_id", - "description": "A name or id to use when selecting BGP config.", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" + "summary": "Remove IP range from Oxide service pool", + "operationId": "ip_pool_service_range_remove", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/IpRange" + } } - } - ], + }, + "required": true + }, "responses": { "204": { "description": "resource updated" @@ -6581,14 +6672,32 @@ } } }, - "/v1/system/networking/bgp-announce-set": { + "/v1/system/metrics/{metric_name}": { "get": { "tags": [ - "system/networking" + "system/metrics" ], - "summary": "List BGP announce sets", - "operationId": "networking_bgp_announce_set_list", + "summary": "View metrics", + "description": "View CPU, memory, or storage utilization metrics at the fleet or silo level.", + "operationId": "system_metric", "parameters": [ + { + "in": "path", + "name": "metric_name", + "required": true, + "schema": { + "$ref": "#/components/schemas/SystemMetricName" + } + }, + { + "in": "query", + "name": "end_time", + "description": "An exclusive end time of metrics.", + "schema": { + "type": "string", + "format": "date-time" + } + }, { "in": "query", "name": "limit", @@ -6600,6 +6709,14 @@ "minimum": 1 } }, + { + "in": "query", + "name": "order", + "description": "Query result order", + "schema": { + "$ref": "#/components/schemas/PaginationOrder" + } + }, { "in": "query", "name": "page_token", @@ -6611,9 +6728,19 @@ }, { "in": "query", - "name": "sort_by", + "name": "start_time", + "description": "An inclusive start time of metrics.", "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" + "type": "string", + "format": "date-time" + } + }, + { + "in": "query", + "name": "silo", + "description": "Name or ID of the silo", + "schema": { + "$ref": "#/components/schemas/NameOrId" } } ], @@ -6623,11 +6750,7 @@ "content": { "application/json": { "schema": { - "title": "Array_of_BgpAnnounceSet", - "type": "array", - "items": { - "$ref": "#/components/schemas/BgpAnnounceSet" - } + "$ref": "#/components/schemas/MeasurementResultsPage" } } } @@ -6640,21 +6763,82 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "end_time", + "start_time" + ] } - }, - "put": { + } + }, + "/v1/system/networking/address-lot": { + "get": { "tags": [ "system/networking" ], - "summary": "Update BGP announce set", - "description": "If the announce set exists, this endpoint replaces the existing announce set with the one specified.", - "operationId": "networking_bgp_announce_set_update", + "summary": "List address lots", + "operationId": "networking_address_lot_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AddressLotResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + }, + "post": { + "tags": [ + "system/networking" + ], + "summary": "Create address lot", + "operationId": "networking_address_lot_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BgpAnnounceSetCreate" + "$ref": "#/components/schemas/AddressLotCreate" } } }, @@ -6666,7 +6850,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BgpAnnounceSet" + "$ref": "#/components/schemas/AddressLotCreateResponse" } } } @@ -6680,18 +6864,18 @@ } } }, - "/v1/system/networking/bgp-announce-set/{announce_set}": { + "/v1/system/networking/address-lot/{address_lot}": { "delete": { "tags": [ "system/networking" ], - "summary": "Delete BGP announce set", - "operationId": "networking_bgp_announce_set_delete", + "summary": "Delete address lot", + "operationId": "networking_address_lot_delete", "parameters": [ { "in": "path", - "name": "announce_set", - "description": "Name or ID of the announce set", + "name": "address_lot", + "description": "Name or ID of the address lot", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -6700,7 +6884,7 @@ ], "responses": { "204": { - "description": "resource updated" + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -6711,22 +6895,49 @@ } } }, - "/v1/system/networking/bgp-announce-set/{announce_set}/announcement": { + "/v1/system/networking/address-lot/{address_lot}/blocks": { "get": { "tags": [ "system/networking" ], - "summary": "Get originated routes for a specified BGP announce set", - "operationId": "networking_bgp_announcement_list", + "summary": "List blocks in address lot", + "operationId": "networking_address_lot_block_list", "parameters": [ { "in": "path", - "name": "announce_set", - "description": "Name or ID of the announce set", + "name": "address_lot", + "description": "Name or ID of the address lot", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" + } } ], "responses": { @@ -6735,11 +6946,7 @@ "content": { "application/json": { "schema": { - "title": "Array_of_BgpAnnouncement", - "type": "array", - "items": { - "$ref": "#/components/schemas/BgpAnnouncement" - } + "$ref": "#/components/schemas/AddressLotBlockResultsPage" } } } @@ -6750,23 +6957,26 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } } }, - "/v1/system/networking/bgp-exported": { + "/v1/system/networking/allow-list": { "get": { "tags": [ "system/networking" ], - "summary": "Get BGP exported routes", - "operationId": "networking_bgp_exported", + "summary": "Get user-facing services IP allowlist", + "operationId": "networking_allow_list_view", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/BgpExported" + "$ref": "#/components/schemas/AllowList" } } } @@ -6778,35 +6988,30 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/system/networking/bgp-message-history": { - "get": { + }, + "put": { "tags": [ "system/networking" ], - "summary": "Get BGP router message history", - "operationId": "networking_bgp_message_history", - "parameters": [ - { - "in": "query", - "name": "asn", - "description": "The ASN to filter on. Required.", - "required": true, - "schema": { - "type": "integer", - "format": "uint32", - "minimum": 0 + "summary": "Update user-facing services IP allowlist", + "operationId": "networking_allow_list_update", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/AllowListUpdate" + } } - } - ], + }, + "required": true + }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/AggregateBgpMessageHistory" + "$ref": "#/components/schemas/AllowList" } } } @@ -6820,41 +7025,57 @@ } } }, - "/v1/system/networking/bgp-routes-ipv4": { - "get": { + "/v1/system/networking/bfd-disable": { + "post": { "tags": [ "system/networking" ], - "summary": "Get imported IPv4 BGP routes", - "operationId": "networking_bgp_imported_routes_ipv4", - "parameters": [ - { - "in": "query", - "name": "asn", - "description": "The ASN to filter on. Required.", - "required": true, - "schema": { - "type": "integer", - "format": "uint32", - "minimum": 0 + "summary": "Disable a BFD session", + "operationId": "networking_bfd_disable", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BfdSessionDisable" + } } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" } + } + } + }, + "/v1/system/networking/bfd-enable": { + "post": { + "tags": [ + "system/networking" ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "title": "Array_of_BgpImportedRouteIpv4", - "type": "array", - "items": { - "$ref": "#/components/schemas/BgpImportedRouteIpv4" - } - } + "summary": "Enable a BFD session", + "operationId": "networking_bfd_enable", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/BfdSessionEnable" } } }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, "4XX": { "$ref": "#/components/responses/Error" }, @@ -6864,23 +7085,23 @@ } } }, - "/v1/system/networking/bgp-status": { + "/v1/system/networking/bfd-status": { "get": { "tags": [ "system/networking" ], - "summary": "Get BGP peer status", - "operationId": "networking_bgp_status", + "summary": "Get BFD status", + "operationId": "networking_bfd_status", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "title": "Array_of_BgpPeerStatus", + "title": "Array_of_BfdStatus", "type": "array", "items": { - "$ref": "#/components/schemas/BgpPeerStatus" + "$ref": "#/components/schemas/BfdStatus" } } } @@ -6895,13 +7116,13 @@ } } }, - "/v1/system/networking/loopback-address": { + "/v1/system/networking/bgp": { "get": { "tags": [ "system/networking" ], - "summary": "List loopback addresses", - "operationId": "networking_loopback_address_list", + "summary": "List BGP configurations", + "operationId": "networking_bgp_config_list", "parameters": [ { "in": "query", @@ -6927,7 +7148,7 @@ "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/IdSortMode" + "$ref": "#/components/schemas/NameOrIdSortMode" } } ], @@ -6937,7 +7158,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/LoopbackAddressResultsPage" + "$ref": "#/components/schemas/BgpConfigResultsPage" } } } @@ -6957,13 +7178,13 @@ "tags": [ "system/networking" ], - "summary": "Create loopback address", - "operationId": "networking_loopback_address_create", + "summary": "Create new BGP configuration", + "operationId": "networking_bgp_config_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/LoopbackAddressCreate" + "$ref": "#/components/schemas/BgpConfigCreate" } } }, @@ -6975,7 +7196,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/LoopbackAddress" + "$ref": "#/components/schemas/BgpConfig" } } } @@ -6987,60 +7208,27 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask}": { + }, "delete": { "tags": [ "system/networking" ], - "summary": "Delete loopback address", - "operationId": "networking_loopback_address_delete", + "summary": "Delete BGP configuration", + "operationId": "networking_bgp_config_delete", "parameters": [ { - "in": "path", - "name": "address", - "description": "The IP address and subnet mask to use when selecting the loopback address.", - "required": true, - "schema": { - "type": "string", - "format": "ip" - } - }, - { - "in": "path", - "name": "rack_id", - "description": "The rack to use when selecting the loopback address.", - "required": true, - "schema": { - "type": "string", - "format": "uuid" - } - }, - { - "in": "path", - "name": "subnet_mask", - "description": "The IP address and subnet mask to use when selecting the loopback address.", - "required": true, - "schema": { - "type": "integer", - "format": "uint8", - "minimum": 0 - } - }, - { - "in": "path", - "name": "switch_location", - "description": "The switch location to use when selecting the loopback address.", + "in": "query", + "name": "name_or_id", + "description": "A name or id to use when selecting BGP config.", "required": true, "schema": { - "$ref": "#/components/schemas/Name" + "$ref": "#/components/schemas/NameOrId" } } ], "responses": { "204": { - "description": "successful deletion" + "description": "resource updated" }, "4XX": { "$ref": "#/components/responses/Error" @@ -7051,13 +7239,13 @@ } } }, - "/v1/system/networking/switch-port-settings": { + "/v1/system/networking/bgp-announce-set": { "get": { "tags": [ "system/networking" ], - "summary": "List switch port settings", - "operationId": "networking_switch_port_settings_list", + "summary": "List BGP announce sets", + "operationId": "networking_bgp_announce_set_list", "parameters": [ { "in": "query", @@ -7079,14 +7267,6 @@ "type": "string" } }, - { - "in": "query", - "name": "port_settings", - "description": "An optional name or id to use when selecting port settings.", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "sort_by", @@ -7101,7 +7281,11 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SwitchPortSettingsResultsPage" + "title": "Array_of_BgpAnnounceSet", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpAnnounceSet" + } } } } @@ -7117,17 +7301,18 @@ "required": [] } }, - "post": { + "put": { "tags": [ "system/networking" ], - "summary": "Create switch port settings", - "operationId": "networking_switch_port_settings_create", + "summary": "Update BGP announce set", + "description": "If the announce set exists, this endpoint replaces the existing announce set with the one specified.", + "operationId": "networking_bgp_announce_set_update", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SwitchPortSettingsCreate" + "$ref": "#/components/schemas/BgpAnnounceSetCreate" } } }, @@ -7139,7 +7324,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SwitchPortSettingsView" + "$ref": "#/components/schemas/BgpAnnounceSet" } } } @@ -7151,18 +7336,21 @@ "$ref": "#/components/responses/Error" } } - }, + } + }, + "/v1/system/networking/bgp-announce-set/{announce_set}": { "delete": { "tags": [ "system/networking" ], - "summary": "Delete switch port settings", - "operationId": "networking_switch_port_settings_delete", + "summary": "Delete BGP announce set", + "operationId": "networking_bgp_announce_set_delete", "parameters": [ { - "in": "query", - "name": "port_settings", - "description": "An optional name or id to use when selecting port settings.", + "in": "path", + "name": "announce_set", + "description": "Name or ID of the announce set", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -7170,7 +7358,7 @@ ], "responses": { "204": { - "description": "successful deletion" + "description": "resource updated" }, "4XX": { "$ref": "#/components/responses/Error" @@ -7181,18 +7369,18 @@ } } }, - "/v1/system/networking/switch-port-settings/{port}": { + "/v1/system/networking/bgp-announce-set/{announce_set}/announcement": { "get": { "tags": [ "system/networking" ], - "summary": "Get information about switch port", - "operationId": "networking_switch_port_settings_view", + "summary": "Get originated routes for a specified BGP announce set", + "operationId": "networking_bgp_announcement_list", "parameters": [ { "in": "path", - "name": "port", - "description": "A name or id to use when selecting switch port settings info objects.", + "name": "announce_set", + "description": "Name or ID of the announce set", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -7205,7 +7393,11 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SwitchPortSettingsView" + "title": "Array_of_BgpAnnouncement", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpAnnouncement" + } } } } @@ -7219,20 +7411,20 @@ } } }, - "/v1/system/policy": { + "/v1/system/networking/bgp-exported": { "get": { "tags": [ - "policy" + "system/networking" ], - "summary": "Fetch top-level IAM policy", - "operationId": "system_policy_view", + "summary": "Get BGP exported routes", + "operationId": "networking_bgp_exported", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FleetRolePolicy" + "$ref": "#/components/schemas/BgpExported" } } } @@ -7244,30 +7436,35 @@ "$ref": "#/components/responses/Error" } } - }, - "put": { + } + }, + "/v1/system/networking/bgp-message-history": { + "get": { "tags": [ - "policy" + "system/networking" ], - "summary": "Update top-level IAM policy", - "operationId": "system_policy_update", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/FleetRolePolicy" - } + "summary": "Get BGP router message history", + "operationId": "networking_bgp_message_history", + "parameters": [ + { + "in": "query", + "name": "asn", + "description": "The ASN to filter on. Required.", + "required": true, + "schema": { + "type": "integer", + "format": "uint32", + "minimum": 0 } - }, - "required": true - }, + } + ], "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/FleetRolePolicy" + "$ref": "#/components/schemas/AggregateBgpMessageHistory" } } } @@ -7281,32 +7478,23 @@ } } }, - "/v1/system/roles": { + "/v1/system/networking/bgp-routes-ipv4": { "get": { "tags": [ - "roles" + "system/networking" ], - "summary": "List built-in roles", - "operationId": "role_list", + "summary": "Get imported IPv4 BGP routes", + "operationId": "networking_bgp_imported_routes_ipv4", "parameters": [ { "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", + "name": "asn", + "description": "The ASN to filter on. Required.", + "required": true, "schema": { - "nullable": true, "type": "integer", "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" + "minimum": 0 } } ], @@ -7316,7 +7504,11 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RoleResultsPage" + "title": "Array_of_BgpImportedRouteIpv4", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpImportedRouteIpv4" + } } } } @@ -7327,37 +7519,27 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } } }, - "/v1/system/roles/{role_name}": { + "/v1/system/networking/bgp-status": { "get": { "tags": [ - "roles" - ], - "summary": "Fetch built-in role", - "operationId": "role_view", - "parameters": [ - { - "in": "path", - "name": "role_name", - "description": "The built-in role's unique name.", - "required": true, - "schema": { - "type": "string" - } - } + "system/networking" ], + "summary": "Get BGP peer status", + "operationId": "networking_bgp_status", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Role" + "title": "Array_of_BgpPeerStatus", + "type": "array", + "items": { + "$ref": "#/components/schemas/BgpPeerStatus" + } } } } @@ -7371,13 +7553,13 @@ } } }, - "/v1/system/silo-quotas": { + "/v1/system/networking/loopback-address": { "get": { "tags": [ - "system/silos" + "system/networking" ], - "summary": "Lists resource quotas for all silos", - "operationId": "system_quotas_list", + "summary": "List loopback addresses", + "operationId": "networking_loopback_address_list", "parameters": [ { "in": "query", @@ -7413,7 +7595,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloQuotasResultsPage" + "$ref": "#/components/schemas/LoopbackAddressResultsPage" } } } @@ -7428,78 +7610,18 @@ "x-dropshot-pagination": { "required": [] } - } - }, - "/v1/system/silos": { - "get": { + }, + "post": { "tags": [ - "system/silos" + "system/networking" ], - "summary": "List silos", - "description": "Lists silos that are discoverable based on the current permissions.", - "operationId": "silo_list", - "parameters": [ - { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", - "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 - } - }, - { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", - "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SiloResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [] - } - }, - "post": { - "tags": [ - "system/silos" - ], - "summary": "Create a silo", - "operationId": "silo_create", + "summary": "Create loopback address", + "operationId": "networking_loopback_address_create", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloCreate" + "$ref": "#/components/schemas/LoopbackAddressCreate" } } }, @@ -7511,7 +7633,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Silo" + "$ref": "#/components/schemas/LoopbackAddress" } } } @@ -7525,59 +7647,52 @@ } } }, - "/v1/system/silos/{silo}": { - "get": { + "/v1/system/networking/loopback-address/{rack_id}/{switch_location}/{address}/{subnet_mask}": { + "delete": { "tags": [ - "system/silos" + "system/networking" ], - "summary": "Fetch silo", - "description": "Fetch silo by name or ID.", - "operationId": "silo_view", + "summary": "Delete loopback address", + "operationId": "networking_loopback_address_delete", "parameters": [ { "in": "path", - "name": "silo", - "description": "Name or ID of the silo", + "name": "address", + "description": "The IP address and subnet mask to use when selecting the loopback address.", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "type": "string", + "format": "ip" } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/Silo" - } - } + }, + { + "in": "path", + "name": "rack_id", + "description": "The rack to use when selecting the loopback address.", + "required": true, + "schema": { + "type": "string", + "format": "uuid" } }, - "4XX": { - "$ref": "#/components/responses/Error" + { + "in": "path", + "name": "subnet_mask", + "description": "The IP address and subnet mask to use when selecting the loopback address.", + "required": true, + "schema": { + "type": "integer", + "format": "uint8", + "minimum": 0 + } }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, - "delete": { - "tags": [ - "system/silos" - ], - "summary": "Delete a silo", - "description": "Delete a silo by name or ID.", - "operationId": "silo_delete", - "parameters": [ { "in": "path", - "name": "silo", - "description": "Name or ID of the silo", + "name": "switch_location", + "description": "The switch location to use when selecting the loopback address.", "required": true, "schema": { - "$ref": "#/components/schemas/NameOrId" + "$ref": "#/components/schemas/Name" } } ], @@ -7594,24 +7709,14 @@ } } }, - "/v1/system/silos/{silo}/ip-pools": { + "/v1/system/networking/switch-port-settings": { "get": { "tags": [ - "system/silos" + "system/networking" ], - "summary": "List IP pools linked to silo", - "description": "Linked IP pools are available to users in the specified silo. A silo can have at most one default pool. IPs are allocated from the default pool when users ask for one without specifying a pool.", - "operationId": "silo_ip_pool_list", + "summary": "List switch port settings", + "operationId": "networking_switch_port_settings_list", "parameters": [ - { - "in": "path", - "name": "silo", - "description": "Name or ID of the silo", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "limit", @@ -7632,6 +7737,14 @@ "type": "string" } }, + { + "in": "query", + "name": "port_settings", + "description": "An optional name or id to use when selecting port settings.", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", @@ -7646,7 +7759,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloIpPoolResultsPage" + "$ref": "#/components/schemas/SwitchPortSettingsResultsPage" } } } @@ -7661,33 +7774,30 @@ "x-dropshot-pagination": { "required": [] } - } - }, - "/v1/system/silos/{silo}/policy": { - "get": { + }, + "post": { "tags": [ - "system/silos" + "system/networking" ], - "summary": "Fetch silo IAM policy", - "operationId": "silo_policy_view", - "parameters": [ - { - "in": "path", - "name": "silo", - "description": "Name or ID of the silo", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" + "summary": "Create switch port settings", + "operationId": "networking_switch_port_settings_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SwitchPortSettingsCreate" + } } - } - ], + }, + "required": true + }, "responses": { - "200": { - "description": "successful operation", + "201": { + "description": "successful creation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloRolePolicy" + "$ref": "#/components/schemas/SwitchPortSettingsView" } } } @@ -7700,43 +7810,25 @@ } } }, - "put": { + "delete": { "tags": [ - "system/silos" + "system/networking" ], - "summary": "Update silo IAM policy", - "operationId": "silo_policy_update", + "summary": "Delete switch port settings", + "operationId": "networking_switch_port_settings_delete", "parameters": [ { - "in": "path", - "name": "silo", - "description": "Name or ID of the silo", - "required": true, + "in": "query", + "name": "port_settings", + "description": "An optional name or id to use when selecting port settings.", "schema": { "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SiloRolePolicy" - } - } - }, - "required": true - }, "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/SiloRolePolicy" - } - } - } + "204": { + "description": "successful deletion" }, "4XX": { "$ref": "#/components/responses/Error" @@ -7747,18 +7839,18 @@ } } }, - "/v1/system/silos/{silo}/quotas": { + "/v1/system/networking/switch-port-settings/{port}": { "get": { "tags": [ - "system/silos" + "system/networking" ], - "summary": "Fetch resource quotas for silo", - "operationId": "silo_quotas_view", + "summary": "Get information about switch port", + "operationId": "networking_switch_port_settings_view", "parameters": [ { "in": "path", - "name": "silo", - "description": "Name or ID of the silo", + "name": "port", + "description": "A name or id to use when selecting switch port settings info objects.", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -7771,7 +7863,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloQuotas" + "$ref": "#/components/schemas/SwitchPortSettingsView" } } } @@ -7783,30 +7875,45 @@ "$ref": "#/components/responses/Error" } } - }, - "put": { + } + }, + "/v1/system/policy": { + "get": { "tags": [ - "system/silos" - ], - "summary": "Update resource quotas for silo", - "description": "If a quota value is not specified, it will remain unchanged.", - "operationId": "silo_quotas_update", - "parameters": [ - { - "in": "path", - "name": "silo", - "description": "Name or ID of the silo", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - } + "policy" ], - "requestBody": { - "content": { + "summary": "Fetch top-level IAM policy", + "operationId": "system_policy_view", + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/FleetRolePolicy" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "policy" + ], + "summary": "Update top-level IAM policy", + "operationId": "system_policy_update", + "requestBody": { + "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloQuotasUpdate" + "$ref": "#/components/schemas/FleetRolePolicy" } } }, @@ -7818,7 +7925,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloQuotas" + "$ref": "#/components/schemas/FleetRolePolicy" } } } @@ -7832,13 +7939,13 @@ } } }, - "/v1/system/users": { + "/v1/system/roles": { "get": { "tags": [ - "system/silos" + "roles" ], - "summary": "List built-in (system) users in silo", - "operationId": "silo_user_list", + "summary": "List built-in roles", + "operationId": "role_list", "parameters": [ { "in": "query", @@ -7859,20 +7966,46 @@ "nullable": true, "type": "string" } - }, - { - "in": "query", - "name": "silo", - "description": "Name or ID of the silo", - "schema": { - "$ref": "#/components/schemas/NameOrId" + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RoleResultsPage" + } + } } }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/system/roles/{role_name}": { + "get": { + "tags": [ + "roles" + ], + "summary": "Fetch built-in role", + "operationId": "role_view", + "parameters": [ { - "in": "query", - "name": "sort_by", + "in": "path", + "name": "role_name", + "description": "The built-in role's unique name.", + "required": true, "schema": { - "$ref": "#/components/schemas/IdSortMode" + "type": "string" } } ], @@ -7882,7 +8015,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UserResultsPage" + "$ref": "#/components/schemas/Role" } } } @@ -7893,39 +8026,42 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [ - "silo" - ] } } }, - "/v1/system/users/{user_id}": { + "/v1/system/silo-quotas": { "get": { "tags": [ "system/silos" ], - "summary": "Fetch built-in (system) user", - "operationId": "silo_user_view", + "summary": "Lists resource quotas for all silos", + "operationId": "system_quotas_list", "parameters": [ { - "in": "path", - "name": "user_id", - "description": "The user's internal ID", - "required": true, + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "type": "string", - "format": "uuid" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 } }, { "in": "query", - "name": "silo", - "description": "Name or ID of the silo", - "required": true, + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/IdSortMode" } } ], @@ -7935,7 +8071,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/User" + "$ref": "#/components/schemas/SiloQuotasResultsPage" } } } @@ -7946,16 +8082,20 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } } }, - "/v1/system/users-builtin": { + "/v1/system/silos": { "get": { "tags": [ "system/silos" ], - "summary": "List built-in users", - "operationId": "user_builtin_list", + "summary": "List silos", + "description": "Lists silos that are discoverable based on the current permissions.", + "operationId": "silo_list", "parameters": [ { "in": "query", @@ -7981,7 +8121,7 @@ "in": "query", "name": "sort_by", "schema": { - "$ref": "#/components/schemas/NameSortMode" + "$ref": "#/components/schemas/NameOrIdSortMode" } } ], @@ -7991,7 +8131,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UserBuiltinResultsPage" + "$ref": "#/components/schemas/SiloResultsPage" } } } @@ -8006,19 +8146,56 @@ "x-dropshot-pagination": { "required": [] } + }, + "post": { + "tags": [ + "system/silos" + ], + "summary": "Create a silo", + "operationId": "silo_create", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SiloCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Silo" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } } }, - "/v1/system/users-builtin/{user}": { + "/v1/system/silos/{silo}": { "get": { "tags": [ "system/silos" ], - "summary": "Fetch built-in user", - "operationId": "user_builtin_view", + "summary": "Fetch silo", + "description": "Fetch silo by name or ID.", + "operationId": "silo_view", "parameters": [ { "in": "path", - "name": "user", + "name": "silo", + "description": "Name or ID of the silo", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8031,7 +8208,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/UserBuiltin" + "$ref": "#/components/schemas/Silo" } } } @@ -8043,16 +8220,56 @@ "$ref": "#/components/responses/Error" } } + }, + "delete": { + "tags": [ + "system/silos" + ], + "summary": "Delete a silo", + "description": "Delete a silo by name or ID.", + "operationId": "silo_delete", + "parameters": [ + { + "in": "path", + "name": "silo", + "description": "Name or ID of the silo", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } } }, - "/v1/system/utilization/silos": { + "/v1/system/silos/{silo}/ip-pools": { "get": { "tags": [ "system/silos" ], - "summary": "List current utilization state for all silos", - "operationId": "silo_utilization_list", + "summary": "List IP pools linked to silo", + "description": "Linked IP pools are available to users in the specified silo. A silo can have at most one default pool. IPs are allocated from the default pool when users ask for one without specifying a pool.", + "operationId": "silo_ip_pool_list", "parameters": [ + { + "in": "path", + "name": "silo", + "description": "Name or ID of the silo", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "limit", @@ -8087,7 +8304,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloUtilizationResultsPage" + "$ref": "#/components/schemas/SiloIpPoolResultsPage" } } } @@ -8104,13 +8321,13 @@ } } }, - "/v1/system/utilization/silos/{silo}": { + "/v1/system/silos/{silo}/policy": { "get": { "tags": [ "system/silos" ], - "summary": "Fetch current utilization for given silo", - "operationId": "silo_utilization_view", + "summary": "Fetch silo IAM policy", + "operationId": "silo_policy_view", "parameters": [ { "in": "path", @@ -8128,7 +8345,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/SiloUtilization" + "$ref": "#/components/schemas/SiloRolePolicy" } } } @@ -8140,21 +8357,29 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/timeseries/query": { - "post": { + }, + "put": { "tags": [ - "metrics" + "system/silos" + ], + "summary": "Update silo IAM policy", + "operationId": "silo_policy_update", + "parameters": [ + { + "in": "path", + "name": "silo", + "description": "Name or ID of the silo", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } ], - "summary": "Run timeseries query", - "description": "Queries are written in OxQL.", - "operationId": "timeseries_query", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TimeseriesQuery" + "$ref": "#/components/schemas/SiloRolePolicy" } } }, @@ -8166,7 +8391,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/OxqlQueryResult" + "$ref": "#/components/schemas/SiloRolePolicy" } } } @@ -8180,42 +8405,78 @@ } } }, - "/v1/timeseries/schema": { + "/v1/system/silos/{silo}/quotas": { "get": { "tags": [ - "metrics" + "system/silos" ], - "summary": "List timeseries schemas", - "operationId": "timeseries_schema_list", + "summary": "Fetch resource quotas for silo", + "operationId": "silo_quotas_view", "parameters": [ { - "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", + "in": "path", + "name": "silo", + "description": "Name or ID of the silo", + "required": true, "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SiloQuotas" + } + } } }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "system/silos" + ], + "summary": "Update resource quotas for silo", + "description": "If a quota value is not specified, it will remain unchanged.", + "operationId": "silo_quotas_update", + "parameters": [ { - "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "in": "path", + "name": "silo", + "description": "Name or ID of the silo", + "required": true, "schema": { - "nullable": true, - "type": "string" + "$ref": "#/components/schemas/NameOrId" } } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SiloQuotasUpdate" + } + } + }, + "required": true + }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/TimeseriesSchemaResultsPage" + "$ref": "#/components/schemas/SiloQuotas" } } } @@ -8226,29 +8487,17 @@ "5XX": { "$ref": "#/components/responses/Error" } - }, - "x-dropshot-pagination": { - "required": [] } } }, - "/v1/users": { + "/v1/system/users": { "get": { "tags": [ - "silos" + "system/silos" ], - "summary": "List users", - "operationId": "user_list", + "summary": "List built-in (system) users in silo", + "operationId": "silo_user_list", "parameters": [ - { - "in": "query", - "name": "group", - "schema": { - "nullable": true, - "type": "string", - "format": "uuid" - } - }, { "in": "query", "name": "limit", @@ -8269,6 +8518,14 @@ "type": "string" } }, + { + "in": "query", + "name": "silo", + "description": "Name or ID of the silo", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", @@ -8296,24 +8553,47 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "silo" + ] } } }, - "/v1/utilization": { + "/v1/system/users/{user_id}": { "get": { "tags": [ - "silos" + "system/silos" + ], + "summary": "Fetch built-in (system) user", + "operationId": "silo_user_view", + "parameters": [ + { + "in": "path", + "name": "user_id", + "description": "The user's internal ID", + "required": true, + "schema": { + "type": "string", + "format": "uuid" + } + }, + { + "in": "query", + "name": "silo", + "description": "Name or ID of the silo", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } ], - "summary": "Fetch resource utilization for user's current silo", - "operationId": "utilization_view", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Utilization" + "$ref": "#/components/schemas/User" } } } @@ -8327,29 +8607,39 @@ } } }, - "/v1/vpc-firewall-rules": { + "/v1/system/users-builtin": { "get": { "tags": [ - "vpcs" + "system/silos" ], - "summary": "List firewall rules", - "operationId": "vpc_firewall_rules_view", + "summary": "List built-in users", + "operationId": "user_builtin_list", "parameters": [ { "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 } }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", - "required": true, + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameSortMode" } } ], @@ -8359,7 +8649,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcFirewallRules" + "$ref": "#/components/schemas/UserBuiltinResultsPage" } } } @@ -8370,51 +8660,36 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } - }, - "put": { + } + }, + "/v1/system/users-builtin/{user}": { + "get": { "tags": [ - "vpcs" + "system/silos" ], - "summary": "Replace firewall rules", - "description": "The maximum number of rules per VPC is 1024.\n\nTargets are used to specify the set of instances to which a firewall rule applies. You can target instances directly by name, or specify a VPC, VPC subnet, IP, or IP subnet, which will apply the rule to traffic going to all matching instances. Targets are additive: the rule applies to instances matching ANY target. The maximum number of targets is 256.\n\nFilters reduce the scope of a firewall rule. Without filters, the rule applies to all packets to the targets (or from the targets, if it's an outbound rule). With multiple filters, the rule applies only to packets matching ALL filters. The maximum number of each type of filter is 256.", - "operationId": "vpc_firewall_rules_update", + "summary": "Fetch built-in user", + "operationId": "user_builtin_view", "parameters": [ { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "in": "path", + "name": "user", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcFirewallRuleUpdateParams" - } - } - }, - "required": true - }, "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcFirewallRules" + "$ref": "#/components/schemas/UserBuiltin" } } } @@ -8428,14 +8703,13 @@ } } }, - "/v1/vpc-router-routes": { + "/v1/system/utilization/silos": { "get": { "tags": [ - "vpcs" + "system/silos" ], - "summary": "List routes", - "description": "List the routes associated with a router in a particular VPC.", - "operationId": "vpc_router_route_list", + "summary": "List current utilization state for all silos", + "operationId": "silo_utilization_list", "parameters": [ { "in": "query", @@ -8457,36 +8731,12 @@ "type": "string" } }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "sort_by", "schema": { "$ref": "#/components/schemas/NameOrIdSortMode" } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } } ], "responses": { @@ -8495,7 +8745,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteResultsPage" + "$ref": "#/components/schemas/SiloUtilizationResultsPage" } } } @@ -8508,61 +8758,73 @@ } }, "x-dropshot-pagination": { - "required": [ - "router" - ] + "required": [] } - }, - "post": { + } + }, + "/v1/system/utilization/silos/{silo}": { + "get": { "tags": [ - "vpcs" + "system/silos" ], - "summary": "Create route", - "operationId": "vpc_router_route_create", + "summary": "Fetch current utilization for given silo", + "operationId": "silo_utilization_view", "parameters": [ { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", + "in": "path", + "name": "silo", + "description": "Name or ID of the silo", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } - }, - { - "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SiloUtilization" + } + } } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" } + } + } + }, + "/v1/timeseries/query": { + "post": { + "tags": [ + "metrics" ], + "summary": "Run timeseries query", + "description": "Queries are written in OxQL.", + "operationId": "timeseries_query", "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRouteCreate" + "$ref": "#/components/schemas/TimeseriesQuery" } } }, "required": true }, "responses": { - "201": { - "description": "successful creation", + "200": { + "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/OxqlQueryResult" } } } @@ -8576,46 +8838,32 @@ } } }, - "/v1/vpc-router-routes/{route}": { + "/v1/timeseries/schema": { "get": { "tags": [ - "vpcs" + "metrics" ], - "summary": "Fetch route", - "operationId": "vpc_router_route_view", + "summary": "List timeseries schemas", + "operationId": "timeseries_schema_list", "parameters": [ - { - "in": "path", - "name": "route", - "description": "Name or ID of the route", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", - "name": "router", - "description": "Name or ID of the router", - "required": true, + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 } }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "string" } } ], @@ -8625,7 +8873,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/TimeseriesSchemaResultsPage" } } } @@ -8636,66 +8884,94 @@ "5XX": { "$ref": "#/components/responses/Error" } + }, + "x-dropshot-pagination": { + "required": [] } - }, - "put": { + } + }, + "/v1/users": { + "get": { "tags": [ - "vpcs" + "silos" ], - "summary": "Update route", - "operationId": "vpc_router_route_update", + "summary": "List users", + "operationId": "user_list", "parameters": [ { - "in": "path", - "name": "route", - "description": "Name or ID of the route", - "required": true, + "in": "query", + "name": "group", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "string", + "format": "uuid" } }, { "in": "query", - "name": "project", - "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 } }, { "in": "query", - "name": "router", - "description": "Name or ID of the router", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", "schema": { - "$ref": "#/components/schemas/NameOrId" + "nullable": true, + "type": "string" } }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", + "name": "sort_by", "schema": { - "$ref": "#/components/schemas/NameOrId" + "$ref": "#/components/schemas/IdSortMode" } } ], - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/RouterRouteUpdate" + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/UserResultsPage" + } } } }, - "required": true + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/utilization": { + "get": { + "tags": [ + "silos" + ], + "summary": "Fetch resource utilization for user's current silo", + "operationId": "utilization_view", "responses": { "200": { "description": "successful operation", "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/RouterRoute" + "$ref": "#/components/schemas/Utilization" } } } @@ -8707,23 +8983,16 @@ "$ref": "#/components/responses/Error" } } - }, - "delete": { + } + }, + "/v1/vpc-firewall-rules": { + "get": { "tags": [ "vpcs" ], - "summary": "Delete route", - "operationId": "vpc_router_route_delete", + "summary": "List firewall rules", + "operationId": "vpc_firewall_rules_view", "parameters": [ - { - "in": "path", - "name": "route", - "description": "Name or ID of the route", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "project", @@ -8732,26 +9001,26 @@ "$ref": "#/components/schemas/NameOrId" } }, - { - "in": "query", - "name": "router", - "description": "Name or ID of the router", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", + "description": "Name or ID of the VPC", + "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } } ], "responses": { - "204": { - "description": "successful deletion" + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcFirewallRules" + } + } + } }, "4XX": { "$ref": "#/components/responses/Error" @@ -8760,20 +9029,76 @@ "$ref": "#/components/responses/Error" } } - } - }, - "/v1/vpc-routers": { - "get": { + }, + "put": { "tags": [ "vpcs" ], - "summary": "List routers", - "operationId": "vpc_router_list", + "summary": "Replace firewall rules", + "description": "The maximum number of rules per VPC is 1024.\n\nTargets are used to specify the set of instances to which a firewall rule applies. You can target instances directly by name, or specify a VPC, VPC subnet, IP, or IP subnet, which will apply the rule to traffic going to all matching instances. Targets are additive: the rule applies to instances matching ANY target. The maximum number of targets is 256.\n\nFilters reduce the scope of a firewall rule. Without filters, the rule applies to all packets to the targets (or from the targets, if it's an outbound rule). With multiple filters, the rule applies only to packets matching ALL filters. The maximum number of each type of filter is 256.", + "operationId": "vpc_firewall_rules_update", "parameters": [ { "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcFirewallRuleUpdateParams" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcFirewallRules" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpc-router-routes": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List routes", + "description": "List the routes associated with a router in a particular VPC.", + "operationId": "vpc_router_route_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", "schema": { "nullable": true, "type": "integer", @@ -8798,6 +9123,14 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "sort_by", @@ -8808,7 +9141,7 @@ { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8820,7 +9153,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterResultsPage" + "$ref": "#/components/schemas/RouterRouteResultsPage" } } } @@ -8834,7 +9167,7 @@ }, "x-dropshot-pagination": { "required": [ - "vpc" + "router" ] } }, @@ -8842,8 +9175,8 @@ "tags": [ "vpcs" ], - "summary": "Create VPC router", - "operationId": "vpc_router_create", + "summary": "Create route", + "operationId": "vpc_router_route_create", "parameters": [ { "in": "query", @@ -8855,19 +9188,27 @@ }, { "in": "query", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } } ], "requestBody": { "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterCreate" + "$ref": "#/components/schemas/RouterRouteCreate" } } }, @@ -8879,7 +9220,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8893,18 +9234,18 @@ } } }, - "/v1/vpc-routers/{router}": { + "/v1/vpc-router-routes/{route}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch router", - "operationId": "vpc_router_view", + "summary": "Fetch route", + "operationId": "vpc_router_route_view", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8918,10 +9259,19 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8933,7 +9283,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -8950,13 +9300,13 @@ "tags": [ "vpcs" ], - "summary": "Update router", - "operationId": "vpc_router_update", + "summary": "Update route", + "operationId": "vpc_router_route_update", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -8970,10 +9320,18 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -8983,7 +9341,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouterUpdate" + "$ref": "#/components/schemas/RouterRouteUpdate" } } }, @@ -8995,7 +9353,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcRouter" + "$ref": "#/components/schemas/RouterRoute" } } } @@ -9012,13 +9370,13 @@ "tags": [ "vpcs" ], - "summary": "Delete router", - "operationId": "vpc_router_delete", + "summary": "Delete route", + "operationId": "vpc_router_route_delete", "parameters": [ { "in": "path", - "name": "router", - "description": "Name or ID of the router", + "name": "route", + "description": "Name or ID of the route", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -9032,10 +9390,18 @@ "$ref": "#/components/schemas/NameOrId" } }, + { + "in": "query", + "name": "router", + "description": "Name or ID of the router", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, { "in": "query", "name": "vpc", - "description": "Name or ID of the VPC", + "description": "Name or ID of the VPC, only required if `router` is provided as a `Name`", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -9054,13 +9420,13 @@ } } }, - "/v1/vpc-subnets": { + "/v1/vpc-routers": { "get": { "tags": [ "vpcs" ], - "summary": "List subnets", - "operationId": "vpc_subnet_list", + "summary": "List routers", + "operationId": "vpc_router_list", "parameters": [ { "in": "query", @@ -9112,7 +9478,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetResultsPage" + "$ref": "#/components/schemas/VpcRouterResultsPage" } } } @@ -9134,8 +9500,8 @@ "tags": [ "vpcs" ], - "summary": "Create subnet", - "operationId": "vpc_subnet_create", + "summary": "Create VPC router", + "operationId": "vpc_router_create", "parameters": [ { "in": "query", @@ -9159,7 +9525,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetCreate" + "$ref": "#/components/schemas/VpcRouterCreate" } } }, @@ -9171,7 +9537,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -9185,18 +9551,18 @@ } } }, - "/v1/vpc-subnets/{subnet}": { + "/v1/vpc-routers/{router}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch subnet", - "operationId": "vpc_subnet_view", + "summary": "Fetch router", + "operationId": "vpc_router_view", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -9225,7 +9591,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -9242,13 +9608,13 @@ "tags": [ "vpcs" ], - "summary": "Update subnet", - "operationId": "vpc_subnet_update", + "summary": "Update router", + "operationId": "vpc_router_update", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -9275,7 +9641,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnetUpdate" + "$ref": "#/components/schemas/VpcRouterUpdate" } } }, @@ -9287,7 +9653,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcSubnet" + "$ref": "#/components/schemas/VpcRouter" } } } @@ -9304,13 +9670,13 @@ "tags": [ "vpcs" ], - "summary": "Delete subnet", - "operationId": "vpc_subnet_delete", + "summary": "Delete router", + "operationId": "vpc_router_delete", "parameters": [ { "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", + "name": "router", + "description": "Name or ID of the router", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -9346,23 +9712,14 @@ } } }, - "/v1/vpc-subnets/{subnet}/network-interfaces": { + "/v1/vpc-subnets": { "get": { "tags": [ "vpcs" ], - "summary": "List network interfaces", - "operationId": "vpc_subnet_list_network_interfaces", + "summary": "List subnets", + "operationId": "vpc_subnet_list", "parameters": [ - { - "in": "path", - "name": "subnet", - "description": "Name or ID of the subnet", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, { "in": "query", "name": "limit", @@ -9413,7 +9770,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + "$ref": "#/components/schemas/VpcSubnetResultsPage" } } } @@ -9426,92 +9783,33 @@ } }, "x-dropshot-pagination": { - "required": [] + "required": [ + "vpc" + ] } - } - }, - "/v1/vpcs": { - "get": { + }, + "post": { "tags": [ "vpcs" ], - "summary": "List VPCs", - "operationId": "vpc_list", + "summary": "Create subnet", + "operationId": "vpc_subnet_create", "parameters": [ { "in": "query", - "name": "limit", - "description": "Maximum number of items returned by a single call", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", "schema": { - "nullable": true, - "type": "integer", - "format": "uint32", - "minimum": 1 + "$ref": "#/components/schemas/NameOrId" } }, { "in": "query", - "name": "page_token", - "description": "Token returned by previous call to retrieve the subsequent page", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, "schema": { - "nullable": true, - "type": "string" - } - }, - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "schema": { - "$ref": "#/components/schemas/NameOrId" - } - }, - { - "in": "query", - "name": "sort_by", - "schema": { - "$ref": "#/components/schemas/NameOrIdSortMode" - } - } - ], - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/VpcResultsPage" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - }, - "x-dropshot-pagination": { - "required": [ - "project" - ] - } - }, - "post": { - "tags": [ - "vpcs" - ], - "summary": "Create VPC", - "operationId": "vpc_create", - "parameters": [ - { - "in": "query", - "name": "project", - "description": "Name or ID of the project", - "required": true, - "schema": { - "$ref": "#/components/schemas/NameOrId" + "$ref": "#/components/schemas/NameOrId" } } ], @@ -9519,7 +9817,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcCreate" + "$ref": "#/components/schemas/VpcSubnetCreate" } } }, @@ -9531,7 +9829,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -9545,18 +9843,18 @@ } } }, - "/v1/vpcs/{vpc}": { + "/v1/vpc-subnets/{subnet}": { "get": { "tags": [ "vpcs" ], - "summary": "Fetch VPC", - "operationId": "vpc_view", + "summary": "Fetch subnet", + "operationId": "vpc_subnet_view", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -9565,7 +9863,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -9577,7 +9883,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -9594,13 +9900,13 @@ "tags": [ "vpcs" ], - "summary": "Update a VPC", - "operationId": "vpc_update", + "summary": "Update subnet", + "operationId": "vpc_subnet_update", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -9609,7 +9915,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -9619,7 +9933,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/VpcUpdate" + "$ref": "#/components/schemas/VpcSubnetUpdate" } } }, @@ -9631,7 +9945,7 @@ "content": { "application/json": { "schema": { - "$ref": "#/components/schemas/Vpc" + "$ref": "#/components/schemas/VpcSubnet" } } } @@ -9648,13 +9962,13 @@ "tags": [ "vpcs" ], - "summary": "Delete VPC", - "operationId": "vpc_delete", + "summary": "Delete subnet", + "operationId": "vpc_subnet_delete", "parameters": [ { "in": "path", - "name": "vpc", - "description": "Name or ID of the VPC", + "name": "subnet", + "description": "Name or ID of the subnet", "required": true, "schema": { "$ref": "#/components/schemas/NameOrId" @@ -9663,7 +9977,15 @@ { "in": "query", "name": "project", - "description": "Name or ID of the project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", "schema": { "$ref": "#/components/schemas/NameOrId" } @@ -9681,87 +10003,423 @@ } } } - } - }, - "components": { - "schemas": { - "Address": { - "description": "An address tied to an address lot.", - "type": "object", - "properties": { - "address": { - "description": "The address and prefix length of this address.", - "allOf": [ - { - "$ref": "#/components/schemas/IpNet" - } - ] + }, + "/v1/vpc-subnets/{subnet}/network-interfaces": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List network interfaces", + "operationId": "vpc_subnet_list_network_interfaces", + "parameters": [ + { + "in": "path", + "name": "subnet", + "description": "Name or ID of the subnet", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } }, - "address_lot": { - "description": "The address lot this address is drawn from.", - "allOf": [ - { - "$ref": "#/components/schemas/NameOrId" - } - ] + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } }, - "vlan_id": { - "nullable": true, - "description": "Optional VLAN ID for this address", - "type": "integer", - "format": "uint16", - "minimum": 0 - } - }, - "required": [ - "address", - "address_lot" - ] - }, - "AddressConfig": { - "description": "A set of addresses associated with a port configuration.", - "type": "object", - "properties": { - "addresses": { - "description": "The set of addresses assigned to the port configuration.", - "type": "array", - "items": { - "$ref": "#/components/schemas/Address" + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" } - } - }, - "required": [ - "addresses" - ] - }, - "AddressLot": { - "description": "Represents an address lot object, containing the id of the lot that can be used in other API calls.", - "type": "object", - "properties": { - "description": { - "description": "human-readable free-form text about a resource", - "type": "string" }, - "id": { - "description": "unique, immutable, system-controlled identifier for each resource", - "type": "string", - "format": "uuid" + { + "in": "query", + "name": "project", + "description": "Name or ID of the project, only required if `vpc` is provided as a `Name`", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } }, - "kind": { - "description": "Desired use of `AddressLot`", - "allOf": [ - { - "$ref": "#/components/schemas/AddressLotKind" - } - ] + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } }, - "name": { - "description": "unique, mutable, user-controlled identifier for each resource", - "allOf": [ - { - "$ref": "#/components/schemas/Name" + { + "in": "query", + "name": "vpc", + "description": "Name or ID of the VPC", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/InstanceNetworkInterfaceResultsPage" + } } - ] + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [] + } + } + }, + "/v1/vpcs": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "List VPCs", + "operationId": "vpc_list", + "parameters": [ + { + "in": "query", + "name": "limit", + "description": "Maximum number of items returned by a single call", + "schema": { + "nullable": true, + "type": "integer", + "format": "uint32", + "minimum": 1 + } + }, + { + "in": "query", + "name": "page_token", + "description": "Token returned by previous call to retrieve the subsequent page", + "schema": { + "nullable": true, + "type": "string" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "sort_by", + "schema": { + "$ref": "#/components/schemas/NameOrIdSortMode" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcResultsPage" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + }, + "x-dropshot-pagination": { + "required": [ + "project" + ] + } + }, + "post": { + "tags": [ + "vpcs" + ], + "summary": "Create VPC", + "operationId": "vpc_create", + "parameters": [ + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcCreate" + } + } + }, + "required": true + }, + "responses": { + "201": { + "description": "successful creation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, + "/v1/vpcs/{vpc}": { + "get": { + "tags": [ + "vpcs" + ], + "summary": "Fetch VPC", + "operationId": "vpc_view", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "put": { + "tags": [ + "vpcs" + ], + "summary": "Update a VPC", + "operationId": "vpc_update", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/VpcUpdate" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "successful operation", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/Vpc" + } + } + } + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + }, + "delete": { + "tags": [ + "vpcs" + ], + "summary": "Delete VPC", + "operationId": "vpc_delete", + "parameters": [ + { + "in": "path", + "name": "vpc", + "description": "Name or ID of the VPC", + "required": true, + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + }, + { + "in": "query", + "name": "project", + "description": "Name or ID of the project", + "schema": { + "$ref": "#/components/schemas/NameOrId" + } + } + ], + "responses": { + "204": { + "description": "successful deletion" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + } + }, + "components": { + "schemas": { + "Address": { + "description": "An address tied to an address lot.", + "type": "object", + "properties": { + "address": { + "description": "The address and prefix length of this address.", + "allOf": [ + { + "$ref": "#/components/schemas/IpNet" + } + ] + }, + "address_lot": { + "description": "The address lot this address is drawn from.", + "allOf": [ + { + "$ref": "#/components/schemas/NameOrId" + } + ] + }, + "vlan_id": { + "nullable": true, + "description": "Optional VLAN ID for this address", + "type": "integer", + "format": "uint16", + "minimum": 0 + } + }, + "required": [ + "address", + "address_lot" + ] + }, + "AddressConfig": { + "description": "A set of addresses associated with a port configuration.", + "type": "object", + "properties": { + "addresses": { + "description": "The set of addresses assigned to the port configuration.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Address" + } + } + }, + "required": [ + "addresses" + ] + }, + "AddressLot": { + "description": "Represents an address lot object, containing the id of the lot that can be used in other API calls.", + "type": "object", + "properties": { + "description": { + "description": "human-readable free-form text about a resource", + "type": "string" + }, + "id": { + "description": "unique, immutable, system-controlled identifier for each resource", + "type": "string", + "format": "uuid" + }, + "kind": { + "description": "Desired use of `AddressLot`", + "allOf": [ + { + "$ref": "#/components/schemas/AddressLotKind" + } + ] + }, + "name": { + "description": "unique, mutable, user-controlled identifier for each resource", + "allOf": [ + { + "$ref": "#/components/schemas/Name" + } + ] }, "time_created": { "description": "timestamp when this resource was created", @@ -12718,7 +13376,7 @@ "type": "string" }, "disk_source": { - "description": "initial source for this disk", + "description": "The initial source for this disk", "allOf": [ { "$ref": "#/components/schemas/DiskSource" @@ -12729,7 +13387,7 @@ "$ref": "#/components/schemas/Name" }, "size": { - "description": "total size of the Disk in bytes", + "description": "The total size of the Disk (in bytes)", "allOf": [ { "$ref": "#/components/schemas/ByteCount" @@ -14023,8 +14681,248 @@ }, "sum_of_samples": { "description": "The sum of all samples in the histogram.", - "type": "number", - "format": "double" + "type": "number", + "format": "double" + } + }, + "required": [ + "bins", + "max", + "min", + "n_samples", + "p50", + "p90", + "p99", + "squared_mean", + "start_time", + "sum_of_samples" + ] + }, + "Histogramfloat": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "description": "The bins of the histogram.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Binfloat" + } + }, + "max": { + "description": "The maximum value of all samples in the histogram.", + "type": "number", + "format": "float" + }, + "min": { + "description": "The minimum value of all samples in the histogram.", + "type": "number", + "format": "float" + }, + "n_samples": { + "description": "The total number of samples in the histogram.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "p50": { + "description": "p50 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "p90": { + "description": "p95 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "p99": { + "description": "p99 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "squared_mean": { + "description": "M2 for Welford's algorithm for variance calculation.\n\nRead about [Welford's algorithm](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm) for more information on the algorithm.", + "type": "number", + "format": "double" + }, + "start_time": { + "description": "The start time of the histogram.", + "type": "string", + "format": "date-time" + }, + "sum_of_samples": { + "description": "The sum of all samples in the histogram.", + "type": "number", + "format": "double" + } + }, + "required": [ + "bins", + "max", + "min", + "n_samples", + "p50", + "p90", + "p99", + "squared_mean", + "start_time", + "sum_of_samples" + ] + }, + "Histogramint16": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "description": "The bins of the histogram.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Binint16" + } + }, + "max": { + "description": "The maximum value of all samples in the histogram.", + "type": "integer", + "format": "int16" + }, + "min": { + "description": "The minimum value of all samples in the histogram.", + "type": "integer", + "format": "int16" + }, + "n_samples": { + "description": "The total number of samples in the histogram.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "p50": { + "description": "p50 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "p90": { + "description": "p95 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "p99": { + "description": "p99 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "squared_mean": { + "description": "M2 for Welford's algorithm for variance calculation.\n\nRead about [Welford's algorithm](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm) for more information on the algorithm.", + "type": "number", + "format": "double" + }, + "start_time": { + "description": "The start time of the histogram.", + "type": "string", + "format": "date-time" + }, + "sum_of_samples": { + "description": "The sum of all samples in the histogram.", + "type": "integer", + "format": "int64" + } + }, + "required": [ + "bins", + "max", + "min", + "n_samples", + "p50", + "p90", + "p99", + "squared_mean", + "start_time", + "sum_of_samples" + ] + }, + "Histogramint32": { + "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "type": "object", + "properties": { + "bins": { + "description": "The bins of the histogram.", + "type": "array", + "items": { + "$ref": "#/components/schemas/Binint32" + } + }, + "max": { + "description": "The maximum value of all samples in the histogram.", + "type": "integer", + "format": "int32" + }, + "min": { + "description": "The minimum value of all samples in the histogram.", + "type": "integer", + "format": "int32" + }, + "n_samples": { + "description": "The total number of samples in the histogram.", + "type": "integer", + "format": "uint64", + "minimum": 0 + }, + "p50": { + "description": "p50 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "p90": { + "description": "p95 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "p99": { + "description": "p99 Quantile", + "allOf": [ + { + "$ref": "#/components/schemas/Quantile" + } + ] + }, + "squared_mean": { + "description": "M2 for Welford's algorithm for variance calculation.\n\nRead about [Welford's algorithm](https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Welford's_online_algorithm) for more information on the algorithm.", + "type": "number", + "format": "double" + }, + "start_time": { + "description": "The start time of the histogram.", + "type": "string", + "format": "date-time" + }, + "sum_of_samples": { + "description": "The sum of all samples in the histogram.", + "type": "integer", + "format": "int64" } }, "required": [ @@ -14040,7 +14938,7 @@ "sum_of_samples" ] }, - "Histogramfloat": { + "Histogramint64": { "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { @@ -14048,18 +14946,18 @@ "description": "The bins of the histogram.", "type": "array", "items": { - "$ref": "#/components/schemas/Binfloat" + "$ref": "#/components/schemas/Binint64" } }, "max": { "description": "The maximum value of all samples in the histogram.", - "type": "number", - "format": "float" + "type": "integer", + "format": "int64" }, "min": { "description": "The minimum value of all samples in the histogram.", - "type": "number", - "format": "float" + "type": "integer", + "format": "int64" }, "n_samples": { "description": "The total number of samples in the histogram.", @@ -14103,8 +15001,8 @@ }, "sum_of_samples": { "description": "The sum of all samples in the histogram.", - "type": "number", - "format": "double" + "type": "integer", + "format": "int64" } }, "required": [ @@ -14120,7 +15018,7 @@ "sum_of_samples" ] }, - "Histogramint16": { + "Histogramint8": { "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { @@ -14128,18 +15026,18 @@ "description": "The bins of the histogram.", "type": "array", "items": { - "$ref": "#/components/schemas/Binint16" + "$ref": "#/components/schemas/Binint8" } }, "max": { "description": "The maximum value of all samples in the histogram.", "type": "integer", - "format": "int16" + "format": "int8" }, "min": { "description": "The minimum value of all samples in the histogram.", "type": "integer", - "format": "int16" + "format": "int8" }, "n_samples": { "description": "The total number of samples in the histogram.", @@ -14200,7 +15098,7 @@ "sum_of_samples" ] }, - "Histogramint32": { + "Histogramuint16": { "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { @@ -14208,18 +15106,20 @@ "description": "The bins of the histogram.", "type": "array", "items": { - "$ref": "#/components/schemas/Binint32" + "$ref": "#/components/schemas/Binuint16" } }, "max": { "description": "The maximum value of all samples in the histogram.", "type": "integer", - "format": "int32" + "format": "uint16", + "minimum": 0 }, "min": { "description": "The minimum value of all samples in the histogram.", "type": "integer", - "format": "int32" + "format": "uint16", + "minimum": 0 }, "n_samples": { "description": "The total number of samples in the histogram.", @@ -14280,7 +15180,7 @@ "sum_of_samples" ] }, - "Histogramint64": { + "Histogramuint32": { "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { @@ -14288,18 +15188,20 @@ "description": "The bins of the histogram.", "type": "array", "items": { - "$ref": "#/components/schemas/Binint64" + "$ref": "#/components/schemas/Binuint32" } }, "max": { "description": "The maximum value of all samples in the histogram.", "type": "integer", - "format": "int64" + "format": "uint32", + "minimum": 0 }, "min": { "description": "The minimum value of all samples in the histogram.", "type": "integer", - "format": "int64" + "format": "uint32", + "minimum": 0 }, "n_samples": { "description": "The total number of samples in the histogram.", @@ -14360,7 +15262,7 @@ "sum_of_samples" ] }, - "Histogramint8": { + "Histogramuint64": { "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { @@ -14368,18 +15270,20 @@ "description": "The bins of the histogram.", "type": "array", "items": { - "$ref": "#/components/schemas/Binint8" + "$ref": "#/components/schemas/Binuint64" } }, "max": { "description": "The maximum value of all samples in the histogram.", "type": "integer", - "format": "int8" + "format": "uint64", + "minimum": 0 }, "min": { "description": "The minimum value of all samples in the histogram.", "type": "integer", - "format": "int8" + "format": "uint64", + "minimum": 0 }, "n_samples": { "description": "The total number of samples in the histogram.", @@ -14440,7 +15344,7 @@ "sum_of_samples" ] }, - "Histogramuint16": { + "Histogramuint8": { "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", "type": "object", "properties": { @@ -14448,19 +15352,19 @@ "description": "The bins of the histogram.", "type": "array", "items": { - "$ref": "#/components/schemas/Binuint16" + "$ref": "#/components/schemas/Binuint8" } }, "max": { "description": "The maximum value of all samples in the histogram.", "type": "integer", - "format": "uint16", + "format": "uint8", "minimum": 0 }, "min": { "description": "The minimum value of all samples in the histogram.", "type": "integer", - "format": "uint16", + "format": "uint8", "minimum": 0 }, "n_samples": { @@ -14522,273 +15426,430 @@ "sum_of_samples" ] }, - "Histogramuint32": { - "description": "Histogram metric\n\nA histogram maintains the count of any number of samples, over a set of bins. Bins are specified on construction via their _left_ edges, inclusive. There can't be any \"gaps\" in the bins, and an additional bin may be added to the left, right, or both so that the bins extend to the entire range of the support.\n\nNote that any gaps, unsorted bins, or non-finite values will result in an error.", + "Hostname": { + "title": "An RFC-1035-compliant hostname", + "description": "A hostname identifies a host on a network, and is usually a dot-delimited sequence of labels, where each label contains only letters, digits, or the hyphen. See RFCs 1035 and 952 for more details.", + "type": "string", + "pattern": "^([a-zA-Z0-9]+[a-zA-Z0-9\\-]*(? internet gateway mappings.", + "operationId": "set_eip_gateways", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExternalIpGatewayMap" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "resource updated" + }, + "4XX": { + "$ref": "#/components/responses/Error" + }, + "5XX": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/inventory": { "get": { "summary": "Fetch basic information about this sled", @@ -404,27 +431,6 @@ } }, "/omicron-zones": { - "get": { - "operationId": "omicron_zones_get", - "responses": { - "200": { - "description": "successful operation", - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/OmicronZonesConfig" - } - } - } - }, - "4XX": { - "$ref": "#/components/responses/Error" - }, - "5XX": { - "$ref": "#/components/responses/Error" - } - } - }, "put": { "operationId": "omicron_zones_put", "requestBody": { @@ -3010,6 +3016,29 @@ "baseboard" ] }, + "ExternalIpGatewayMap": { + "description": "Per-NIC mappings from external IP addresses to the Internet Gateways which can choose them as a source.", + "type": "object", + "properties": { + "mappings": { + "type": "object", + "additionalProperties": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "string", + "format": "uuid" + }, + "uniqueItems": true + } + } + } + }, + "required": [ + "mappings" + ] + }, "Generation": { "description": "Generation numbers stored in the database, used for optimistic concurrency control", "type": "integer", @@ -3391,6 +3420,9 @@ "$ref": "#/components/schemas/InventoryDisk" } }, + "omicron_zones": { + "$ref": "#/components/schemas/OmicronZonesConfig" + }, "reservoir_size": { "$ref": "#/components/schemas/ByteCount" }, @@ -3422,6 +3454,7 @@ "baseboard", "datasets", "disks", + "omicron_zones", "reservoir_size", "sled_agent_address", "sled_id", @@ -4674,19 +4707,19 @@ } ] }, - "local_pref": { - "nullable": true, - "description": "The local preference associated with this route.", - "default": null, - "type": "integer", - "format": "uint32", - "minimum": 0 - }, "nexthop": { "description": "The nexthop/gateway address.", "type": "string", "format": "ip" }, + "rib_priority": { + "nullable": true, + "description": "The RIB priority (i.e. Admin Distance) associated with this route.", + "default": null, + "type": "integer", + "format": "uint8", + "minimum": 0 + }, "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", @@ -4779,10 +4812,16 @@ "enum": [ "internet_gateway" ] + }, + "value": { + "nullable": true, + "type": "string", + "format": "uuid" } }, "required": [ - "type" + "type", + "value" ] }, { diff --git a/openapi/wicketd.json b/openapi/wicketd.json index c7f6ea68d8..55db469210 100644 --- a/openapi/wicketd.json +++ b/openapi/wicketd.json @@ -3136,19 +3136,19 @@ } ] }, - "local_pref": { - "nullable": true, - "description": "The local preference associated with this route.", - "default": null, - "type": "integer", - "format": "uint32", - "minimum": 0 - }, "nexthop": { "description": "The nexthop/gateway address.", "type": "string", "format": "ip" }, + "rib_priority": { + "nullable": true, + "description": "The RIB priority (i.e. Admin Distance) associated with this route.", + "default": null, + "type": "integer", + "format": "uint8", + "minimum": 0 + }, "vlan_id": { "nullable": true, "description": "The VLAN id associated with this route.", diff --git a/oximeter/collector/Cargo.toml b/oximeter/collector/Cargo.toml index c6852b5599..334b091770 100644 --- a/oximeter/collector/Cargo.toml +++ b/oximeter/collector/Cargo.toml @@ -16,7 +16,8 @@ chrono.workspace = true clap.workspace = true dropshot.workspace = true futures.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true nexus-types.workspace = true omicron-common.workspace = true oximeter.workspace = true diff --git a/oximeter/collector/src/lib.rs b/oximeter/collector/src/lib.rs index a9f03a5b09..01770c9540 100644 --- a/oximeter/collector/src/lib.rs +++ b/oximeter/collector/src/lib.rs @@ -11,7 +11,7 @@ use dropshot::ConfigLogging; use dropshot::HttpError; use dropshot::HttpServer; use dropshot::HttpServerStarter; -use internal_dns::ServiceName; +use internal_dns_types::names::ServiceName; use omicron_common::address::get_internal_dns_server_addresses; use omicron_common::address::DNS_PORT; use omicron_common::api::internal::nexus::ProducerEndpoint; diff --git a/oximeter/db/src/client/oxql.rs b/oximeter/db/src/client/oxql.rs index 7c23eb0517..4fdfc71b76 100644 --- a/oximeter/db/src/client/oxql.rs +++ b/oximeter/db/src/client/oxql.rs @@ -162,6 +162,9 @@ impl Client { schema: &TimeseriesSchema, preds: &filter::Filter, ) -> Result, Error> { + // Potentially negate the predicate. + let maybe_not = if preds.negated { "NOT " } else { "" }; + // Walk the set of predicates, keeping those which apply to this schema. match &preds.expr { filter::FilterExpr::Simple(inner) => { @@ -183,7 +186,7 @@ impl Client { field_schema.field_type, ))); } - Ok(Some(inner.as_db_safe_string())) + Ok(Some(format!("{}{}", maybe_not, inner.as_db_safe_string()))) } filter::FilterExpr::Compound(inner) => { let left_pred = @@ -192,7 +195,8 @@ impl Client { Self::rewrite_predicate_for_fields(schema, &inner.right)?; let out = match (left_pred, right_pred) { (Some(left), Some(right)) => Some(format!( - "{}({left}, {right})", + "{}{}({left}, {right})", + maybe_not, inner.op.as_db_function_name() )), (Some(single), None) | (None, Some(single)) => Some(single), @@ -209,6 +213,9 @@ impl Client { schema: &TimeseriesSchema, preds: &oxql::ast::table_ops::filter::Filter, ) -> Result, Error> { + // Potentially negate the predicate. + let maybe_not = if preds.negated { "NOT " } else { "" }; + // Walk the set of predicates, keeping those which apply to this schema. match &preds.expr { filter::FilterExpr::Simple(inner) => { @@ -220,7 +227,11 @@ impl Client { inner.value, oxql::ast::literal::Literal::Timestamp(_) ) { - return Ok(Some(inner.as_db_safe_string())); + return Ok(Some(format!( + "{}{}", + maybe_not, + inner.as_db_safe_string() + ))); } return Err(Error::from(anyhow::anyhow!( "Literal cannot be compared with a timestamp" @@ -242,7 +253,11 @@ impl Client { inner.value, oxql::ast::literal::Literal::Timestamp(_) ) { - return Ok(Some(inner.as_db_safe_string())); + return Ok(Some(format!( + "{}{}", + maybe_not, + inner.as_db_safe_string() + ))); } return Err(Error::from(anyhow::anyhow!( "Literal cannot be compared with a timestamp" @@ -264,7 +279,8 @@ impl Client { )?; let out = match (left_pred, right_pred) { (Some(left), Some(right)) => Some(format!( - "{}({left}, {right})", + "{}{}({left}, {right})", + maybe_not, inner.op.as_db_function_name() )), (Some(single), None) | (None, Some(single)) => Some(single), @@ -1110,16 +1126,20 @@ fn update_total_rows_and_check( mod tests { use super::ConsistentKeyGroup; use crate::client::oxql::chunk_consistent_key_groups_impl; - use crate::{Client, DbWrite}; + use crate::oxql::ast::grammar::query_parser; + use crate::{Client, DbWrite, DATABASE_TIMESTAMP_FORMAT}; use crate::{Metric, Target}; - use chrono::{DateTime, Utc}; + use chrono::{DateTime, NaiveDate, Utc}; use dropshot::test_util::LogContext; use omicron_test_utils::dev::clickhouse::ClickHouseDeployment; use omicron_test_utils::dev::test_setup_log; use oximeter::{types::Cumulative, FieldValue}; - use oximeter::{DatumType, Sample}; + use oximeter::{ + AuthzScope, DatumType, FieldSchema, FieldSource, FieldType, Sample, + TimeseriesSchema, Units, + }; use oxql_types::{point::Points, Table, Timeseries}; - use std::collections::BTreeMap; + use std::collections::{BTreeMap, BTreeSet}; use std::time::Duration; #[derive( @@ -1648,4 +1668,62 @@ mod tests { ctx.cleanup_successful().await; } + + fn test_schema() -> TimeseriesSchema { + TimeseriesSchema { + timeseries_name: "foo:bar".parse().unwrap(), + description: Default::default(), + field_schema: BTreeSet::from([FieldSchema { + name: String::from("f0"), + field_type: FieldType::U32, + source: FieldSource::Target, + description: String::new(), + }]), + datum_type: DatumType::U64, + version: 1.try_into().unwrap(), + authz_scope: AuthzScope::Fleet, + units: Units::None, + created: Utc::now(), + } + } + + #[test] + fn correctly_negate_field_predicate_expression() { + let logctx = + test_setup_log("correctly_negate_field_predicate_expression"); + let schema = test_schema(); + let filt = query_parser::filter("filter !(f0 == 0)").unwrap(); + let rewritten = Client::rewrite_predicate_for_fields(&schema, &filt) + .unwrap() + .expect("Should have rewritten the field predicate"); + assert_eq!(rewritten, "NOT equals(f0, 0)"); + logctx.cleanup_successful(); + } + + #[test] + fn correctly_negate_timestamp_predicate_expression() { + let logctx = + test_setup_log("correctly_negate_field_predicate_expression"); + let schema = test_schema(); + let now = NaiveDate::from_ymd_opt(2024, 1, 1) + .unwrap() + .and_hms_opt(0, 0, 0) + .unwrap() + .and_utc(); + let now_str = "2024-01-01"; + let filter_str = format!("filter !(timestamp > @{})", now_str); + let filt = query_parser::filter(&filter_str).unwrap(); + let rewritten = + Client::rewrite_predicate_for_measurements(&schema, &filt) + .unwrap() + .expect("Should have rewritten the timestamp predicate"); + assert_eq!( + rewritten, + format!( + "NOT greater(timestamp, '{}')", + now.format(DATABASE_TIMESTAMP_FORMAT) + ) + ); + logctx.cleanup_successful(); + } } diff --git a/oximeter/db/src/oxql/ast/mod.rs b/oximeter/db/src/oxql/ast/mod.rs index 7037b74a7f..21fe5b0387 100644 --- a/oximeter/db/src/oxql/ast/mod.rs +++ b/oximeter/db/src/oxql/ast/mod.rs @@ -14,7 +14,7 @@ use self::table_ops::BasicTableOp; use self::table_ops::GroupedTableOp; use self::table_ops::TableOp; pub mod cmp; -pub(super) mod grammar; +pub(crate) mod grammar; pub mod ident; pub mod literal; pub mod logical_op; diff --git a/oximeter/producer/Cargo.toml b/oximeter/producer/Cargo.toml index dfac555a49..cc1036a824 100644 --- a/oximeter/producer/Cargo.toml +++ b/oximeter/producer/Cargo.toml @@ -11,6 +11,8 @@ workspace = true [dependencies] chrono.workspace = true dropshot.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true nexus-client.workspace = true omicron-common.workspace = true oximeter.workspace = true @@ -22,7 +24,6 @@ tokio.workspace = true thiserror.workspace = true uuid.workspace = true omicron-workspace-hack.workspace = true -internal-dns.workspace = true [dev-dependencies] anyhow.workspace = true diff --git a/oximeter/producer/src/lib.rs b/oximeter/producer/src/lib.rs index e9223b62f3..4bde5f69ff 100644 --- a/oximeter/producer/src/lib.rs +++ b/oximeter/producer/src/lib.rs @@ -15,9 +15,9 @@ use dropshot::HttpServer; use dropshot::HttpServerStarter; use dropshot::Path; use dropshot::RequestContext; -use internal_dns::resolver::ResolveError; -use internal_dns::resolver::Resolver; -use internal_dns::ServiceName; +use internal_dns_resolver::ResolveError; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use nexus_client::types::ProducerEndpoint as ApiProducerEndpoint; use omicron_common::api::internal::nexus::ProducerEndpoint; use omicron_common::backoff; diff --git a/package-manifest.toml b/package-manifest.toml index 9179212755..e724486e77 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -178,6 +178,7 @@ source.paths = [ { from = "out/clickhouse", to = "/opt/oxide/clickhouse" }, { from = "smf/clickhouse/manifest.xml", to = "/var/svc/manifest/site/clickhouse/manifest.xml" }, { from = "smf/clickhouse/method_script.sh", to = "/opt/oxide/lib/svc/manifest/clickhouse.sh" }, + { from = "smf/clickhouse/config.xml", to = "/opt/oxide/clickhouse/config.xml" }, ] output.type = "zone" output.intermediate_only = true @@ -242,13 +243,14 @@ output.intermediate_only = true setup_hint = "Run `cargo xtask download clickhouse` to download the necessary binaries" [package.omicron-clickhouse-admin] -service_name = "clickhouse-admin" +service_name = "omicron-clickhouse-admin" only_for_targets.image = "standard" source.type = "local" -source.rust.binary_names = ["clickhouse-admin"] +source.rust.binary_names = ["clickhouse-admin-keeper", "clickhouse-admin-server"] source.rust.release = true source.paths = [ - { from = "smf/clickhouse-admin", to = "/var/svc/manifest/site/clickhouse-admin" }, + { from = "smf/clickhouse-admin-keeper", to = "/var/svc/manifest/site/clickhouse-admin-keeper" }, + { from = "smf/clickhouse-admin-server", to = "/var/svc/manifest/site/clickhouse-admin-server" }, ] output.type = "zone" output.intermediate_only = true @@ -633,10 +635,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "c92d6ff85db8992066f49da176cf686acfd8fe0f" +source.commit = "056283eb02b6887fbf27f66a215662520f7c159c" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm-gz.sha256.txt -source.sha256 = "c33915998894dd36a2d1078f7e13717aa20760924c30640d7647d4791dd5f2ee" +source.sha256 = "973fc43ed3b0727d72e3493339e1fdb69e7cb2767ee4aa27f65c4a2da8f8126b" output.type = "tarball" [package.mg-ddm] @@ -649,10 +651,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "c92d6ff85db8992066f49da176cf686acfd8fe0f" +source.commit = "056283eb02b6887fbf27f66a215662520f7c159c" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "be9d657ec22a69468b18f2b4d48e55621538eade8b8d3e367a1d8d5cc686cfbe" +source.sha256 = "ed60620a32a35a6885064e7c777369d5092455cd5c1aa240672dfaac05c31f56" output.type = "zone" output.intermediate_only = true @@ -664,10 +666,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "c92d6ff85db8992066f49da176cf686acfd8fe0f" +source.commit = "056283eb02b6887fbf27f66a215662520f7c159c" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mgd.sha256.txt -source.sha256 = "e000485f7e04ac1cf9b3532b60bcf23598ab980331ba4f1c6788a7e95c1e9ef8" +source.sha256 = "7c10ac7d284ce78e70e652ad91bebf3fee7a2274ee403a09cc986c6ee73cf1eb" output.type = "zone" output.intermediate_only = true @@ -715,8 +717,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -source.commit = "8dab641b522375a4b403ae4bd0c9a22d905fae5d" -source.sha256 = "f848e54d5ea7ddcbc91b50d986dd7ee6617809690fb7ed6463a51fc2235786a8" +source.commit = "f3810e7bc1f0d746b5e95b3aaff32e52b02dfdfa" +source.sha256 = "c1506f6f818327523e6ff3102432a2038d319338b883235664b34f9132ff676a" output.type = "zone" output.intermediate_only = true @@ -742,8 +744,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -source.commit = "8dab641b522375a4b403ae4bd0c9a22d905fae5d" -source.sha256 = "d41d8f026da7db7044d1fb8aa2b7a5b64be9e3b606f72f23dd8169c859caa58f" +source.commit = "f3810e7bc1f0d746b5e95b3aaff32e52b02dfdfa" +source.sha256 = "061d40085e733e60d7c53ebfd2a4cf64f54a856e7eb5fd4b82ac65ec6a5b847b" output.type = "zone" output.intermediate_only = true @@ -762,8 +764,8 @@ only_for_targets.image = "standard" # the other `source.*` keys. source.type = "prebuilt" source.repo = "dendrite" -source.commit = "8dab641b522375a4b403ae4bd0c9a22d905fae5d" -source.sha256 = "3b42df3c28a2aefb052bd0bd5b64a595295b5900f97112d8502eaa1edea7e390" +source.commit = "f3810e7bc1f0d746b5e95b3aaff32e52b02dfdfa" +source.sha256 = "c6cb4c077f0ddfc78ab06e07316d1312657f95526ced60c2b8e7baf1c73ae24a" output.type = "zone" output.intermediate_only = true diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index a6cd9b38fe..f78e034721 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -1586,6 +1586,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.network_interface ( transit_ips INET[] NOT NULL DEFAULT ARRAY[] ); +CREATE INDEX IF NOT EXISTS instance_network_interface_mac + ON omicron.public.network_interface (mac) STORING (time_deleted); + /* A view of the network_interface table for just instance-kind records. */ CREATE VIEW IF NOT EXISTS omicron.public.instance_network_interface AS SELECT @@ -1764,6 +1767,12 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_router_by_vpc ON omicron.public.vpc_rou ) WHERE time_deleted IS NULL; +/* Index used to accelerate vpc_increment_rpw_version and list. */ +CREATE INDEX IF NOT EXISTS lookup_routers_in_vpc ON omicron.public.vpc_router ( + vpc_id +) WHERE + time_deleted IS NULL; + CREATE TYPE IF NOT EXISTS omicron.public.router_route_kind AS ENUM ( 'default', 'vpc_subnet', @@ -1794,6 +1803,57 @@ CREATE UNIQUE INDEX IF NOT EXISTS lookup_route_by_router ON omicron.public.route ) WHERE time_deleted IS NULL; +CREATE TABLE IF NOT EXISTS omicron.public.internet_gateway ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + vpc_id UUID NOT NULL, + rcgen INT NOT NULL, + resolved_version INT NOT NULL DEFAULT 0 +); + +CREATE UNIQUE INDEX IF NOT EXISTS lookup_internet_gateway_by_vpc ON omicron.public.internet_gateway ( + vpc_id, + name +) WHERE + time_deleted IS NULL; + +CREATE TABLE IF NOT EXISTS omicron.public.internet_gateway_ip_pool ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + internet_gateway_id UUID, + ip_pool_id UUID +); + +CREATE INDEX IF NOT EXISTS lookup_internet_gateway_ip_pool_by_igw_id ON omicron.public.internet_gateway_ip_pool ( + internet_gateway_id +) WHERE + time_deleted IS NULL; + +CREATE TABLE IF NOT EXISTS omicron.public.internet_gateway_ip_address ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + internet_gateway_id UUID, + address INET +); + +CREATE UNIQUE INDEX IF NOT EXISTS lookup_internet_gateway_ip_address_by_igw_id ON omicron.public.internet_gateway_ip_address ( + internet_gateway_id +) WHERE + time_deleted IS NULL; + + /* * An IP Pool, a collection of zero or more IP ranges for external IPs. */ @@ -3044,6 +3104,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_collection ( CREATE INDEX IF NOT EXISTS inv_collection_by_time_started ON omicron.public.inv_collection (time_started); +CREATE INDEX IF NOT EXISTS inv_collectionby_time_done + ON omicron.public.inv_collection (time_done DESC); + -- list of errors generated during a collection CREATE TABLE IF NOT EXISTS omicron.public.inv_collection_error ( inv_collection_id UUID NOT NULL, @@ -3328,6 +3391,8 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_dataset ( PRIMARY KEY (inv_collection_id, sled_id, name) ); +-- TODO: This table is vestigial and can be combined with `inv_sled_agent`. See +-- https://github.com/oxidecomputer/omicron/issues/6770. CREATE TABLE IF NOT EXISTS omicron.public.inv_sled_omicron_zones ( -- where this observation came from -- (foreign key into `inv_collection` table) @@ -3435,6 +3500,9 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_omicron_zone ( PRIMARY KEY (inv_collection_id, id) ); +CREATE INDEX IF NOT EXISTS inv_omicron_zone_nic_id ON omicron.public.inv_omicron_zone + (nic_id) STORING (sled_id, primary_service_ip, second_service_ip, snat_ip); + CREATE TABLE IF NOT EXISTS omicron.public.inv_omicron_zone_nic ( inv_collection_id UUID NOT NULL, id UUID NOT NULL, @@ -3449,6 +3517,15 @@ CREATE TABLE IF NOT EXISTS omicron.public.inv_omicron_zone_nic ( PRIMARY KEY (inv_collection_id, id) ); +CREATE TABLE IF NOT EXISTS omicron.public.inv_clickhouse_keeper_membership ( + inv_collection_id UUID NOT NULL, + queried_keeper_id INT8 NOT NULL, + leader_committed_log_index INT8 NOT NULL, + raft_config INT8[] NOT NULL, + + PRIMARY KEY (inv_collection_id, queried_keeper_id) +); + /* * System-level blueprints * @@ -4421,7 +4498,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - (TRUE, NOW(), NOW(), '107.0.0', NULL) + (TRUE, NOW(), NOW(), '109.0.0', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/schema/crdb/internet-gateway/up01.sql b/schema/crdb/internet-gateway/up01.sql new file mode 100644 index 0000000000..bf08bfccc3 --- /dev/null +++ b/schema/crdb/internet-gateway/up01.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS omicron.public.internet_gateway ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + vpc_id UUID NOT NULL, + rcgen INT NOT NULL, + resolved_version INT NOT NULL DEFAULT 0 +); diff --git a/schema/crdb/internet-gateway/up02.sql b/schema/crdb/internet-gateway/up02.sql new file mode 100644 index 0000000000..5deca8277b --- /dev/null +++ b/schema/crdb/internet-gateway/up02.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS omicron.public.internet_gateway_ip_pool ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + internet_gateway_id UUID, + ip_pool_id UUID +); + diff --git a/schema/crdb/internet-gateway/up03.sql b/schema/crdb/internet-gateway/up03.sql new file mode 100644 index 0000000000..b95fb8f43c --- /dev/null +++ b/schema/crdb/internet-gateway/up03.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS omicron.public.internet_gateway_ip_address ( + id UUID PRIMARY KEY, + name STRING(63) NOT NULL, + description STRING(512) NOT NULL, + time_created TIMESTAMPTZ NOT NULL, + time_modified TIMESTAMPTZ NOT NULL, + time_deleted TIMESTAMPTZ, + internet_gateway_id UUID, + address INET +); + diff --git a/schema/crdb/internet-gateway/up04.sql b/schema/crdb/internet-gateway/up04.sql new file mode 100644 index 0000000000..c3608be9a9 --- /dev/null +++ b/schema/crdb/internet-gateway/up04.sql @@ -0,0 +1,5 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_internet_gateway_by_vpc ON omicron.public.internet_gateway ( + vpc_id, + name +) WHERE + time_deleted IS NULL; diff --git a/schema/crdb/internet-gateway/up05.sql b/schema/crdb/internet-gateway/up05.sql new file mode 100644 index 0000000000..390e984aef --- /dev/null +++ b/schema/crdb/internet-gateway/up05.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_internet_gateway_ip_pool_by_igw_id ON omicron.public.internet_gateway_ip_pool ( + internet_gateway_id +) WHERE + time_deleted IS NULL; diff --git a/schema/crdb/internet-gateway/up06.sql b/schema/crdb/internet-gateway/up06.sql new file mode 100644 index 0000000000..4e37708f1d --- /dev/null +++ b/schema/crdb/internet-gateway/up06.sql @@ -0,0 +1,4 @@ +CREATE UNIQUE INDEX IF NOT EXISTS lookup_internet_gateway_ip_address_by_igw_id ON omicron.public.internet_gateway_ip_address ( + internet_gateway_id +) WHERE + time_deleted IS NULL; diff --git a/schema/crdb/internet-gateway/up07.sql b/schema/crdb/internet-gateway/up07.sql new file mode 100644 index 0000000000..01d9eb5174 --- /dev/null +++ b/schema/crdb/internet-gateway/up07.sql @@ -0,0 +1,4 @@ +DELETE FROM omicron.public.router_route WHERE +time_deleted IS NULL AND +name = 'default-v4' AND +vpc_router_id = '001de000-074c-4000-8000-000000000001'; diff --git a/schema/crdb/internet-gateway/up08.sql b/schema/crdb/internet-gateway/up08.sql new file mode 100644 index 0000000000..1a1e3e5879 --- /dev/null +++ b/schema/crdb/internet-gateway/up08.sql @@ -0,0 +1,4 @@ +DELETE FROM omicron.public.router_route WHERE +time_deleted IS NULL AND +name = 'default-v6' AND +vpc_router_id = '001de000-074c-4000-8000-000000000001'; diff --git a/schema/crdb/internet-gateway/up09.sql b/schema/crdb/internet-gateway/up09.sql new file mode 100644 index 0000000000..e0b767514d --- /dev/null +++ b/schema/crdb/internet-gateway/up09.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS inv_collectionby_time_done + ON omicron.public.inv_collection (time_done DESC); diff --git a/schema/crdb/internet-gateway/up10.sql b/schema/crdb/internet-gateway/up10.sql new file mode 100644 index 0000000000..8e987d3e66 --- /dev/null +++ b/schema/crdb/internet-gateway/up10.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS inv_omicron_zone_nic_id ON omicron.public.inv_omicron_zone + (nic_id) STORING (sled_id, primary_service_ip, second_service_ip, snat_ip); diff --git a/schema/crdb/internet-gateway/up11.sql b/schema/crdb/internet-gateway/up11.sql new file mode 100644 index 0000000000..71adf01acd --- /dev/null +++ b/schema/crdb/internet-gateway/up11.sql @@ -0,0 +1,2 @@ +CREATE INDEX IF NOT EXISTS instance_network_interface_mac + ON omicron.public.network_interface (mac) STORING (time_deleted); diff --git a/schema/crdb/internet-gateway/up12.sql b/schema/crdb/internet-gateway/up12.sql new file mode 100644 index 0000000000..4540ee4b84 --- /dev/null +++ b/schema/crdb/internet-gateway/up12.sql @@ -0,0 +1,4 @@ +CREATE INDEX IF NOT EXISTS lookup_routers_in_vpc ON omicron.public.vpc_router ( + vpc_id +) WHERE + time_deleted IS NULL; diff --git a/schema/crdb/internet-gateway/up13.sql b/schema/crdb/internet-gateway/up13.sql new file mode 100644 index 0000000000..9074196bf3 --- /dev/null +++ b/schema/crdb/internet-gateway/up13.sql @@ -0,0 +1,96 @@ +-- Add default gateway for services + +set local disallow_full_table_scans = off; + +-- delete old style default routes +DELETE FROM omicron.public.router_route + WHERE target = 'inetgw:outbound'; + +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +SELECT + gen_random_uuid(), 'default-v4', + 'The default route of a vpc', + now(), now(), + omicron.public.vpc_router.id, 'default', + 'inetgw:default', 'ipnet:0.0.0.0/0' +FROM + omicron.public.vpc_router +ON CONFLICT DO NOTHING; + +INSERT INTO omicron.public.router_route + ( + id, name, + description, + time_created, time_modified, + vpc_router_id, kind, + target, destination + ) +SELECT + gen_random_uuid(), 'default-v6', + 'The default route of a vpc', + now(), now(), + omicron.public.vpc_router.id, 'default', + 'inetgw:default', 'ipnet:::/0' +FROM + omicron.public.vpc_router +ON CONFLICT DO NOTHING; + +-- insert default internet gateways for all VPCs + +INSERT INTO omicron.public.internet_gateway + ( + id, name, + description, + time_created, time_modified, time_deleted, + vpc_id, + rcgen, + resolved_version + ) +SELECT + gen_random_uuid(), 'default', + 'Default VPC gateway', + now(), now(), NULL, + omicron.public.vpc.id, + 0, + 0 +FROM + omicron.public.vpc +ON CONFLICT DO NOTHING; + +-- link default gateways to default ip pools + +INSERT INTO omicron.public.internet_gateway_ip_pool + ( + id, name, + description, + time_created, time_modified, time_deleted, + internet_gateway_id, + ip_pool_id + ) +SELECT + gen_random_uuid(), 'default', + 'Default internet gateway IP pool', + now(), now(), NULL, + igw.id, + ipp.ip_pool_id +FROM + omicron.public.vpc AS vpc +JOIN + omicron.public.internet_gateway as igw + ON igw.vpc_id = vpc.id +JOIN + omicron.public.project as project + ON vpc.project_id = project.id +JOIN + omicron.public.ip_pool_resource as ipp + ON project.silo_id = ipp.resource_id +WHERE + ipp.is_default = true +ON CONFLICT DO NOTHING; diff --git a/schema/crdb/inv-clickhouse-keeper-membership/up.sql b/schema/crdb/inv-clickhouse-keeper-membership/up.sql new file mode 100644 index 0000000000..5f7ffd8f5a --- /dev/null +++ b/schema/crdb/inv-clickhouse-keeper-membership/up.sql @@ -0,0 +1,8 @@ +CREATE TABLE IF NOT EXISTS omicron.public.inv_clickhouse_keeper_membership ( + inv_collection_id UUID NOT NULL, + queried_keeper_id INT8 NOT NULL, + leader_committed_log_index INT8 NOT NULL, + raft_config INT8[] NOT NULL, + + PRIMARY KEY (inv_collection_id, queried_keeper_id) +); diff --git a/schema/rss-service-plan-v4.json b/schema/rss-service-plan-v4.json index 0b29ffdd38..8b9260c208 100644 --- a/schema/rss-service-plan-v4.json +++ b/schema/rss-service-plan-v4.json @@ -459,7 +459,6 @@ } }, "DnsConfigParams": { - "description": "DnsConfigParams\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"generation\", \"time_created\", \"zones\" ], \"properties\": { \"generation\": { \"type\": \"integer\", \"format\": \"uint64\", \"minimum\": 0.0 }, \"time_created\": { \"type\": \"string\", \"format\": \"date-time\" }, \"zones\": { \"type\": \"array\", \"items\": { \"$ref\": \"#/components/schemas/DnsConfigZone\" } } } } ```
", "type": "object", "required": [ "generation", @@ -485,7 +484,6 @@ } }, "DnsConfigZone": { - "description": "DnsConfigZone\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"records\", \"zone_name\" ], \"properties\": { \"records\": { \"type\": \"object\", \"additionalProperties\": { \"type\": \"array\", \"items\": { \"$ref\": \"#/components/schemas/DnsRecord\" } } }, \"zone_name\": { \"type\": \"string\" } } } ```
", "type": "object", "required": [ "records", @@ -507,7 +505,6 @@ } }, "DnsRecord": { - "description": "DnsRecord\n\n
JSON schema\n\n```json { \"oneOf\": [ { \"type\": \"object\", \"required\": [ \"data\", \"type\" ], \"properties\": { \"data\": { \"type\": \"string\", \"format\": \"ipv4\" }, \"type\": { \"type\": \"string\", \"enum\": [ \"A\" ] } } }, { \"type\": \"object\", \"required\": [ \"data\", \"type\" ], \"properties\": { \"data\": { \"type\": \"string\", \"format\": \"ipv6\" }, \"type\": { \"type\": \"string\", \"enum\": [ \"AAAA\" ] } } }, { \"type\": \"object\", \"required\": [ \"data\", \"type\" ], \"properties\": { \"data\": { \"$ref\": \"#/components/schemas/Srv\" }, \"type\": { \"type\": \"string\", \"enum\": [ \"SRV\" ] } } } ] } ```
", "oneOf": [ { "type": "object", @@ -922,7 +919,6 @@ } }, "Srv": { - "description": "Srv\n\n
JSON schema\n\n```json { \"type\": \"object\", \"required\": [ \"port\", \"prio\", \"target\", \"weight\" ], \"properties\": { \"port\": { \"type\": \"integer\", \"format\": \"uint16\", \"minimum\": 0.0 }, \"prio\": { \"type\": \"integer\", \"format\": \"uint16\", \"minimum\": 0.0 }, \"target\": { \"type\": \"string\" }, \"weight\": { \"type\": \"integer\", \"format\": \"uint16\", \"minimum\": 0.0 } } } ```
", "type": "object", "required": [ "port", diff --git a/schema/rss-sled-plan.json b/schema/rss-sled-plan.json index 5150d9c471..7e8513e1e3 100644 --- a/schema/rss-sled-plan.json +++ b/schema/rss-sled-plan.json @@ -978,21 +978,21 @@ } ] }, - "local_pref": { - "description": "The local preference associated with this route.", + "nexthop": { + "description": "The nexthop/gateway address.", + "type": "string", + "format": "ip" + }, + "rib_priority": { + "description": "The RIB priority (i.e. Admin Distance) associated with this route.", "default": null, "type": [ "integer", "null" ], - "format": "uint32", + "format": "uint8", "minimum": 0.0 }, - "nexthop": { - "description": "The nexthop/gateway address.", - "type": "string", - "format": "ip" - }, "vlan_id": { "description": "The VLAN id associated with this route.", "default": null, diff --git a/sled-agent/Cargo.toml b/sled-agent/Cargo.toml index 3208f1c031..360ba7f499 100644 --- a/sled-agent/Cargo.toml +++ b/sled-agent/Cargo.toml @@ -40,7 +40,8 @@ hyper-staticfile.workspace = true gateway-client.workspace = true illumos-utils.workspace = true installinator-common.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true ipnetwork.workspace = true itertools.workspace = true key-manager.workspace = true diff --git a/sled-agent/api/src/lib.rs b/sled-agent/api/src/lib.rs index d9e49a5c56..e0d76a857b 100644 --- a/sled-agent/api/src/lib.rs +++ b/sled-agent/api/src/lib.rs @@ -17,8 +17,8 @@ use omicron_common::{ api::internal::{ nexus::{DiskRuntimeState, SledVmmState, UpdateArtifactId}, shared::{ - ResolvedVpcRouteSet, ResolvedVpcRouteState, SledIdentifiers, - SwitchPorts, VirtualNetworkInterfaceHost, + ExternalIpGatewayMap, ResolvedVpcRouteSet, ResolvedVpcRouteState, + SledIdentifiers, SwitchPorts, VirtualNetworkInterfaceHost, }, }, disk::{ @@ -154,14 +154,6 @@ pub trait SledAgentApi { rqctx: RequestContext, ) -> Result>, HttpError>; - #[endpoint { - method = GET, - path = "/omicron-zones", - }] - async fn omicron_zones_get( - rqctx: RequestContext, - ) -> Result, HttpError>; - #[endpoint { method = PUT, path = "/omicron-zones", @@ -487,6 +479,16 @@ pub trait SledAgentApi { request_context: RequestContext, body: TypedBody>, ) -> Result; + + /// Update per-NIC IP address <-> internet gateway mappings. + #[endpoint { + method = PUT, + path = "/eip-gateways", + }] + async fn set_eip_gateways( + request_context: RequestContext, + body: TypedBody, + ) -> Result; } #[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] diff --git a/sled-agent/bootstrap-agent-api/Cargo.toml b/sled-agent/bootstrap-agent-api/Cargo.toml index 368c5afe93..24b4866dcd 100644 --- a/sled-agent/bootstrap-agent-api/Cargo.toml +++ b/sled-agent/bootstrap-agent-api/Cargo.toml @@ -9,7 +9,6 @@ workspace = true [dependencies] dropshot.workspace = true -nexus-client.workspace = true omicron-common.workspace = true omicron-uuid-kinds.workspace = true omicron-workspace-hack.workspace = true diff --git a/sled-agent/src/bootstrap/early_networking.rs b/sled-agent/src/bootstrap/early_networking.rs index abc88d67c1..9685780a0e 100644 --- a/sled-agent/src/bootstrap/early_networking.rs +++ b/sled-agent/src/bootstrap/early_networking.rs @@ -12,8 +12,8 @@ use dpd_client::Client as DpdClient; use futures::future; use gateway_client::Client as MgsClient; use http::StatusCode; -use internal_dns::resolver::{ResolveError, Resolver as DnsResolver}; -use internal_dns::ServiceName; +use internal_dns_resolver::{ResolveError, Resolver as DnsResolver}; +use internal_dns_types::names::ServiceName; use mg_admin_client::types::BfdPeerConfig as MgBfdPeerConfig; use mg_admin_client::types::BgpPeerConfig as MgBgpPeerConfig; use mg_admin_client::types::ImportExportPolicy as MgImportExportPolicy; @@ -45,6 +45,10 @@ use tokio::time::sleep; const BGP_SESSION_RESOLUTION: u64 = 100; +// This is the default RIB Priority used for static routes. This mirrors +// the const defined in maghemite in rdb/src/lib.rs. +const DEFAULT_RIB_PRIORITY_STATIC: u8 = 1; + /// Errors that can occur during early network setup #[derive(Error, Debug)] pub enum EarlyNetworkSetupError { @@ -631,8 +635,14 @@ impl<'a> EarlyNetworkSetup<'a> { IpAddr::V6(_) => continue, }; let vlan_id = r.vlan_id; - let local_pref = r.local_pref; - let sr = StaticRoute4 { nexthop, prefix, vlan_id, local_pref }; + let rib_priority = r.rib_priority; + let sr = StaticRoute4 { + nexthop, + prefix, + vlan_id, + rib_priority: rib_priority + .unwrap_or(DEFAULT_RIB_PRIORITY_STATIC), + }; rq.routes.list.push(sr); } } diff --git a/sled-agent/src/bootstrap/server.rs b/sled-agent/src/bootstrap/server.rs index fe480142ca..d52da69e0e 100644 --- a/sled-agent/src/bootstrap/server.rs +++ b/sled-agent/src/bootstrap/server.rs @@ -32,7 +32,7 @@ use illumos_utils::dladm; use illumos_utils::zfs; use illumos_utils::zone; use illumos_utils::zone::Zones; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; use omicron_common::address::{Ipv6Subnet, AZ_PREFIX}; use omicron_common::ledger; use omicron_common::ledger::Ledger; diff --git a/sled-agent/src/fakes/nexus.rs b/sled-agent/src/fakes/nexus.rs index 3efd6951f9..1a40dbcfbe 100644 --- a/sled-agent/src/fakes/nexus.rs +++ b/sled-agent/src/fakes/nexus.rs @@ -12,7 +12,8 @@ use dropshot::{ endpoint, ApiDescription, FreeformBody, HttpError, HttpResponseOk, HttpResponseUpdatedNoContent, Path, RequestContext, TypedBody, }; -use internal_dns::ServiceName; +use internal_dns_types::config::DnsConfigBuilder; +use internal_dns_types::names::ServiceName; use nexus_client::types::SledAgentInfo; use omicron_common::api::external::Error; use omicron_common::api::internal::nexus::{SledVmmState, UpdateArtifactId}; @@ -168,7 +169,7 @@ pub async fn start_dns_server( nexus: &dropshot::HttpServer, ) -> dns_server::TransientServer { let dns = dns_server::TransientServer::new(log).await.unwrap(); - let mut dns_config_builder = internal_dns::DnsConfigBuilder::new(); + let mut dns_config_builder = DnsConfigBuilder::new(); let nexus_addr = match nexus.local_addr() { std::net::SocketAddr::V6(addr) => addr, diff --git a/sled-agent/src/http_entrypoints.rs b/sled-agent/src/http_entrypoints.rs index d008f4a45a..489fc9ab0d 100644 --- a/sled-agent/src/http_entrypoints.rs +++ b/sled-agent/src/http_entrypoints.rs @@ -24,8 +24,8 @@ use omicron_common::api::internal::nexus::{ DiskRuntimeState, SledVmmState, UpdateArtifactId, }; use omicron_common::api::internal::shared::{ - ResolvedVpcRouteSet, ResolvedVpcRouteState, SledIdentifiers, SwitchPorts, - VirtualNetworkInterfaceHost, + ExternalIpGatewayMap, ResolvedVpcRouteSet, ResolvedVpcRouteState, + SledIdentifiers, SwitchPorts, VirtualNetworkInterfaceHost, }; use omicron_common::disk::{ DatasetsConfig, DatasetsManagementResult, DiskVariant, @@ -258,13 +258,6 @@ impl SledAgentApi for SledAgentImpl { sa.zones_list().await.map(HttpResponseOk).map_err(HttpError::from) } - async fn omicron_zones_get( - rqctx: RequestContext, - ) -> Result, HttpError> { - let sa = rqctx.context(); - Ok(HttpResponseOk(sa.omicron_zones_list().await)) - } - async fn omicron_zones_put( rqctx: RequestContext, body: TypedBody, @@ -725,4 +718,13 @@ impl SledAgentApi for SledAgentImpl { sa.set_vpc_routes(body.into_inner())?; Ok(HttpResponseUpdatedNoContent()) } + + async fn set_eip_gateways( + request_context: RequestContext, + body: TypedBody, + ) -> Result { + let sa = request_context.context(); + sa.set_eip_gateways(body.into_inner()).await?; + Ok(HttpResponseUpdatedNoContent()) + } } diff --git a/sled-agent/src/instance.rs b/sled-agent/src/instance.rs index 35380fa666..8a4c5cf669 100644 --- a/sled-agent/src/instance.rs +++ b/sled-agent/src/instance.rs @@ -102,7 +102,7 @@ pub enum Error { InvalidHostname(&'static str), #[error("Error resolving DNS name: {0}")] - ResolveError(#[from] internal_dns::resolver::ResolveError), + ResolveError(#[from] internal_dns_resolver::ResolveError), #[error("Propolis job with ID {0} is registered but not running")] VmNotRunning(PropolisUuid), @@ -243,6 +243,9 @@ enum InstanceRequest { ip: InstanceExternalIpBody, tx: oneshot::Sender>, }, + RefreshExternalIps { + tx: oneshot::Sender>, + }, } // A small task which tracks the state of the instance, by constantly querying @@ -521,6 +524,10 @@ impl InstanceRunner { tx.send(self.delete_external_ip(&ip).await.map_err(|e| e.into())) .map_err(|_| Error::FailedSendClientClosed) }, + Some(RefreshExternalIps { tx }) => { + tx.send(self.refresh_external_ips().map_err(|e| e.into())) + .map_err(|_| Error::FailedSendClientClosed) + } None => { warn!(self.log, "Instance request channel closed; shutting down"); let mark_failed = false; @@ -574,6 +581,9 @@ impl InstanceRunner { DeleteExternalIp { tx, .. } => { tx.send(Err(Error::Terminating.into())).map_err(|_| ()) } + RefreshExternalIps { tx } => { + tx.send(Err(Error::Terminating.into())).map_err(|_| ()) + } }; } } @@ -993,6 +1003,22 @@ impl InstanceRunner { Ok(()) } + fn refresh_external_ips_inner(&mut self) -> Result<(), Error> { + let Some(primary_nic) = self.requested_nics.get(0) else { + return Err(Error::Opte(illumos_utils::opte::Error::NoPrimaryNic)); + }; + + self.port_manager.external_ips_ensure( + primary_nic.id, + primary_nic.kind, + Some(self.source_nat), + self.ephemeral_ip, + &self.floating_ips, + )?; + + Ok(()) + } + async fn delete_external_ip_inner( &mut self, ip: &InstanceExternalIpBody, @@ -1340,6 +1366,19 @@ impl Instance { .map_err(|_| Error::FailedSendChannelClosed)?; Ok(()) } + + /// Reinstalls an instance's set of external IPs within OPTE, using + /// up-to-date IP<->IGW mappings. This will not disrupt existing flows. + pub async fn refresh_external_ips( + &self, + tx: oneshot::Sender>, + ) -> Result<(), Error> { + self.tx + .send(InstanceRequest::RefreshExternalIps { tx }) + .await + .map_err(|_| Error::FailedSendChannelClosed)?; + Ok(()) + } } // TODO: Move this implementation higher. I'm just keeping it here to make the @@ -1699,6 +1738,10 @@ impl InstanceRunner { } out } + + fn refresh_external_ips(&mut self) -> Result<(), Error> { + self.refresh_external_ips_inner() + } } #[cfg(all(test, target_os = "illumos"))] @@ -1718,7 +1761,7 @@ mod tests { use illumos_utils::zone::MockZones; use illumos_utils::zone::__mock_MockZones::__boot::Context as MockZonesBootContext; use illumos_utils::zone::__mock_MockZones::__id::Context as MockZonesIdContext; - use internal_dns::resolver::Resolver; + use internal_dns_resolver::Resolver; use omicron_common::api::external::{ ByteCount, Generation, Hostname, InstanceCpuCount, }; diff --git a/sled-agent/src/instance_manager.rs b/sled-agent/src/instance_manager.rs index 585d73ac0f..af09def0c1 100644 --- a/sled-agent/src/instance_manager.rs +++ b/sled-agent/src/instance_manager.rs @@ -279,6 +279,16 @@ impl InstanceManager { rx.await? } + pub async fn refresh_external_ips(&self) -> Result<(), Error> { + let (tx, rx) = oneshot::channel(); + self.inner + .tx + .send(InstanceManagerRequest::RefreshExternalIps { tx }) + .await + .map_err(|_| Error::FailedSendInstanceManagerClosed)?; + rx.await? + } + /// Returns the last-set size of the reservoir pub fn reservoir_size(&self) -> ByteCount { self.inner.vmm_reservoir_manager.reservoir_size() @@ -364,6 +374,9 @@ enum InstanceManagerRequest { ip: InstanceExternalIpBody, tx: oneshot::Sender>, }, + RefreshExternalIps { + tx: oneshot::Sender>, + }, GetState { propolis_id: PropolisUuid, tx: oneshot::Sender>, @@ -469,6 +482,9 @@ impl InstanceManagerRunner { Some(DeleteExternalIp { propolis_id, ip, tx }) => { self.delete_external_ip(tx, propolis_id, &ip).await }, + Some(RefreshExternalIps { tx }) => { + self.refresh_external_ips(tx).await + } Some(GetState { propolis_id, tx }) => { // TODO(eliza): it could potentially be nice to // refactor this to use `tokio::sync::watch`, rather @@ -725,6 +741,31 @@ impl InstanceManagerRunner { Ok(()) } + async fn refresh_external_ips( + &self, + tx: oneshot::Sender>, + ) -> Result<(), Error> { + let mut channels = vec![]; + for (_, instance) in &self.jobs { + let (tx, rx_new) = oneshot::channel(); + instance.refresh_external_ips(tx).await?; + channels.push(rx_new); + } + + tokio::spawn(async move { + for channel in channels { + if let Err(e) = channel.await { + let _ = tx.send(Err(e.into())); + return; + } + } + + let _ = tx.send(Ok(())); + }); + + Ok(()) + } + async fn get_instance_state( &self, tx: oneshot::Sender>, diff --git a/sled-agent/src/nexus.rs b/sled-agent/src/nexus.rs index 9f7a4372aa..d1646823bb 100644 --- a/sled-agent/src/nexus.rs +++ b/sled-agent/src/nexus.rs @@ -7,8 +7,8 @@ use omicron_common::disk::DiskVariant; use omicron_uuid_kinds::SledUuid; use crate::vmm_reservoir::VmmReservoirManagerHandle; -use internal_dns::resolver::Resolver; -use internal_dns::ServiceName; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use nexus_client::types::SledAgentInfo; use omicron_common::address::NEXUS_INTERNAL_PORT; use sled_hardware::HardwareManager; @@ -47,50 +47,6 @@ pub(crate) fn make_nexus_client_with_port( ) } -pub fn d2n_params( - params: &dns_service_client::types::DnsConfigParams, -) -> nexus_client::types::DnsConfigParams { - nexus_client::types::DnsConfigParams { - generation: params.generation, - time_created: params.time_created, - zones: params.zones.iter().map(d2n_zone).collect(), - } -} - -fn d2n_zone( - zone: &dns_service_client::types::DnsConfigZone, -) -> nexus_client::types::DnsConfigZone { - nexus_client::types::DnsConfigZone { - zone_name: zone.zone_name.clone(), - records: zone - .records - .iter() - .map(|(n, r)| (n.clone(), r.iter().map(d2n_record).collect())) - .collect(), - } -} - -fn d2n_record( - record: &dns_service_client::types::DnsRecord, -) -> nexus_client::types::DnsRecord { - match record { - dns_service_client::types::DnsRecord::A(addr) => { - nexus_client::types::DnsRecord::A(*addr) - } - dns_service_client::types::DnsRecord::Aaaa(addr) => { - nexus_client::types::DnsRecord::Aaaa(*addr) - } - dns_service_client::types::DnsRecord::Srv(srv) => { - nexus_client::types::DnsRecord::Srv(nexus_client::types::Srv { - port: srv.port, - prio: srv.prio, - target: srv.target.clone(), - weight: srv.weight, - }) - } - } -} - // Although it is a bit awkward to define these conversions here, it frees us // from depending on sled_storage/sled_hardware in the nexus_client crate. diff --git a/sled-agent/src/rack_setup/plan/service.rs b/sled-agent/src/rack_setup/plan/service.rs index 2dad436a18..fb85c48bdf 100644 --- a/sled-agent/src/rack_setup/plan/service.rs +++ b/sled-agent/src/rack_setup/plan/service.rs @@ -5,10 +5,11 @@ //! Plan generation for "where should services be initialized". use camino::Utf8PathBuf; -use dns_service_client::types::DnsConfigParams; use illumos_utils::zpool::ZpoolName; -use internal_dns::config::{Host, Zone}; -use internal_dns::ServiceName; +use internal_dns_types::config::{ + DnsConfigBuilder, DnsConfigParams, Host, Zone, +}; +use internal_dns_types::names::ServiceName; use nexus_sled_agent_shared::inventory::{ Inventory, OmicronZoneDataset, SledRole, }; @@ -37,8 +38,10 @@ use omicron_common::disk::{ }; use omicron_common::ledger::{self, Ledger, Ledgerable}; use omicron_common::policy::{ - BOUNDARY_NTP_REDUNDANCY, COCKROACHDB_REDUNDANCY, INTERNAL_DNS_REDUNDANCY, - NEXUS_REDUNDANCY, RESERVED_INTERNAL_DNS_REDUNDANCY, + BOUNDARY_NTP_REDUNDANCY, COCKROACHDB_REDUNDANCY, + CRUCIBLE_PANTRY_REDUNDANCY, INTERNAL_DNS_REDUNDANCY, NEXUS_REDUNDANCY, + OXIMETER_REDUNDANCY, RESERVED_INTERNAL_DNS_REDUNDANCY, + SINGLE_NODE_CLICKHOUSE_REDUNDANCY, }; use omicron_uuid_kinds::{ ExternalIpUuid, GenericUuid, OmicronZoneUuid, SledUuid, ZpoolUuid, @@ -60,36 +63,7 @@ use std::num::Wrapping; use thiserror::Error; use uuid::Uuid; -// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove -// when Nexus provisions Oximeter. -const OXIMETER_COUNT: usize = 1; -// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove -// when Nexus provisions Clickhouse. -// TODO(https://github.com/oxidecomputer/omicron/issues/4000): Use -// omicron_common::policy::CLICKHOUSE_SERVER_REDUNDANCY once we enable -// replicated ClickHouse. -// Set to 0 when testing replicated ClickHouse. -const CLICKHOUSE_COUNT: usize = 1; -// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove -// when Nexus provisions Clickhouse keeper. -// TODO(https://github.com/oxidecomputer/omicron/issues/4000): Use -// omicron_common::policy::CLICKHOUSE_KEEPER_REDUNDANCY once we enable -// replicated ClickHouse -// Set to 3 when testing replicated ClickHouse. -const CLICKHOUSE_KEEPER_COUNT: usize = 0; -// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove -// when Nexus provisions Clickhouse server. -// TODO(https://github.com/oxidecomputer/omicron/issues/4000): Use -// omicron_common::policy::CLICKHOUSE_SERVER_REDUNDANCY once we enable -// replicated ClickHouse. -// Set to 2 when testing replicated ClickHouse -const CLICKHOUSE_SERVER_COUNT: usize = 0; -// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove. -// when Nexus provisions Crucible. const MINIMUM_U2_COUNT: usize = 3; -// TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove. -// when Nexus provisions the Pantry. -const PANTRY_COUNT: usize = 3; /// Describes errors which may occur while generating a plan for services. #[derive(Error, Debug)] @@ -409,7 +383,7 @@ impl Plan { config: &Config, mut sled_info: Vec, ) -> Result { - let mut dns_builder = internal_dns::DnsConfigBuilder::new(); + let mut dns_builder = DnsConfigBuilder::new(); let mut svc_port_builder = ServicePortBuilder::new(config); // Scrimlets get DNS records for running Dendrite. @@ -658,7 +632,7 @@ impl Plan { // Provision Oximeter zones, continuing to stripe across sleds. // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove - for _ in 0..OXIMETER_COUNT { + for _ in 0..OXIMETER_REDUNDANCY { let sled = { let which_sled = sled_allocator.next().ok_or(PlanError::NotEnoughSleds)?; @@ -695,7 +669,7 @@ impl Plan { // Provision Clickhouse zones, continuing to stripe across sleds. // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove - for _ in 0..CLICKHOUSE_COUNT { + for _ in 0..SINGLE_NODE_CLICKHOUSE_REDUNDANCY { let sled = { let which_sled = sled_allocator.next().ok_or(PlanError::NotEnoughSleds)?; @@ -732,93 +706,9 @@ impl Plan { }); } - // Provision Clickhouse server zones, continuing to stripe across sleds. - // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove - // Temporary linter rule until replicated Clickhouse is enabled - #[allow(clippy::reversed_empty_ranges)] - for _ in 0..CLICKHOUSE_SERVER_COUNT { - let sled = { - let which_sled = - sled_allocator.next().ok_or(PlanError::NotEnoughSleds)?; - &mut sled_info[which_sled] - }; - let id = OmicronZoneUuid::new_v4(); - let ip = sled.addr_alloc.next().expect("Not enough addrs"); - // TODO: This may need to be a different port if/when to have single node - // and replicated running side by side as per stage 1 of RFD 468. - let http_port = omicron_common::address::CLICKHOUSE_HTTP_PORT; - let http_address = SocketAddrV6::new(ip, http_port, 0, 0); - dns_builder - .host_zone_clickhouse( - id, - ip, - ServiceName::ClickhouseServer, - http_port, - ) - .unwrap(); - let dataset_name = - sled.alloc_dataset_from_u2s(DatasetKind::ClickhouseServer)?; - let filesystem_pool = Some(dataset_name.pool().clone()); - sled.request.zones.push(BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id, - underlay_address: ip, - zone_type: BlueprintZoneType::ClickhouseServer( - blueprint_zone_type::ClickhouseServer { - address: http_address, - dataset: OmicronZoneDataset { - pool_name: dataset_name.pool().clone(), - }, - }, - ), - filesystem_pool, - }); - } - - // Provision Clickhouse Keeper zones, continuing to stripe across sleds. - // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove - // Temporary linter rule until replicated Clickhouse is enabled - #[allow(clippy::reversed_empty_ranges)] - for _ in 0..CLICKHOUSE_KEEPER_COUNT { - let sled = { - let which_sled = - sled_allocator.next().ok_or(PlanError::NotEnoughSleds)?; - &mut sled_info[which_sled] - }; - let id = OmicronZoneUuid::new_v4(); - let ip = sled.addr_alloc.next().expect("Not enough addrs"); - let port = omicron_common::address::CLICKHOUSE_KEEPER_TCP_PORT; - let address = SocketAddrV6::new(ip, port, 0, 0); - dns_builder - .host_zone_with_one_backend( - id, - ip, - ServiceName::ClickhouseKeeper, - port, - ) - .unwrap(); - let dataset_name = - sled.alloc_dataset_from_u2s(DatasetKind::ClickhouseKeeper)?; - let filesystem_pool = Some(dataset_name.pool().clone()); - sled.request.zones.push(BlueprintZoneConfig { - disposition: BlueprintZoneDisposition::InService, - id, - underlay_address: ip, - zone_type: BlueprintZoneType::ClickhouseKeeper( - blueprint_zone_type::ClickhouseKeeper { - address, - dataset: OmicronZoneDataset { - pool_name: dataset_name.pool().clone(), - }, - }, - ), - filesystem_pool, - }); - } - // Provision Crucible Pantry zones, continuing to stripe across sleds. // TODO(https://github.com/oxidecomputer/omicron/issues/732): Remove - for _ in 0..PANTRY_COUNT { + for _ in 0..CRUCIBLE_PANTRY_REDUNDANCY { let sled = { let which_sled = sled_allocator.next().ok_or(PlanError::NotEnoughSleds)?; diff --git a/sled-agent/src/rack_setup/service.rs b/sled-agent/src/rack_setup/service.rs index f4edf7f024..5124096c1e 100644 --- a/sled-agent/src/rack_setup/service.rs +++ b/sled-agent/src/rack_setup/service.rs @@ -70,7 +70,6 @@ use crate::bootstrap::early_networking::{ EarlyNetworkSetup, EarlyNetworkSetupError, }; use crate::bootstrap::rss_handle::BootstrapAgentHandle; -use crate::nexus::d2n_params; use crate::rack_setup::plan::service::{ Plan as ServicePlan, PlanError as ServicePlanError, }; @@ -81,8 +80,9 @@ use anyhow::{bail, Context}; use bootstore::schemes::v0 as bootstore; use camino::Utf8PathBuf; use chrono::Utc; -use internal_dns::resolver::{DnsError, Resolver as DnsResolver}; -use internal_dns::ServiceName; +use dns_service_client::DnsError; +use internal_dns_resolver::Resolver as DnsResolver; +use internal_dns_types::names::ServiceName; use nexus_client::{ types as NexusTypes, Client as NexusClient, Error as NexusError, }; @@ -200,7 +200,7 @@ pub enum SetupServiceError { Dendrite(String), #[error("Error during DNS lookup: {0}")] - DnsResolver(#[from] internal_dns::resolver::ResolveError), + DnsResolver(#[from] internal_dns_resolver::ResolveError), #[error("Bootstore error: {0}")] Bootstore(#[from] bootstore::NodeRequestError), @@ -778,7 +778,7 @@ impl ServiceInner { destination: r.destination, nexthop: r.nexthop, vlan_id: r.vlan_id, - local_pref: r.local_pref, + rib_priority: r.rib_priority, }) .collect(), addresses: config @@ -931,7 +931,7 @@ impl ServiceInner { datasets, internal_services_ip_pool_ranges, certs: config.external_certificates.clone(), - internal_dns_zone_config: d2n_params(&service_plan.dns_config), + internal_dns_zone_config: service_plan.dns_config.clone(), external_dns_zone_name: config.external_dns_zone_name.clone(), recovery_silo: config.recovery_silo.clone(), rack_network_config, @@ -1486,6 +1486,9 @@ pub(crate) fn build_initial_blueprint_from_sled_configs( cockroachdb_fingerprint: String::new(), cockroachdb_setting_preserve_downgrade: CockroachDbPreserveDowngrade::DoNotModify, + // We do not create clickhouse clusters in RSS. We create them via + // reconfigurator only. + clickhouse_cluster_config: None, time_created: Utc::now(), creator: "RSS".to_string(), comment: "initial blueprint from rack setup".to_string(), @@ -1590,7 +1593,8 @@ mod test { use super::{Config, OmicronZonesConfigGenerator}; use crate::rack_setup::plan::service::{Plan as ServicePlan, SledInfo}; use nexus_sled_agent_shared::inventory::{ - Baseboard, Inventory, InventoryDisk, OmicronZoneType, SledRole, + Baseboard, Inventory, InventoryDisk, OmicronZoneType, + OmicronZonesConfig, SledRole, }; use omicron_common::{ address::{get_sled_address, Ipv6Subnet, SLED_PREFIX}, @@ -1617,6 +1621,10 @@ mod test { usable_hardware_threads: 32, usable_physical_ram: ByteCount::from_gibibytes_u32(16), reservoir_size: ByteCount::from_gibibytes_u32(0), + omicron_zones: OmicronZonesConfig { + generation: Generation::new(), + zones: vec![], + }, disks: (0..u2_count) .map(|i| InventoryDisk { identity: DiskIdentity { diff --git a/sled-agent/src/server.rs b/sled-agent/src/server.rs index b8deb2f1cb..6709e2d75f 100644 --- a/sled-agent/src/server.rs +++ b/sled-agent/src/server.rs @@ -10,7 +10,7 @@ use super::sled_agent::SledAgent; use crate::long_running_tasks::LongRunningTaskHandles; use crate::nexus::make_nexus_client; use crate::services::ServiceManager; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; use omicron_uuid_kinds::SledUuid; use sled_agent_types::sled::StartSledAgentRequest; use slog::Logger; diff --git a/sled-agent/src/services.rs b/sled-agent/src/services.rs index 506995d1c7..36360ddeae 100644 --- a/sled-agent/src/services.rs +++ b/sled-agent/src/services.rs @@ -57,9 +57,9 @@ use illumos_utils::zfs::ZONE_ZFS_RAMDISK_DATASET_MOUNTPOINT; use illumos_utils::zone::AddressRequest; use illumos_utils::zpool::{PathInPool, ZpoolName}; use illumos_utils::{execute, PFEXEC}; -use internal_dns::names::BOUNDARY_NTP_DNS_NAME; -use internal_dns::names::DNS_ZONE; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::BOUNDARY_NTP_DNS_NAME; +use internal_dns_types::names::DNS_ZONE; use itertools::Itertools; use nexus_config::{ConfigDropshotWithTls, DeploymentConfig}; use nexus_sled_agent_shared::inventory::{ @@ -267,7 +267,7 @@ pub enum Error { ExecutionError(#[from] illumos_utils::ExecutionError), #[error("Error resolving DNS name: {0}")] - ResolveError(#[from] internal_dns::resolver::ResolveError), + ResolveError(#[from] internal_dns_resolver::ResolveError), #[error("Serde error: {0}")] SerdeError(#[from] serde_json::Error), @@ -1650,7 +1650,6 @@ impl ServiceManager { }; let listen_addr = *underlay_address; - let listen_port = CLICKHOUSE_HTTP_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( Some(&info.underlay_address), @@ -1660,19 +1659,10 @@ impl ServiceManager { let dns_service = Self::dns_install(info, None, None).await?; - let config = PropertyGroupBuilder::new("config") - .add_property( - "listen_addr", - "astring", - listen_addr.to_string(), - ) - .add_property("listen_port", "astring", listen_port) - .add_property("store", "astring", "/data"); - let clickhouse_server_service = + let disabled_clickhouse_server_service = ServiceBuilder::new("oxide/clickhouse_server") .add_instance( - ServiceInstanceBuilder::new("default") - .add_property_group(config), + ServiceInstanceBuilder::new("default").disable(), ); let admin_address = SocketAddr::new( @@ -1705,7 +1695,7 @@ impl ServiceManager { let profile = ProfileBuilder::new("omicron") .add_service(nw_setup_service) .add_service(disabled_ssh_service) - .add_service(clickhouse_server_service) + .add_service(disabled_clickhouse_server_service) .add_service(dns_service) .add_service(enabled_dns_client_service) .add_service(clickhouse_admin_service); @@ -1735,7 +1725,6 @@ impl ServiceManager { }; let listen_addr = *underlay_address; - let listen_port = &CLICKHOUSE_KEEPER_TCP_PORT.to_string(); let nw_setup_service = Self::zone_network_setup_install( Some(&info.underlay_address), @@ -1745,19 +1734,10 @@ impl ServiceManager { let dns_service = Self::dns_install(info, None, None).await?; - let config = PropertyGroupBuilder::new("config") - .add_property( - "listen_addr", - "astring", - listen_addr.to_string(), - ) - .add_property("listen_port", "astring", listen_port) - .add_property("store", "astring", "/data"); - let clickhouse_keeper_service = + let disaled_clickhouse_keeper_service = ServiceBuilder::new("oxide/clickhouse_keeper") .add_instance( - ServiceInstanceBuilder::new("default") - .add_property_group(config), + ServiceInstanceBuilder::new("default").disable(), ); let admin_address = SocketAddr::new( @@ -1790,7 +1770,7 @@ impl ServiceManager { let profile = ProfileBuilder::new("omicron") .add_service(nw_setup_service) .add_service(disabled_ssh_service) - .add_service(clickhouse_keeper_service) + .add_service(disaled_clickhouse_keeper_service) .add_service(dns_service) .add_service(enabled_dns_client_service) .add_service(clickhouse_admin_service); diff --git a/sled-agent/src/sim/http_entrypoints.rs b/sled-agent/src/sim/http_entrypoints.rs index 53d209725d..ca7f5e3410 100644 --- a/sled-agent/src/sim/http_entrypoints.rs +++ b/sled-agent/src/sim/http_entrypoints.rs @@ -25,6 +25,7 @@ use nexus_sled_agent_shared::inventory::{Inventory, OmicronZonesConfig}; use omicron_common::api::internal::nexus::DiskRuntimeState; use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::api::internal::nexus::UpdateArtifactId; +use omicron_common::api::internal::shared::ExternalIpGatewayMap; use omicron_common::api::internal::shared::SledIdentifiers; use omicron_common::api::internal::shared::VirtualNetworkInterfaceHost; use omicron_common::api::internal::shared::{ @@ -325,13 +326,6 @@ impl SledAgentApi for SledAgentSimImpl { Ok(HttpResponseOk(sa.omicron_physical_disks_list().await?)) } - async fn omicron_zones_get( - rqctx: RequestContext, - ) -> Result, HttpError> { - let sa = rqctx.context(); - Ok(HttpResponseOk(sa.omicron_zones_list().await)) - } - async fn omicron_zones_put( rqctx: RequestContext, body: TypedBody, @@ -365,6 +359,15 @@ impl SledAgentApi for SledAgentSimImpl { Ok(HttpResponseUpdatedNoContent()) } + async fn set_eip_gateways( + rqctx: RequestContext, + _body: TypedBody, + ) -> Result { + let _sa = rqctx.context(); + // sa.set_vpc_routes(body.into_inner()).await; + Ok(HttpResponseUpdatedNoContent()) + } + // --- Unimplemented endpoints --- async fn zone_bundle_list_all( diff --git a/sled-agent/src/sim/http_entrypoints_pantry.rs b/sled-agent/src/sim/http_entrypoints_pantry.rs index a93cb6fca9..0348cc3099 100644 --- a/sled-agent/src/sim/http_entrypoints_pantry.rs +++ b/sled-agent/src/sim/http_entrypoints_pantry.rs @@ -23,7 +23,10 @@ pub fn api() -> CruciblePantryApiDescription { fn register_endpoints( api: &mut CruciblePantryApiDescription, ) -> Result<(), ApiDescriptionRegisterError> { + api.register(pantry_status)?; + api.register(volume_status)?; api.register(attach)?; + api.register(attach_activate_background)?; api.register(is_job_finished)?; api.register(job_result_ok)?; api.register(import_from_url)?; @@ -46,11 +49,64 @@ pub fn api() -> CruciblePantryApiDescription { // pantry here, to avoid skew. However, this was wholesale copied from the // crucible repo! +#[derive(Serialize, JsonSchema)] +pub struct PantryStatus { + /// Which volumes does this Pantry know about? Note this may include volumes + /// that are no longer active, and haven't been garbage collected yet. + pub volumes: Vec, + + /// How many job handles? + pub num_job_handles: usize, +} + +/// Get the Pantry's status +#[endpoint { + method = GET, + path = "/crucible/pantry/0", +}] +async fn pantry_status( + rc: RequestContext>, +) -> Result, HttpError> { + let pantry = rc.context(); + + let status = pantry.status().await?; + + Ok(HttpResponseOk(status)) +} + #[derive(Deserialize, JsonSchema)] struct VolumePath { pub id: String, } +#[derive(Clone, Deserialize, Serialize, JsonSchema)] +pub struct VolumeStatus { + /// Is the Volume currently active? + pub active: bool, + + /// Has the Pantry ever seen this Volume active? + pub seen_active: bool, + + /// How many job handles are there for this Volume? + pub num_job_handles: usize, +} + +/// Get a current Volume's status +#[endpoint { + method = GET, + path = "/crucible/pantry/0/volume/{id}", +}] +async fn volume_status( + rc: RequestContext>, + path: TypedPath, +) -> Result, HttpError> { + let path = path.into_inner(); + let pantry = rc.context(); + + let status = pantry.volume_status(path.id.clone()).await?; + Ok(HttpResponseOk(status)) +} + #[derive(Deserialize, JsonSchema)] struct AttachRequest { pub volume_construction_request: VolumeConstructionRequest, @@ -84,6 +140,38 @@ async fn attach( Ok(HttpResponseOk(AttachResult { id: path.id })) } +#[derive(Deserialize, JsonSchema)] +struct AttachBackgroundRequest { + pub volume_construction_request: VolumeConstructionRequest, + pub job_id: String, +} + +/// Construct a volume from a VolumeConstructionRequest, storing the result in +/// the Pantry. Activate in a separate job so as not to block the request. +#[endpoint { + method = POST, + path = "/crucible/pantry/0/volume/{id}/background", +}] +async fn attach_activate_background( + rc: RequestContext>, + path: TypedPath, + body: TypedBody, +) -> Result { + let path = path.into_inner(); + let body = body.into_inner(); + let pantry = rc.context(); + + pantry + .attach_activate_background( + path.id.clone(), + body.job_id, + body.volume_construction_request, + ) + .await?; + + Ok(HttpResponseUpdatedNoContent()) +} + #[derive(Deserialize, JsonSchema)] struct JobPath { pub id: String, diff --git a/sled-agent/src/sim/server.rs b/sled-agent/src/sim/server.rs index 03efa56369..5ebe56ae19 100644 --- a/sled-agent/src/sim/server.rs +++ b/sled-agent/src/sim/server.rs @@ -8,7 +8,6 @@ use super::config::Config; use super::http_entrypoints::api as http_api; use super::sled_agent::SledAgent; use super::storage::PantryServer; -use crate::nexus::d2n_params; use crate::nexus::NexusClient; use crate::rack_setup::service::build_initial_blueprint_from_sled_configs; use crate::rack_setup::SledConfig; @@ -19,7 +18,9 @@ use crate::rack_setup::{ use anyhow::anyhow; use crucible_agent_client::types::State as RegionState; use illumos_utils::zpool::ZpoolName; -use internal_dns::ServiceName; +use internal_dns_types::config::DnsConfigBuilder; +use internal_dns_types::names::ServiceName; +use internal_dns_types::names::DNS_ZONE_EXTERNAL_TESTING; use nexus_client::types as NexusTypes; use nexus_client::types::{IpRange, Ipv4Range, Ipv6Range}; use nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; @@ -334,7 +335,7 @@ pub async fn run_standalone_server( } else { dns_server::TransientServer::new(&log).await? }; - let mut dns_config_builder = internal_dns::DnsConfigBuilder::new(); + let mut dns_config_builder = DnsConfigBuilder::new(); // Start the Crucible Pantry let pantry_server = server.start_pantry().await; @@ -554,9 +555,8 @@ pub async fn run_standalone_server( datasets, internal_services_ip_pool_ranges, certs, - internal_dns_zone_config: d2n_params(&dns_config), - external_dns_zone_name: internal_dns::names::DNS_ZONE_EXTERNAL_TESTING - .to_owned(), + internal_dns_zone_config: dns_config, + external_dns_zone_name: DNS_ZONE_EXTERNAL_TESTING.to_owned(), recovery_silo, external_port_count: NexusTypes::ExternalPortDiscovery::Static( HashMap::new(), diff --git a/sled-agent/src/sim/sled_agent.rs b/sled-agent/src/sim/sled_agent.rs index 0652c021cb..321e9cc34f 100644 --- a/sled-agent/src/sim/sled_agent.rs +++ b/sled-agent/src/sim/sled_agent.rs @@ -839,6 +839,7 @@ impl SledAgent { self.config.hardware.reservoir_ram, ) .context("reservoir_size")?, + omicron_zones: self.fake_zones.lock().await.clone(), disks: storage .physical_disks() .values() @@ -957,7 +958,12 @@ impl SledAgent { { continue; } - _ => {} + _ => { + println!( + "sled {} successfully installed routes {new:?}", + self.id + ); + } }; routes.insert( diff --git a/sled-agent/src/sim/storage.rs b/sled-agent/src/sim/storage.rs index 5bbafa2ac3..589ba87700 100644 --- a/sled-agent/src/sim/storage.rs +++ b/sled-agent/src/sim/storage.rs @@ -9,6 +9,8 @@ //! through Nexus' external API. use crate::sim::http_entrypoints_pantry::ExpectedDigest; +use crate::sim::http_entrypoints_pantry::PantryStatus; +use crate::sim::http_entrypoints_pantry::VolumeStatus; use crate::sim::SledAgent; use anyhow::{self, bail, Result}; use chrono::prelude::*; @@ -1152,10 +1154,17 @@ impl Storage { } } +pub struct PantryVolume { + vcr: VolumeConstructionRequest, // Please rewind! + status: VolumeStatus, + activate_job: Option, +} + /// Simulated crucible pantry pub struct Pantry { pub id: OmicronZoneUuid, - vcrs: Mutex>, // Please rewind! + /// Map Volume UUID to PantryVolume struct + volumes: Mutex>, sled_agent: Arc, jobs: Mutex>, } @@ -1164,19 +1173,26 @@ impl Pantry { pub fn new(sled_agent: Arc) -> Self { Self { id: OmicronZoneUuid::new_v4(), - vcrs: Mutex::new(HashMap::default()), + volumes: Mutex::new(HashMap::default()), sled_agent, jobs: Mutex::new(HashSet::default()), } } + pub async fn status(&self) -> Result { + Ok(PantryStatus { + volumes: self.volumes.lock().await.keys().cloned().collect(), + num_job_handles: self.jobs.lock().await.len(), + }) + } + pub async fn entry( &self, volume_id: String, ) -> Result { - let vcrs = self.vcrs.lock().await; - match vcrs.get(&volume_id) { - Some(entry) => Ok(entry.clone()), + let volumes = self.volumes.lock().await; + match volumes.get(&volume_id) { + Some(entry) => Ok(entry.vcr.clone()), None => Err(HttpError::for_not_found(None, volume_id)), } @@ -1187,11 +1203,100 @@ impl Pantry { volume_id: String, volume_construction_request: VolumeConstructionRequest, ) -> Result<()> { - let mut vcrs = self.vcrs.lock().await; - vcrs.insert(volume_id, volume_construction_request); + let mut volumes = self.volumes.lock().await; + + volumes.insert( + volume_id, + PantryVolume { + vcr: volume_construction_request, + status: VolumeStatus { + active: true, + seen_active: true, + num_job_handles: 0, + }, + activate_job: None, + }, + ); + Ok(()) } + pub async fn attach_activate_background( + &self, + volume_id: String, + activate_job_id: String, + volume_construction_request: VolumeConstructionRequest, + ) -> Result<(), HttpError> { + let mut volumes = self.volumes.lock().await; + let mut jobs = self.jobs.lock().await; + + volumes.insert( + volume_id, + PantryVolume { + vcr: volume_construction_request, + status: VolumeStatus { + active: false, + seen_active: false, + num_job_handles: 1, + }, + activate_job: Some(activate_job_id.clone()), + }, + ); + + jobs.insert(activate_job_id); + + Ok(()) + } + + pub async fn activate_background_attachment( + &self, + volume_id: String, + ) -> Result { + let activate_job = { + let volumes = self.volumes.lock().await; + volumes.get(&volume_id).unwrap().activate_job.clone().unwrap() + }; + + let mut status = self.volume_status(volume_id.clone()).await?; + + status.active = true; + status.seen_active = true; + + self.update_volume_status(volume_id, status).await?; + + Ok(activate_job) + } + + pub async fn volume_status( + &self, + volume_id: String, + ) -> Result { + let volumes = self.volumes.lock().await; + + match volumes.get(&volume_id) { + Some(pantry_volume) => Ok(pantry_volume.status.clone()), + + None => Err(HttpError::for_not_found(None, volume_id)), + } + } + + pub async fn update_volume_status( + &self, + volume_id: String, + status: VolumeStatus, + ) -> Result<(), HttpError> { + let mut volumes = self.volumes.lock().await; + + match volumes.get_mut(&volume_id) { + Some(pantry_volume) => { + pantry_volume.status = status; + Ok(()) + } + + None => Err(HttpError::for_not_found(None, volume_id)), + } + } + pub async fn is_job_finished( &self, job_id: String, @@ -1240,11 +1345,11 @@ impl Pantry { // the simulated instance ensure, then call // [`instance_issue_disk_snapshot_request`] as the snapshot logic is the // same. - let vcrs = self.vcrs.lock().await; - let volume_construction_request = vcrs.get(&volume_id).unwrap(); + let volumes = self.volumes.lock().await; + let volume_construction_request = &volumes.get(&volume_id).unwrap().vcr; self.sled_agent - .map_disk_ids_to_region_ids(&volume_construction_request) + .map_disk_ids_to_region_ids(volume_construction_request) .await?; self.sled_agent @@ -1329,8 +1434,8 @@ impl Pantry { } pub async fn detach(&self, volume_id: String) -> Result<()> { - let mut vcrs = self.vcrs.lock().await; - vcrs.remove(&volume_id); + let mut volumes = self.volumes.lock().await; + volumes.remove(&volume_id); Ok(()) } } diff --git a/sled-agent/src/sled_agent.rs b/sled-agent/src/sled_agent.rs index 78a61c894e..4a4be08f76 100644 --- a/sled-agent/src/sled_agent.rs +++ b/sled-agent/src/sled_agent.rs @@ -41,9 +41,9 @@ use omicron_common::address::{ use omicron_common::api::external::{ByteCount, ByteCountRangeError, Vni}; use omicron_common::api::internal::nexus::SledVmmState; use omicron_common::api::internal::shared::{ - HostPortConfig, RackNetworkConfig, ResolvedVpcFirewallRule, - ResolvedVpcRouteSet, ResolvedVpcRouteState, SledIdentifiers, - VirtualNetworkInterfaceHost, + ExternalIpGatewayMap, HostPortConfig, RackNetworkConfig, + ResolvedVpcFirewallRule, ResolvedVpcRouteSet, ResolvedVpcRouteState, + SledIdentifiers, VirtualNetworkInterfaceHost, }; use omicron_common::api::{ internal::nexus::DiskRuntimeState, internal::nexus::UpdateArtifactId, @@ -145,7 +145,7 @@ pub enum Error { Hardware(String), #[error("Error resolving DNS name: {0}")] - ResolveError(#[from] internal_dns::resolver::ResolveError), + ResolveError(#[from] internal_dns_resolver::ResolveError), #[error(transparent)] ZpoolList(#[from] illumos_utils::zpool::ListError), @@ -917,11 +917,6 @@ impl SledAgent { Ok(disk_result) } - /// List the Omicron zone configuration that's currently running - pub async fn omicron_zones_list(&self) -> OmicronZonesConfig { - self.inner.services.omicron_zones_list().await - } - /// Ensures that the specific set of Omicron zones are running as configured /// (and that no other zones are running) pub async fn omicron_zones_ensure( @@ -1177,6 +1172,42 @@ impl SledAgent { self.inner.port_manager.vpc_routes_ensure(routes).map_err(Error::from) } + pub async fn set_eip_gateways( + &self, + mappings: ExternalIpGatewayMap, + ) -> Result<(), Error> { + info!( + self.log, + "IGW mapping received"; + "values" => ?mappings + ); + let changed = self.inner.port_manager.set_eip_gateways(mappings); + + // TODO(kyle) + // There is a substantial downside to this approach, which is that + // we can currently only do correct Internet Gateway association for + // *Instances* -- sled agent does not remember the ExtIPs associated + // with Services or with Probes. + // + // In practice, services should not have more than one IGW. Not having + // identical source IP selection for Probes is a little sad, though. + // OPTE will follow the old (single-IGW) behaviour when no mappings + // are installed. + // + // My gut feeling is that the correct place for External IPs to + // live is on each NetworkInterface, which makes it far simpler for + // nexus to administer and add/remove IPs on *all* classes of port + // via RPW. This is how we would make this correct in general. + // My understanding is that NetworkInterface's schema makes its way into + // the ledger, and I'm not comfortable redoing that this close to a release. + if changed { + self.inner.instances.refresh_external_ips().await?; + info!(self.log, "IGW mapping changed; external IPs refreshed"); + } + + Ok(()) + } + pub(crate) fn storage(&self) -> &StorageHandle { &self.inner.storage } @@ -1225,7 +1256,10 @@ impl SledAgent { let mut disks = vec![]; let mut zpools = vec![]; let mut datasets = vec![]; - let all_disks = self.storage().get_latest_disks().await; + let (all_disks, omicron_zones) = tokio::join!( + self.storage().get_latest_disks(), + self.inner.services.omicron_zones_list() + ); for (identity, variant, slot, firmware) in all_disks.iter_all() { disks.push(InventoryDisk { identity: identity.clone(), @@ -1309,6 +1343,7 @@ impl SledAgent { usable_hardware_threads, usable_physical_ram: ByteCount::try_from(usable_physical_ram)?, reservoir_size, + omicron_zones, disks, zpools, datasets, diff --git a/sled-agent/tests/integration_tests/early_network.rs b/sled-agent/tests/integration_tests/early_network.rs index 9b69975054..c0ecc09f12 100644 --- a/sled-agent/tests/integration_tests/early_network.rs +++ b/sled-agent/tests/integration_tests/early_network.rs @@ -126,7 +126,7 @@ fn current_config_example() -> (&'static str, EarlyNetworkConfig) { destination: "10.1.9.32/16".parse().unwrap(), nexthop: "10.1.9.32".parse().unwrap(), vlan_id: None, - local_pref: None, + rib_priority: None, }], addresses: vec!["2001:db8::/96".parse().unwrap()], switch: SwitchLocation::Switch0, diff --git a/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json b/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json index 2da814042d..270d926ea8 100644 --- a/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json +++ b/sled-agent/tests/output/new-rss-sled-plans/madrid-rss-sled-plan.json @@ -129,7 +129,7 @@ "destination": "0.0.0.0/0", "nexthop": "172.20.15.33", "vlan_id": null, - "local_pref": null + "rib_priority": null } ], "addresses": [ @@ -152,7 +152,7 @@ "destination": "0.0.0.0/0", "nexthop": "172.20.15.33", "vlan_id": null, - "local_pref": null + "rib_priority": null } ], "addresses": [ diff --git a/sled-agent/types/src/early_networking.rs b/sled-agent/types/src/early_networking.rs index 755033dc23..46ceb2dbbf 100644 --- a/sled-agent/types/src/early_networking.rs +++ b/sled-agent/types/src/early_networking.rs @@ -323,8 +323,8 @@ pub mod back_compat { pub uplink_cidr: Ipv4Net, /// VLAN id to use for uplink pub uplink_vid: Option, - /// Local preference - pub local_pref: Option, + /// RIB Priority + pub rib_priority: Option, } impl From for PortConfigV2 { @@ -334,7 +334,7 @@ pub mod back_compat { destination: "0.0.0.0/0".parse().unwrap(), nexthop: value.gateway_ip.into(), vlan_id: value.uplink_vid, - local_pref: value.local_pref, + rib_priority: value.rib_priority, }], addresses: vec![UplinkAddressConfig { address: value.uplink_cidr.into(), @@ -477,7 +477,7 @@ mod tests { uplink_port_fec: PortFec::None, uplink_cidr: "192.168.0.1/16".parse().unwrap(), uplink_vid: None, - local_pref: None, + rib_priority: None, }], }), }; @@ -507,7 +507,7 @@ mod tests { destination: "0.0.0.0/0".parse().unwrap(), nexthop: uplink.gateway_ip.into(), vlan_id: None, - local_pref: None, + rib_priority: None, }], addresses: vec![UplinkAddressConfig { address: uplink.uplink_cidr.into(), @@ -553,7 +553,7 @@ mod tests { destination: "0.0.0.0/0".parse().unwrap(), nexthop: "192.168.0.2".parse().unwrap(), vlan_id: None, - local_pref: None, + rib_priority: None, }], addresses: vec!["192.168.0.1/16".parse().unwrap()], switch: SwitchLocation::Switch0, diff --git a/smf/clickhouse-admin/config.toml b/smf/clickhouse-admin-keeper/config.toml similarity index 100% rename from smf/clickhouse-admin/config.toml rename to smf/clickhouse-admin-keeper/config.toml diff --git a/smf/clickhouse-admin/manifest.xml b/smf/clickhouse-admin-keeper/manifest.xml similarity index 73% rename from smf/clickhouse-admin/manifest.xml rename to smf/clickhouse-admin-keeper/manifest.xml index 379c738f5d..46b0a33e3c 100644 --- a/smf/clickhouse-admin/manifest.xml +++ b/smf/clickhouse-admin-keeper/manifest.xml @@ -1,9 +1,9 @@ - + - + @@ -35,10 +35,10 @@ diff --git a/smf/clickhouse-admin-server/config.toml b/smf/clickhouse-admin-server/config.toml new file mode 100644 index 0000000000..86ee2c5d4b --- /dev/null +++ b/smf/clickhouse-admin-server/config.toml @@ -0,0 +1,10 @@ +[dropshot] +# 1 MiB; we don't expect any requests of more than nominal size. +request_body_max_bytes = 1048576 + +[log] +# Show log messages of this level and more severe +level = "info" +mode = "file" +path = "/dev/stdout" +if_exists = "append" diff --git a/smf/clickhouse-admin-server/manifest.xml b/smf/clickhouse-admin-server/manifest.xml new file mode 100644 index 0000000000..0b72b56323 --- /dev/null +++ b/smf/clickhouse-admin-server/manifest.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/smf/clickhouse/config.xml b/smf/clickhouse/config.xml new file mode 100644 index 0000000000..b5b1f9c17f --- /dev/null +++ b/smf/clickhouse/config.xml @@ -0,0 +1,41 @@ + + + trace + true + /var/tmp/clickhouse-server.log + /var/tmp/clickhouse-server.errlog + + + + system + query_log
+ Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 7 DAY + 10000 + + + true + + 9000 + + + + + + + ::/0 + + + default + default + 1 + + + + + + + + + + + diff --git a/smf/clickhouse/method_script.sh b/smf/clickhouse/method_script.sh index bb5dd960a1..27864cb1ed 100755 --- a/smf/clickhouse/method_script.sh +++ b/smf/clickhouse/method_script.sh @@ -11,12 +11,11 @@ LISTEN_PORT="$(svcprop -c -p config/listen_port "${SMF_FMRI}")" DATASTORE="$(svcprop -c -p config/store "${SMF_FMRI}")" args=( -"--log-file" "/var/tmp/clickhouse-server.log" -"--errorlog-file" "/var/tmp/clickhouse-server.errlog" +"--config-file" "/opt/oxide/clickhouse/config.xml" "--" "--path" "${DATASTORE}" "--listen_host" "$LISTEN_ADDR" "--http_port" "$LISTEN_PORT" ) -exec /opt/oxide/clickhouse/clickhouse server "${args[@]}" & \ No newline at end of file +exec /opt/oxide/clickhouse/clickhouse server "${args[@]}" & diff --git a/smf/clickhouse_keeper/manifest.xml b/smf/clickhouse_keeper/manifest.xml index ba97e78d8c..1b5ff3c0e1 100644 --- a/smf/clickhouse_keeper/manifest.xml +++ b/smf/clickhouse_keeper/manifest.xml @@ -4,7 +4,7 @@ - + @@ -26,12 +26,6 @@ timeout_seconds='0' /> - - - - - - diff --git a/smf/clickhouse_keeper/method_script.sh b/smf/clickhouse_keeper/method_script.sh index 74da9b2aee..e36ccb647a 100755 --- a/smf/clickhouse_keeper/method_script.sh +++ b/smf/clickhouse_keeper/method_script.sh @@ -6,102 +6,6 @@ set -o pipefail . /lib/svc/share/smf_include.sh -LISTEN_ADDR="$(svcprop -c -p config/listen_addr "${SMF_FMRI}")" -LISTEN_PORT="$(svcprop -c -p config/listen_port "${SMF_FMRI}")" -DATASTORE="$(svcprop -c -p config/store "${SMF_FMRI}")" - -# Retrieve hostnames (SRV records in internal DNS) of all keeper nodes. -K_ADDRS="$(/opt/oxide/internal-dns-cli/bin/dnswait clickhouse-keeper -H)" - -if [[ -z "$K_ADDRS" ]]; then - printf 'ERROR: found no hostnames for other ClickHouse Keeper nodes\n' >&2 - exit "$SMF_EXIT_ERR_CONFIG" -fi - -declare -a keepers=($K_ADDRS) - -for i in "${keepers[@]}" -do - if ! grep -q "host.control-plane.oxide.internal" <<< "${i}"; then - printf 'ERROR: retrieved ClickHouse Keeper hostname does not match the expected format\n' >&2 - exit "$SMF_EXIT_ERR_CONFIG" - fi -done - -if [[ "${#keepers[@]}" != 3 ]] -then - printf "ERROR: expected 3 ClickHouse Keeper hosts, found "${#keepers[@]}" instead\n" >&2 - exit "$SMF_EXIT_ERR_CONFIG" -fi - -# Assign hostnames to replicas and keeper nodes -KEEPER_HOST_01="${keepers[0]}" -KEEPER_HOST_02="${keepers[1]}" -KEEPER_HOST_03="${keepers[2]}" - -# Generate unique reproduceable number IDs by removing letters from -# KEEPER_IDENTIFIER_* Keeper IDs must be numbers, and they cannot be reused -# (i.e. when a keeper node is unrecoverable the ID must be changed to something -# new). By trimming the hosts we can make sure all keepers will always be up to -# date when a new keeper is spun up. Clickhouse does not allow very large -# numbers, so we will be reducing to 7 characters. This should be enough -# entropy given the small amount of keepers we have. -KEEPER_ID_01="$( echo "${KEEPER_HOST_01}" | tr -dc [:digit:] | cut -c1-7)" -KEEPER_ID_02="$( echo "${KEEPER_HOST_02}" | tr -dc [:digit:] | cut -c1-7)" -KEEPER_ID_03="$( echo "${KEEPER_HOST_03}" | tr -dc [:digit:] | cut -c1-7)" - -# Identify the node type this is as this will influence how the config is -# constructed -# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are -# probably much better ways to do this service name lookup, but this works for -# now. The services contain the same IDs as the hostnames. -KEEPER_SVC="$(zonename | tr -dc [:digit:] | cut -c1-7)" -if [[ $KEEPER_ID_01 == $KEEPER_SVC ]] -then - KEEPER_ID_CURRENT=$KEEPER_ID_01 -elif [[ $KEEPER_ID_02 == $KEEPER_SVC ]] -then - KEEPER_ID_CURRENT=$KEEPER_ID_02 -elif [[ $KEEPER_ID_03 == $KEEPER_SVC ]] -then - KEEPER_ID_CURRENT=$KEEPER_ID_03 -else - printf 'ERROR: service name does not match any of the identified ClickHouse Keeper hostnames\n' >&2 - exit "$SMF_EXIT_ERR_CONFIG" -fi - -curl -X put http://[${LISTEN_ADDR}]:8888/keeper/config \ --H "Content-Type: application/json" \ --d '{ - "generation": 0, - "settings": { - "id": '${KEEPER_ID_CURRENT}', - "raft_servers": [ - { - "id": '${KEEPER_ID_01}', - "host": { - "domain_name": "'${KEEPER_HOST_01}'" - } - }, - { - "id": '${KEEPER_ID_02}', - "host": { - "domain_name": "'${KEEPER_HOST_02}'" - } - }, - { - "id": '${KEEPER_ID_03}', - "host": { - "domain_name": "'${KEEPER_HOST_03}'" - } - } - ], - "config_dir": "/opt/oxide/clickhouse_keeper", - "datastore_path": "'${DATASTORE}'", - "listen_addr": "'${LISTEN_ADDR}'" - } -}' - # The clickhouse binary must be run from within the directory that contains it. # Otherwise, it does not automatically detect the configuration files, nor does # it append them when necessary diff --git a/smf/clickhouse_server/manifest.xml b/smf/clickhouse_server/manifest.xml index b32417d537..943344eba2 100644 --- a/smf/clickhouse_server/manifest.xml +++ b/smf/clickhouse_server/manifest.xml @@ -4,7 +4,7 @@ - + @@ -26,12 +26,6 @@ timeout_seconds='0' /> - - - - - - diff --git a/smf/clickhouse_server/method_script.sh b/smf/clickhouse_server/method_script.sh index 10a813166c..b4915e77fd 100755 --- a/smf/clickhouse_server/method_script.sh +++ b/smf/clickhouse_server/method_script.sh @@ -6,110 +6,8 @@ set -o pipefail . /lib/svc/share/smf_include.sh -LISTEN_ADDR="$(svcprop -c -p config/listen_addr "${SMF_FMRI}")" -LISTEN_PORT="$(svcprop -c -p config/listen_port "${SMF_FMRI}")" -DATASTORE="$(svcprop -c -p config/store "${SMF_FMRI}")" - -# Retrieve hostnames (SRV records in internal DNS) of the clickhouse nodes. -CH_ADDRS="$(/opt/oxide/internal-dns-cli/bin/dnswait clickhouse-server -H)" - -if [[ -z "$CH_ADDRS" ]]; then - printf 'ERROR: found no hostnames for other ClickHouse server nodes\n' >&2 - exit "$SMF_EXIT_ERR_CONFIG" -fi - -declare -a nodes=($CH_ADDRS) - -for i in "${nodes[@]}" -do - if ! grep -q "host.control-plane.oxide.internal" <<< "${i}"; then - printf 'ERROR: retrieved ClickHouse hostname does not match the expected format\n' >&2 - exit "$SMF_EXIT_ERR_CONFIG" - fi -done - -# Assign hostnames to replicas -REPLICA_HOST_01="${nodes[0]}" -REPLICA_HOST_02="${nodes[1]}" - -# Retrieve hostnames (SRV records in internal DNS) of the keeper nodes. -K_ADDRS="$(/opt/oxide/internal-dns-cli/bin/dnswait clickhouse-keeper -H)" - -if [[ -z "$K_ADDRS" ]]; then - printf 'ERROR: found no hostnames for other ClickHouse Keeper nodes\n' >&2 - exit "$SMF_EXIT_ERR_CONFIG" -fi - -declare -a keepers=($K_ADDRS) - -for i in "${keepers[@]}" -do - if ! grep -q "host.control-plane.oxide.internal" <<< "${i}"; then - printf 'ERROR: retrieved ClickHouse Keeper hostname does not match the expected format\n' >&2 - exit "$SMF_EXIT_ERR_CONFIG" - fi -done - -if [[ "${#keepers[@]}" != 3 ]] -then - printf "ERROR: expected 3 ClickHouse Keeper hosts, found "${#keepers[@]}" instead\n" >&2 - exit "$SMF_EXIT_ERR_CONFIG" -fi - -# Identify the node type this is as this will influence how the config is constructed -# TODO(https://github.com/oxidecomputer/omicron/issues/3824): There are probably much -# better ways to do this service discovery, but this works for now. -# The services contain the same IDs as the hostnames. -CLICKHOUSE_SVC="$(zonename | tr -dc [:digit:])" -REPLICA_IDENTIFIER_01="$( echo "${REPLICA_HOST_01}" | tr -dc [:digit:])" -REPLICA_IDENTIFIER_02="$( echo "${REPLICA_HOST_02}" | tr -dc [:digit:])" -if [[ $REPLICA_IDENTIFIER_01 == $CLICKHOUSE_SVC ]] -then - REPLICA_DISPLAY_NAME="oximeter_cluster node 1" - REPLICA_NUMBER=1 -elif [[ $REPLICA_IDENTIFIER_02 == $CLICKHOUSE_SVC ]] -then - REPLICA_DISPLAY_NAME="oximeter_cluster node 2" - REPLICA_NUMBER=2 -else - printf 'ERROR: service name does not match any of the identified ClickHouse hostnames\n' >&2 - exit "$SMF_EXIT_ERR_CONFIG" -fi - -curl -X put http://[${LISTEN_ADDR}]:8888/server/config \ - -H "Content-Type: application/json" \ - -d '{ - "generation": 0, - "settings": { - "id": '${REPLICA_NUMBER}', - "keepers": [ - { - "domain_name": "'${keepers[0]}'" - }, - { - "domain_name": "'${keepers[1]}'" - }, - { - "domain_name": "'${keepers[2]}'" - } - ], - "remote_servers": [ - { - "domain_name": "'${REPLICA_HOST_01}'" - }, - { - "domain_name": "'${REPLICA_HOST_02}'" - } - ], - "config_dir": "/opt/oxide/clickhouse_server/config.d", - "datastore_path": "'${DATASTORE}'", - "listen_addr": "'${LISTEN_ADDR}'" - } -}' - # The clickhouse binary must be run from within the directory that contains it. # Otherwise, it does not automatically detect the configuration files, nor does # it append them when necessary cd /opt/oxide/clickhouse_server/ - exec ./clickhouse server & \ No newline at end of file diff --git a/test-utils/src/dev/test_cmds.rs b/test-utils/src/dev/test_cmds.rs index 792ade0c53..220efa4628 100644 --- a/test-utils/src/dev/test_cmds.rs +++ b/test-utils/src/dev/test_cmds.rs @@ -125,7 +125,83 @@ pub fn error_for_enoent() -> String { /// invocation to invocation (e.g., assigned TCP port numbers, timestamps) /// /// This allows use to use expectorate to verify the shape of the CLI output. -pub fn redact_variable(input: &str) -> String { +#[derive(Clone, Debug)] +pub struct Redactor<'a> { + basic: bool, + uuids: bool, + extra: Vec<(&'a str, String)>, +} + +impl Default for Redactor<'_> { + fn default() -> Self { + Self { basic: true, uuids: true, extra: Vec::new() } + } +} + +impl<'a> Redactor<'a> { + /// Create a new redactor that does not do any redactions. + pub fn noop() -> Self { + Self { basic: false, uuids: false, extra: Vec::new() } + } + + pub fn basic(&mut self, basic: bool) -> &mut Self { + self.basic = basic; + self + } + + pub fn uuids(&mut self, uuids: bool) -> &mut Self { + self.uuids = uuids; + self + } + + pub fn extra_fixed_length( + &mut self, + name: &str, + text_to_redact: &'a str, + ) -> &mut Self { + // Use the same number of chars as the number of bytes in + // text_to_redact. We're almost entirely in ASCII-land so they're the + // same, and getting the length right is nice but doesn't matter for + // correctness. + // + // A technically more correct impl would use unicode-width, but ehhh. + let replacement = fill_redaction_text(name, text_to_redact.len()); + self.extra.push((text_to_redact, replacement)); + self + } + + pub fn extra_variable_length( + &mut self, + name: &str, + text_to_redact: &'a str, + ) -> &mut Self { + let replacement = format!("<{}_REDACTED>", name.to_uppercase()); + self.extra.push((text_to_redact, replacement)); + self + } + + pub fn do_redact(&self, input: &str) -> String { + // Perform extra redactions at the beginning, not the end. This is because + // some of the built-in redactions in redact_variable might match a + // substring of something that should be handled by extra_redactions (e.g. + // a temporary path). + let mut s = input.to_owned(); + for (name, replacement) in &self.extra { + s = s.replace(name, replacement); + } + + if self.basic { + s = redact_basic(&s); + } + if self.uuids { + s = redact_uuids(&s); + } + + s + } +} + +fn redact_basic(input: &str) -> String { // Replace TCP port numbers. We include the localhost // characters to avoid catching any random sequence of numbers. let s = regex::Regex::new(r"\[::1\]:\d{4,5}") @@ -141,19 +217,6 @@ pub fn redact_variable(input: &str) -> String { .replace_all(&s, "127.0.0.1:REDACTED_PORT") .to_string(); - // Replace uuids. - // - // The length of a UUID is 32 nibbles for the hex encoding of a u128 + 4 - // dashes = 36. - const UUID_LEN: usize = 36; - let s = regex::Regex::new( - "[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-\ - [a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}", - ) - .unwrap() - .replace_all(&s, fill_redaction_text("uuid", UUID_LEN)) - .to_string(); - // Replace timestamps. // // Format: RFC 3339 (ISO 8601) @@ -188,6 +251,12 @@ pub fn redact_variable(input: &str) -> String { .replace_all(&s, "ms") .to_string(); + // Replace interval (m). + let s = regex::Regex::new(r"\d+m") + .unwrap() + .replace_all(&s, "m") + .to_string(); + let s = regex::Regex::new( r"note: database schema version matches expected \(\d+\.\d+\.\d+\)", ) @@ -207,63 +276,14 @@ pub fn redact_variable(input: &str) -> String { s } -/// Redact text from a string, allowing for extra redactions to be specified. -pub fn redact_extra( - input: &str, - extra_redactions: &ExtraRedactions<'_>, -) -> String { - // Perform extra redactions at the beginning, not the end. This is because - // some of the built-in redactions in redact_variable might match a - // substring of something that should be handled by extra_redactions (e.g. - // a temporary path). - let mut s = input.to_owned(); - for (name, replacement) in &extra_redactions.redactions { - s = s.replace(name, replacement); - } - redact_variable(&s) -} - -/// Represents a list of extra redactions for [`redact_variable`]. -/// -/// Extra redactions are applied in-order, before any builtin redactions. -#[derive(Clone, Debug, Default)] -pub struct ExtraRedactions<'a> { - // A pair of redaction and replacement strings. - redactions: Vec<(&'a str, String)>, -} - -impl<'a> ExtraRedactions<'a> { - pub fn new() -> Self { - Self { redactions: Vec::new() } - } - - pub fn fixed_length( - &mut self, - name: &str, - text_to_redact: &'a str, - ) -> &mut Self { - // Use the same number of chars as the number of bytes in - // text_to_redact. We're almost entirely in ASCII-land so they're the - // same, and getting the length right is nice but doesn't matter for - // correctness. - // - // A technically more correct impl would use unicode-width, but ehhh. - let replacement = fill_redaction_text(name, text_to_redact.len()); - self.redactions.push((text_to_redact, replacement)); - self - } - - pub fn variable_length( - &mut self, - name: &str, - text_to_redact: &'a str, - ) -> &mut Self { - let gen = format!("<{}_REDACTED>", name.to_uppercase()); - let replacement = gen.to_string(); - - self.redactions.push((text_to_redact, replacement)); - self - } +fn redact_uuids(input: &str) -> String { + // The length of a UUID is 32 nibbles for the hex encoding of a u128 + 4 + // dashes = 36. + const UUID_LEN: usize = 36; + regex::Regex::new(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") + .unwrap() + .replace_all(&input, fill_redaction_text("uuid", UUID_LEN)) + .to_string() } fn fill_redaction_text(name: &str, text_to_redact_len: usize) -> String { @@ -303,13 +323,11 @@ mod tests { let input = "time: 123ms, path: /var/tmp/tmp.456ms123s, \ path2: /short, \ path3: /variable-length/path"; - let actual = redact_extra( - input, - ExtraRedactions::new() - .fixed_length("tp", "/var/tmp/tmp.456ms123s") - .fixed_length("short_redact", "/short") - .variable_length("variable", "/variable-length/path"), - ); + let actual = Redactor::default() + .extra_fixed_length("tp", "/var/tmp/tmp.456ms123s") + .extra_fixed_length("short_redact", "/short") + .extra_variable_length("variable", "/variable-length/path") + .do_redact(input); assert_eq!( actual, "time: ms, path: ........., \ @@ -341,7 +359,7 @@ mod tests { for time in times { let input = format!("{:?}", time); assert_eq!( - redact_variable(&input), + Redactor::default().do_redact(&input), "", "Failed to redact {:?}", time diff --git a/tools/console_version b/tools/console_version index 75ae9b6dd4..5e0e27111e 100644 --- a/tools/console_version +++ b/tools/console_version @@ -1,2 +1,2 @@ -COMMIT="5561f285a84e43d24d48089ac3eea38be98be551" -SHA2="9c1e901dd9b69f0cf5dce7aff0113ecba363d890908481b73659c9cfdc223049" +COMMIT="1733b764ac2dea21153634cf6287f9276b2499da" +SHA2="7d4794c583581123413f6e003f4d2ef338ed6e9c417d45d6dd10353ece375462" diff --git a/tools/dendrite_openapi_version b/tools/dendrite_openapi_version index 2069e45253..75c57e9d29 100755 --- a/tools/dendrite_openapi_version +++ b/tools/dendrite_openapi_version @@ -1,2 +1,2 @@ -COMMIT="8dab641b522375a4b403ae4bd0c9a22d905fae5d" +COMMIT="f3810e7bc1f0d746b5e95b3aaff32e52b02dfdfa" SHA2="3a54305ab4b1270c9a5fb0603f481fce199f3767c174a03559ff642f7f44687e" diff --git a/tools/dendrite_stub_checksums b/tools/dendrite_stub_checksums index d937a7f0c0..fe420e299e 100644 --- a/tools/dendrite_stub_checksums +++ b/tools/dendrite_stub_checksums @@ -1,3 +1,3 @@ -CIDL_SHA256_ILLUMOS="f848e54d5ea7ddcbc91b50d986dd7ee6617809690fb7ed6463a51fc2235786a8" -CIDL_SHA256_LINUX_DPD="3e068ab0a0593024cbba8efa4d9ee3b3e3242e8d24f5cd37d58dc38a7dc09eb0" -CIDL_SHA256_LINUX_SWADM="e1e35784538a4fdd76dc257cc636ac3f43f7ef2842dabfe981f17f8ce6b8e1a2" +CIDL_SHA256_ILLUMOS="c1506f6f818327523e6ff3102432a2038d319338b883235664b34f9132ff676a" +CIDL_SHA256_LINUX_DPD="fc9ea4dc22e761dce3aa4d252983360f799426a0c23ea8f347653664d3e2b55a" +CIDL_SHA256_LINUX_SWADM="9da0dd6c972206338971a90144b1c35e101d69aaacf26240a45cef45d828b090" diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 0c223c85a8..920e67ac5b 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="c92d6ff85db8992066f49da176cf686acfd8fe0f" +COMMIT="056283eb02b6887fbf27f66a215662520f7c159c" SHA2="007bfb717ccbc077c0250dee3121aeb0c5bb0d1c16795429a514fa4f8635a5ef" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 0db6a3b63d..0811251c05 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="c92d6ff85db8992066f49da176cf686acfd8fe0f" -SHA2="5b327f213f8f341cf9072d428980f53757b2c6383f684ac80bbccfb1984ffe5f" +COMMIT="056283eb02b6887fbf27f66a215662520f7c159c" +SHA2="28389b4a5fb5d9767b518aacdd09470135aefa2f6704a3b3fb05cd71b21613ae" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 2e180a83db..42465a5a34 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="e000485f7e04ac1cf9b3532b60bcf23598ab980331ba4f1c6788a7e95c1e9ef8" -MGD_LINUX_SHA256="1c3d93bbfbe4ce97af7cb81c13e42a2eea464e18de6827794a55d5bfd971b66c" \ No newline at end of file +CIDL_SHA256="7c10ac7d284ce78e70e652ad91bebf3fee7a2274ee403a09cc986c6ee73cf1eb" +MGD_LINUX_SHA256="be4c6f7375ff3e619102783513348e8769579bd011a85e33779b690759fd0868" diff --git a/tools/opte_version_override b/tools/opte_version_override index 8d57f7ae9f..5d1e38278a 100644 --- a/tools/opte_version_override +++ b/tools/opte_version_override @@ -2,4 +2,4 @@ # only set this if you want to override the version of opte/xde installed by the # install_opte.sh script -OPTE_COMMIT="" +OPTE_COMMIT="f3002b356da7d0e4ca15beb66a5566a92919baaa" diff --git a/tools/permslip_staging b/tools/permslip_staging index ea8185e4e0..62cf33bace 100644 --- a/tools/permslip_staging +++ b/tools/permslip_staging @@ -1,5 +1,5 @@ -0633c699265a0c0d982e9fbf196b807493bd9ee73fb9541b41a047bf439d94ab manifest-gimlet-v1.0.28.toml +ffb2be39e9bd1c5f2203d414c63e86afb8f820a8938119d7c6b29f9a8aa42c29 manifest-gimlet-v1.0.30 82f68363c5f89acb8f1e9f0fdec00694d84bd69291a63f9c3c3141721a42af9a manifest-oxide-rot-1-v1.0.28.toml -b6c3fdd7cae192e7f40c6a3b7615820ba2e0ca9e8b0c003648ab6367e2c98b8b manifest-psc-v1.0.27.toml -5dd8569d77df45bacf22a422f9ca5fd19f1e790a84d9dac9e067c3cdd9a41c77 manifest-sidecar-v1.0.28.toml +ce877624a26ac5b2a122816eda89316ea98c101cbfd61fbb4c393188a963c3de manifest-psc-v1.0.29.toml +81e9889178ce147d9aef6925c416e1e75c1b38e29a4daed7f7f287f86a9360b7 manifest-sidecar-v1.0.29.toml 6f8459afe22c27d5920356878e4d8d639464f39a15ce7b5b040c2d908d52a570 manifest-bootleby-v1.3.1.toml diff --git a/wicket-common/src/example.rs b/wicket-common/src/example.rs index 34af11e906..3951520f01 100644 --- a/wicket-common/src/example.rs +++ b/wicket-common/src/example.rs @@ -197,7 +197,7 @@ impl ExampleRackSetupData { destination: "0.0.0.0/0".parse().unwrap(), nexthop: "172.30.0.10".parse().unwrap(), vlan_id: Some(1), - local_pref: None, + rib_priority: None, }], bgp_peers: switch0_port0_bgp_peers, uplink_port_speed: PortSpeed::Speed400G, @@ -215,7 +215,7 @@ impl ExampleRackSetupData { destination: "0.0.0.0/0".parse().unwrap(), nexthop: "172.33.0.10".parse().unwrap(), vlan_id: Some(1), - local_pref: None, + rib_priority: None, }], bgp_peers: switch1_port0_bgp_peers, uplink_port_speed: PortSpeed::Speed400G, diff --git a/wicket/src/cli/rack_setup/config_toml.rs b/wicket/src/cli/rack_setup/config_toml.rs index 17b31e7730..92496e94d5 100644 --- a/wicket/src/cli/rack_setup/config_toml.rs +++ b/wicket/src/cli/rack_setup/config_toml.rs @@ -329,15 +329,15 @@ fn populate_uplink_table(cfg: &UserSpecifiedPortConfig) -> Table { // routes = [] let mut routes_out = Array::new(); for r in routes { - let RouteConfig { destination, nexthop, vlan_id, local_pref } = r; + let RouteConfig { destination, nexthop, vlan_id, rib_priority } = r; let mut route = InlineTable::new(); route.insert("nexthop", string_value(nexthop)); route.insert("destination", string_value(destination)); if let Some(vlan_id) = vlan_id { route.insert("vlan_id", i64_value(i64::from(*vlan_id))); } - if let Some(local_pref) = local_pref { - route.insert("local_pref", i64_value(i64::from(*local_pref))); + if let Some(rib_priority) = rib_priority { + route.insert("rib_priority", i64_value(i64::from(*rib_priority))); } routes_out.push(Value::InlineTable(route)); } diff --git a/wicket/src/ui/panes/rack_setup.rs b/wicket/src/ui/panes/rack_setup.rs index cbf66a1cf3..be299ef022 100644 --- a/wicket/src/ui/panes/rack_setup.rs +++ b/wicket/src/ui/panes/rack_setup.rs @@ -804,34 +804,39 @@ fn rss_config_text<'a>( ], ]; - let routes = routes.iter().map(|r| { - let RouteConfig { destination, nexthop, vlan_id, local_pref } = - r; - - let mut items = vec![ - Span::styled(" • Route : ", label_style), - Span::styled( - format!("{} -> {}", destination, nexthop), - ok_style, - ), - ]; - if let Some(vlan_id) = vlan_id { - items.extend([ - Span::styled(" (vlan_id=", label_style), - Span::styled(vlan_id.to_string(), ok_style), - Span::styled(")", label_style), - ]); - } - if let Some(local_pref) = local_pref { - items.extend([ - Span::styled(" (local_pref=", label_style), - Span::styled(local_pref.to_string(), ok_style), - Span::styled(")", label_style), - ]); - } + let routes = + routes.iter().map(|r| { + let RouteConfig { + destination, + nexthop, + vlan_id, + rib_priority, + } = r; + + let mut items = vec![ + Span::styled(" • Route : ", label_style), + Span::styled( + format!("{} -> {}", destination, nexthop), + ok_style, + ), + ]; + if let Some(vlan_id) = vlan_id { + items.extend([ + Span::styled(" (vlan_id=", label_style), + Span::styled(vlan_id.to_string(), ok_style), + Span::styled(")", label_style), + ]); + } + if let Some(rib_priority) = rib_priority { + items.extend([ + Span::styled(" (rib_priority=", label_style), + Span::styled(rib_priority.to_string(), ok_style), + Span::styled(")", label_style), + ]); + } - items - }); + items + }); let addresses = addresses.iter().map(|a| { let mut items = vec![ diff --git a/wicketd/Cargo.toml b/wicketd/Cargo.toml index adb0e43036..3145add700 100644 --- a/wicketd/Cargo.toml +++ b/wicketd/Cargo.toml @@ -31,7 +31,8 @@ http-body-util.workspace = true hubtools.workspace = true hyper.workspace = true illumos-utils.workspace = true -internal-dns.workspace = true +internal-dns-resolver.workspace = true +internal-dns-types.workspace = true itertools.workspace = true once_cell.workspace = true oxnet.workspace = true diff --git a/wicketd/src/context.rs b/wicketd/src/context.rs index 8f4dfb451b..307898200b 100644 --- a/wicketd/src/context.rs +++ b/wicketd/src/context.rs @@ -12,7 +12,7 @@ use crate::MgsHandle; use anyhow::anyhow; use anyhow::bail; use anyhow::Result; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; use sled_hardware_types::Baseboard; use slog::info; use std::net::Ipv6Addr; diff --git a/wicketd/src/http_entrypoints.rs b/wicketd/src/http_entrypoints.rs index 3f460f1e37..ada8422de4 100644 --- a/wicketd/src/http_entrypoints.rs +++ b/wicketd/src/http_entrypoints.rs @@ -20,7 +20,7 @@ use dropshot::RequestContext; use dropshot::StreamingBody; use dropshot::TypedBody; use http::StatusCode; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; use omicron_common::api::internal::shared::SwitchLocation; use omicron_uuid_kinds::RackInitUuid; use omicron_uuid_kinds::RackResetUuid; diff --git a/wicketd/src/lib.rs b/wicketd/src/lib.rs index 430b94985f..1ef7df610d 100644 --- a/wicketd/src/lib.rs +++ b/wicketd/src/lib.rs @@ -26,7 +26,7 @@ pub(crate) use context::ServerContext; use display_error_chain::DisplayErrorChain; use dropshot::{ConfigDropshot, HandlerTaskMode, HttpServer}; pub use installinator_progress::{IprUpdateTracker, RunningUpdateState}; -use internal_dns::resolver::Resolver; +use internal_dns_resolver::Resolver; use mgs::make_mgs_client; pub(crate) use mgs::{MgsHandle, MgsManager}; use nexus_proxy::NexusTcpProxy; diff --git a/wicketd/src/nexus_proxy.rs b/wicketd/src/nexus_proxy.rs index 33ff02a945..b55a449853 100644 --- a/wicketd/src/nexus_proxy.rs +++ b/wicketd/src/nexus_proxy.rs @@ -4,8 +4,8 @@ //! TCP proxy to expose Nexus's external API via the techport. -use internal_dns::resolver::Resolver; -use internal_dns::ServiceName; +use internal_dns_resolver::Resolver; +use internal_dns_types::names::ServiceName; use omicron_common::address::NEXUS_TECHPORT_EXTERNAL_PORT; use slog::info; use slog::o; diff --git a/wicketd/src/rss_config.rs b/wicketd/src/rss_config.rs index 46ede25eaa..2be098dbc2 100644 --- a/wicketd/src/rss_config.rs +++ b/wicketd/src/rss_config.rs @@ -716,7 +716,7 @@ fn build_port_config( destination: r.destination, nexthop: r.nexthop, vlan_id: r.vlan_id, - local_pref: r.local_pref, + rib_priority: r.rib_priority, }) .collect(), addresses: config